diff mbox series

[FFmpeg-devel,v16,08/16] fftools/ffmpeg: Replace sub2video with subtitle frame filtering

Message ID DM8P223MB03659FC04F4B2E3C5B893E33BA629@DM8P223MB0365.NAMP223.PROD.OUTLOOK.COM
State Superseded, archived
Headers show
Series [FFmpeg-devel,v16,01/16] global: Prepare AVFrame for subtitle handling | expand

Checks

Context Check Description
andriy/make_x86 success Make finished
andriy/make_fate_x86 success Make fate finished
andriy/make_ppc success Make finished
andriy/make_fate_ppc success Make fate finished

Commit Message

Soft Works Nov. 25, 2021, 5:53 p.m. UTC 
This commit actually enables subtitle filtering in ffmpeg by
sending and receiving subtitle frames to and from a filtergraph.

The heartbeat functionality from the previous sub2video implementation
is retained and applied to all subtitle frames (bitmap, text, ..).

The other part of sub2video functionality is retained by
auto-insertion of the new graphicsub2video filter.

Justification for changed test refs:

- sub2video
  The new results are identical excepting the last frame which
  is due to the implementation changes

- sub2video_basic
  The previous results had some incorrect output because multiple
  frames had the same dts
  The non-empty content frames are visually identical, the different
  CRC is due to the different blending algorithm that is being used.

- sub2video_time_limited
  The third frame in the previous ref was a repetition, which doesn't
  happen anymore with the new subtitle filtering.

- sub-dvb
  Running ffprobe -show_frames on the source file shows that there
  are 7 subtitle frames with 0 rects in the source at the start
  and 2 at the end. This translates to the 14 and 4 additional
  entries in the new test results.

- filter-overlay-dvdsub-2397
  Overlay results have slightly different CRCs due to different
  blending implementation

Signed-off-by: softworkz <softworkz@hotmail.com>
---
 fftools/ffmpeg.c                          | 523 +++++++++++-----------
 fftools/ffmpeg.h                          |  15 +-
 fftools/ffmpeg_filter.c                   | 217 ++++++---
 fftools/ffmpeg_hw.c                       |   2 +-
 fftools/ffmpeg_opt.c                      |   3 +-
 tests/ref/fate/filter-overlay-dvdsub-2397 | 181 ++++----
 tests/ref/fate/sub-dvb                    | 162 ++++---
 tests/ref/fate/sub2video                  | 116 ++---
 tests/ref/fate/sub2video_basic            | 135 ++----
 tests/ref/fate/sub2video_time_limited     |   4 +-
 10 files changed, 684 insertions(+), 674 deletions(-)

Comments

Andreas Rheinhardt Nov. 26, 2021, 1:02 p.m. UTC | #1 
Soft Works:
> This commit actually enables subtitle filtering in ffmpeg by
> sending and receiving subtitle frames to and from a filtergraph.
> 
> The heartbeat functionality from the previous sub2video implementation
> is retained and applied to all subtitle frames (bitmap, text, ..).
> 
> The other part of sub2video functionality is retained by
> auto-insertion of the new graphicsub2video filter.
> 
> Justification for changed test refs:
> 
> - sub2video
>   The new results are identical excepting the last frame which
>   is due to the implementation changes
> 
> - sub2video_basic
>   The previous results had some incorrect output because multiple
>   frames had the same dts
>   The non-empty content frames are visually identical, the different
>   CRC is due to the different blending algorithm that is being used.
> 
> - sub2video_time_limited
>   The third frame in the previous ref was a repetition, which doesn't
>   happen anymore with the new subtitle filtering.
> 
> - sub-dvb
>   Running ffprobe -show_frames on the source file shows that there
>   are 7 subtitle frames with 0 rects in the source at the start
>   and 2 at the end. This translates to the 14 and 4 additional
>   entries in the new test results.
> 
> - filter-overlay-dvdsub-2397
>   Overlay results have slightly different CRCs due to different
>   blending implementation
> 
> Signed-off-by: softworkz <softworkz@hotmail.com>
> ---
>  fftools/ffmpeg.c                          | 523 +++++++++++-----------
>  fftools/ffmpeg.h                          |  15 +-
>  fftools/ffmpeg_filter.c                   | 217 ++++++---
>  fftools/ffmpeg_hw.c                       |   2 +-
>  fftools/ffmpeg_opt.c                      |   3 +-
>  tests/ref/fate/filter-overlay-dvdsub-2397 | 181 ++++----
>  tests/ref/fate/sub-dvb                    | 162 ++++---
>  tests/ref/fate/sub2video                  | 116 ++---
>  tests/ref/fate/sub2video_basic            | 135 ++----
>  tests/ref/fate/sub2video_time_limited     |   4 +-
>  10 files changed, 684 insertions(+), 674 deletions(-)
> 
> diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c
> index 3761ea0c38..c697c12777 100644
> --- a/fftools/ffmpeg.c
> +++ b/fftools/ffmpeg.c
> @@ -169,163 +169,6 @@ static int restore_tty;
>  static void free_input_threads(void);
>  #endif
>  
> -/* sub2video hack:
> -   Convert subtitles to video with alpha to insert them in filter graphs.
> -   This is a temporary solution until libavfilter gets real subtitles support.
> - */
> -
> -static int sub2video_get_blank_frame(InputStream *ist)
> -{
> -    int ret;
> -    AVFrame *frame = ist->sub2video.frame;
> -
> -    av_frame_unref(frame);
> -    ist->sub2video.frame->width  = ist->dec_ctx->width  ? ist->dec_ctx->width  : ist->sub2video.w;
> -    ist->sub2video.frame->height = ist->dec_ctx->height ? ist->dec_ctx->height : ist->sub2video.h;
> -    ist->sub2video.frame->format = AV_PIX_FMT_RGB32;
> -    if ((ret = av_frame_get_buffer(frame, 0)) < 0)
> -        return ret;
> -    memset(frame->data[0], 0, frame->height * frame->linesize[0]);
> -    return 0;
> -}
> -
> -static void sub2video_copy_rect(uint8_t *dst, int dst_linesize, int w, int h,
> -                                AVSubtitleRect *r)
> -{
> -    uint32_t *pal, *dst2;
> -    uint8_t *src, *src2;
> -    int x, y;
> -
> -    if (r->type != SUBTITLE_BITMAP) {
> -        av_log(NULL, AV_LOG_WARNING, "sub2video: non-bitmap subtitle\n");
> -        return;
> -    }
> -    if (r->x < 0 || r->x + r->w > w || r->y < 0 || r->y + r->h > h) {
> -        av_log(NULL, AV_LOG_WARNING, "sub2video: rectangle (%d %d %d %d) overflowing %d %d\n",
> -            r->x, r->y, r->w, r->h, w, h
> -        );
> -        return;
> -    }
> -
> -    dst += r->y * dst_linesize + r->x * 4;
> -    src = r->data[0];
> -    pal = (uint32_t *)r->data[1];
> -    for (y = 0; y < r->h; y++) {
> -        dst2 = (uint32_t *)dst;
> -        src2 = src;
> -        for (x = 0; x < r->w; x++)
> -            *(dst2++) = pal[*(src2++)];
> -        dst += dst_linesize;
> -        src += r->linesize[0];
> -    }
> -}
> -
> -static void sub2video_push_ref(InputStream *ist, int64_t pts)
> -{
> -    AVFrame *frame = ist->sub2video.frame;
> -    int i;
> -    int ret;
> -
> -    av_assert1(frame->data[0]);
> -    ist->sub2video.last_pts = frame->pts = pts;
> -    for (i = 0; i < ist->nb_filters; i++) {
> -        ret = av_buffersrc_add_frame_flags(ist->filters[i]->filter, frame,
> -                                           AV_BUFFERSRC_FLAG_KEEP_REF |
> -                                           AV_BUFFERSRC_FLAG_PUSH);
> -        if (ret != AVERROR_EOF && ret < 0)
> -            av_log(NULL, AV_LOG_WARNING, "Error while add the frame to buffer source(%s).\n",
> -                   av_err2str(ret));
> -    }
> -}
> -
> -void sub2video_update(InputStream *ist, int64_t heartbeat_pts, AVSubtitle *sub)
> -{
> -    AVFrame *frame = ist->sub2video.frame;
> -    int8_t *dst;
> -    int     dst_linesize;
> -    int num_rects, i;
> -    int64_t pts, end_pts;
> -
> -    if (!frame)
> -        return;
> -    if (sub) {
> -        pts       = av_rescale_q(sub->pts + sub->start_display_time * 1000LL,
> -                                 AV_TIME_BASE_Q, ist->st->time_base);
> -        end_pts   = av_rescale_q(sub->pts + sub->end_display_time   * 1000LL,
> -                                 AV_TIME_BASE_Q, ist->st->time_base);
> -        num_rects = sub->num_rects;
> -    } else {
> -        /* If we are initializing the system, utilize current heartbeat
> -           PTS as the start time, and show until the following subpicture
> -           is received. Otherwise, utilize the previous subpicture's end time
> -           as the fall-back value. */
> -        pts       = ist->sub2video.initialize ?
> -                    heartbeat_pts : ist->sub2video.end_pts;
> -        end_pts   = INT64_MAX;
> -        num_rects = 0;
> -    }
> -    if (sub2video_get_blank_frame(ist) < 0) {
> -        av_log(ist->dec_ctx, AV_LOG_ERROR,
> -               "Impossible to get a blank canvas.\n");
> -        return;
> -    }
> -    dst          = frame->data    [0];
> -    dst_linesize = frame->linesize[0];
> -    for (i = 0; i < num_rects; i++)
> -        sub2video_copy_rect(dst, dst_linesize, frame->width, frame->height, sub->rects[i]);
> -    sub2video_push_ref(ist, pts);
> -    ist->sub2video.end_pts = end_pts;
> -    ist->sub2video.initialize = 0;
> -}
> -
> -static void sub2video_heartbeat(InputStream *ist, int64_t pts)
> -{
> -    InputFile *infile = input_files[ist->file_index];
> -    int i, j, nb_reqs;
> -    int64_t pts2;
> -
> -    /* When a frame is read from a file, examine all sub2video streams in
> -       the same file and send the sub2video frame again. Otherwise, decoded
> -       video frames could be accumulating in the filter graph while a filter
> -       (possibly overlay) is desperately waiting for a subtitle frame. */
> -    for (i = 0; i < infile->nb_streams; i++) {
> -        InputStream *ist2 = input_streams[infile->ist_index + i];
> -        if (!ist2->sub2video.frame)
> -            continue;
> -        /* subtitles seem to be usually muxed ahead of other streams;
> -           if not, subtracting a larger time here is necessary */
> -        pts2 = av_rescale_q(pts, ist->st->time_base, ist2->st->time_base) - 1;
> -        /* do not send the heartbeat frame if the subtitle is already ahead */
> -        if (pts2 <= ist2->sub2video.last_pts)
> -            continue;
> -        if (pts2 >= ist2->sub2video.end_pts || ist2->sub2video.initialize)
> -            /* if we have hit the end of the current displayed subpicture,
> -               or if we need to initialize the system, update the
> -               overlayed subpicture and its start/end times */
> -            sub2video_update(ist2, pts2 + 1, NULL);
> -        for (j = 0, nb_reqs = 0; j < ist2->nb_filters; j++)
> -            nb_reqs += av_buffersrc_get_nb_failed_requests(ist2->filters[j]->filter);
> -        if (nb_reqs)
> -            sub2video_push_ref(ist2, pts2);
> -    }
> -}
> -
> -static void sub2video_flush(InputStream *ist)
> -{
> -    int i;
> -    int ret;
> -
> -    if (ist->sub2video.end_pts < INT64_MAX)
> -        sub2video_update(ist, INT64_MAX, NULL);
> -    for (i = 0; i < ist->nb_filters; i++) {
> -        ret = av_buffersrc_add_frame(ist->filters[i]->filter, NULL);
> -        if (ret != AVERROR_EOF && ret < 0)
> -            av_log(NULL, AV_LOG_WARNING, "Flush the frame error.\n");
> -    }
> -}
> -
> -/* end of sub2video hack */
> -
>  static void term_exit_sigsafe(void)
>  {
>  #if HAVE_TERMIOS_H
> @@ -526,7 +369,6 @@ static void ffmpeg_cleanup(int ret)
>          avfilter_graph_free(&fg->graph);
>          for (j = 0; j < fg->nb_inputs; j++) {
>              InputFilter *ifilter = fg->inputs[j];
> -            struct InputStream *ist = ifilter->ist;
>  
>              while (av_fifo_size(ifilter->frame_queue)) {
>                  AVFrame *frame;
> @@ -536,15 +378,6 @@ static void ffmpeg_cleanup(int ret)
>              }
>              av_fifo_freep(&ifilter->frame_queue);
>              av_freep(&ifilter->displaymatrix);
> -            if (ist->sub2video.sub_queue) {
> -                while (av_fifo_size(ist->sub2video.sub_queue)) {
> -                    AVSubtitle sub;
> -                    av_fifo_generic_read(ist->sub2video.sub_queue,
> -                                         &sub, sizeof(sub), NULL);
> -                    avsubtitle_free(&sub);
> -                }
> -                av_fifo_freep(&ist->sub2video.sub_queue);
> -            }
>              av_buffer_unref(&ifilter->hw_frames_ctx);
>              av_freep(&ifilter->name);
>              av_freep(&fg->inputs[j]);
> @@ -635,12 +468,14 @@ static void ffmpeg_cleanup(int ret)
>          av_frame_free(&ist->decoded_frame);
>          av_packet_free(&ist->pkt);
>          av_dict_free(&ist->decoder_opts);
> -        avsubtitle_free(&ist->prev_sub.subtitle);
> -        av_frame_free(&ist->sub2video.frame);
> +        av_frame_free(&ist->prev_sub.subtitle);
>          av_freep(&ist->filters);
>          av_freep(&ist->hwaccel_device);
>          av_freep(&ist->dts_buffer);
>  
> +        av_frame_free(&ist->subtitle_heartbeat.recent_sub);
> +        av_buffer_unref(&ist->subtitle_heartbeat.header);
> +
>          avcodec_free_context(&ist->dec_ctx);
>  
>          av_freep(&input_streams[i]);
> @@ -1058,17 +893,21 @@ error:
>      exit_program(1);
>  }
>  
> -static void do_subtitle_out(OutputFile *of,
> -                            OutputStream *ost,
> -                            AVSubtitle *sub)
> +static void do_subtitle_out(OutputFile *of, OutputStream *ost, AVFrame *frame)
>  {
> -    int subtitle_out_max_size = 1024 * 1024;
> +    const int subtitle_out_max_size = 1024 * 1024;
>      int subtitle_out_size, nb, i;
>      AVCodecContext *enc;
>      AVPacket *pkt = ost->pkt;
> +    AVSubtitle out_sub = { 0 };

You are adding some stuff here which is removed in patch 16. These
patches should be merged.

>      int64_t pts;
>  
> -    if (sub->pts == AV_NOPTS_VALUE) {
> +    if (!frame)
> +        return;
> +
> +    av_log(NULL, AV_LOG_TRACE, "do_subtitle_out: sub->pts: %"PRId64"  frame->pts: %"PRId64"\n", frame->subtitle_pts, frame->pts);
> +
> +    if (frame->subtitle_pts == AV_NOPTS_VALUE) {
>          av_log(NULL, AV_LOG_ERROR, "Subtitle packets must have a pts\n");
>          if (exit_on_error)
>              exit_program(1);
> @@ -1094,51 +933,58 @@ static void do_subtitle_out(OutputFile *of,
>          nb = 1;
>  
>      /* shift timestamp to honor -ss and make check_recording_time() work with -t */
> -    pts = sub->pts;
> +    pts = frame->subtitle_pts;
>      if (output_files[ost->file_index]->start_time != AV_NOPTS_VALUE)
>          pts -= output_files[ost->file_index]->start_time;
> +
> +    ost->sync_opts = av_rescale_q(pts, AV_TIME_BASE_Q, enc->time_base);
> +    if (!check_recording_time(ost))
> +        return;
> +
> +    frame->subtitle_pts = pts;
> +    // subtitle_start_time is required to be 0
> +    frame->subtitle_pts               += av_rescale_q(frame->subtitle_start_time, (AVRational){ 1, 1000 }, AV_TIME_BASE_Q);
> +    frame->subtitle_end_time  -= frame->subtitle_start_time;
> +    frame->subtitle_start_time = 0;
> +
> +    av_frame_get_subtitle(&out_sub, frame);
> +
>      for (i = 0; i < nb; i++) {
> -        unsigned save_num_rects = sub->num_rects;
> +        const unsigned save_num_rects = out_sub.num_rects;
>  
> -        ost->sync_opts = av_rescale_q(pts, AV_TIME_BASE_Q, enc->time_base);
> -        if (!check_recording_time(ost))
> -            return;
> +        ost->frames_encoded++;
>  
> -        sub->pts = pts;
> -        // start_display_time is required to be 0
> -        sub->pts               += av_rescale_q(sub->start_display_time, (AVRational){ 1, 1000 }, AV_TIME_BASE_Q);
> -        sub->end_display_time  -= sub->start_display_time;
> -        sub->start_display_time = 0;
>          if (i == 1)
> -            sub->num_rects = 0;
> +            out_sub.num_rects = 0;
>  
> -        ost->frames_encoded++;
> +        subtitle_out_size = avcodec_encode_subtitle(enc, subtitle_out, subtitle_out_max_size, &out_sub);
>  
> -        subtitle_out_size = avcodec_encode_subtitle(enc, subtitle_out,
> -                                                    subtitle_out_max_size, sub);
>          if (i == 1)
> -            sub->num_rects = save_num_rects;
> +            out_sub.num_rects = save_num_rects;
> +
>          if (subtitle_out_size < 0) {
>              av_log(NULL, AV_LOG_FATAL, "Subtitle encoding failed\n");
>              exit_program(1);
>          }
>  
> -        av_packet_unref(pkt);
> +        //av_packet_unref(pkt);
>          pkt->data = subtitle_out;
>          pkt->size = subtitle_out_size;
> -        pkt->pts  = av_rescale_q(sub->pts, AV_TIME_BASE_Q, ost->mux_timebase);
> -        pkt->duration = av_rescale_q(sub->end_display_time, (AVRational){ 1, 1000 }, ost->mux_timebase);
> +        pkt->pts  = av_rescale_q(frame->subtitle_pts, AV_TIME_BASE_Q, ost->mux_timebase);
> +        pkt->duration = av_rescale_q(frame->subtitle_end_time, (AVRational){ 1, 1000 }, ost->mux_timebase);
>          if (enc->codec_id == AV_CODEC_ID_DVB_SUBTITLE) {
>              /* XXX: the pts correction is handled here. Maybe handling
>                 it in the codec would be better */
>              if (i == 0)
> -                pkt->pts += av_rescale_q(sub->start_display_time, (AVRational){ 1, 1000 }, ost->mux_timebase);
> +                pkt->pts += av_rescale_q(frame->subtitle_start_time, (AVRational){ 1, 1000 }, ost->mux_timebase);
>              else
> -                pkt->pts += av_rescale_q(sub->end_display_time, (AVRational){ 1, 1000 }, ost->mux_timebase);
> +                pkt->pts += av_rescale_q(frame->subtitle_end_time, (AVRational){ 1, 1000 }, ost->mux_timebase);
>          }
>          pkt->dts = pkt->pts;
>          output_packet(of, pkt, ost, 0);
>      }
> +
> +    avsubtitle_free(&out_sub);
>  }
>  
>  static void do_video_out(OutputFile *of,
> @@ -1568,8 +1414,26 @@ static int reap_filters(int flush)
>                  }
>                  do_audio_out(of, ost, filtered_frame);
>                  break;
> +            case AVMEDIA_TYPE_SUBTITLE:
> +
> +                if (filtered_frame->format == AV_SUBTITLE_FMT_ASS && !enc->subtitle_header 
> +                    && filtered_frame->subtitle_header) {
> +                    const char *subtitle_header = (char *)filtered_frame->subtitle_header->data;
> +                    enc->subtitle_header = (uint8_t *)av_strdup(subtitle_header);
> +                    if (!enc->subtitle_header)
> +                        return AVERROR(ENOMEM);
> +                    enc->subtitle_header_size = strlen(subtitle_header);
> +                }
> +
> +                if ((ost->enc_ctx->width == 0 || ost->enc_ctx->height == 0)
> +                    && filter->inputs[0]->w > 0 && filter->inputs[0]->h > 0 ) {
> +                    ost->enc_ctx->width = filter->inputs[0]->w;
> +                    ost->enc_ctx->height = filter->inputs[0]->h;
> +                }
> +                    
> +                do_subtitle_out(of, ost, filtered_frame);
> +                break;
>              default:
> -                // TODO support subtitle filters
>                  av_assert0(0);
>              }
>  
> @@ -1972,6 +1836,9 @@ static void flush_encoders(void)
>              AVPacket *pkt = ost->pkt;
>              int pkt_size;
>  
> +            if (!pkt)
> +                break;
> +

This seems like a rebase error: I added it in
fb215798c7a72b32e889b72efd018f26bb3f88ce and removed it in
21914e7a4e802772cc9cdeec3eec8b30da4fa95a (because it was no longer
necessary).

>              switch (enc->codec_type) {
>              case AVMEDIA_TYPE_AUDIO:
>                  desc   = "audio";
> @@ -2164,7 +2031,8 @@ static int ifilter_has_all_input_formats(FilterGraph *fg)
>      int i;
>      for (i = 0; i < fg->nb_inputs; i++) {
>          if (fg->inputs[i]->format < 0 && (fg->inputs[i]->type == AVMEDIA_TYPE_AUDIO ||
> -                                          fg->inputs[i]->type == AVMEDIA_TYPE_VIDEO))
> +                                          fg->inputs[i]->type == AVMEDIA_TYPE_VIDEO ||
> +                                          fg->inputs[i]->type == AVMEDIA_TYPE_SUBTITLE))
>              return 0;
>      }
>      return 1;
> @@ -2269,7 +2137,7 @@ static int ifilter_send_eof(InputFilter *ifilter, int64_t pts)
>          // the filtergraph was never configured
>          if (ifilter->format < 0)
>              ifilter_parameters_from_codecpar(ifilter, ifilter->ist->st->codecpar);
> -        if (ifilter->format < 0 && (ifilter->type == AVMEDIA_TYPE_AUDIO || ifilter->type == AVMEDIA_TYPE_VIDEO)) {
> +        if (ifilter->format < 0 && (ifilter->type == AVMEDIA_TYPE_AUDIO || ifilter->type == AVMEDIA_TYPE_VIDEO || ifilter->type == AVMEDIA_TYPE_SUBTITLE)) {
>              av_log(NULL, AV_LOG_ERROR, "Cannot determine format of input stream %d:%d after EOF\n", ifilter->ist->file_index, ifilter->ist->st->index);
>              return AVERROR_INVALIDDATA;
>          }
> @@ -2307,7 +2175,8 @@ static int decode(AVCodecContext *avctx, AVFrame *frame, int *got_frame, AVPacke
>  
>  static int send_frame_to_filters(InputStream *ist, AVFrame *decoded_frame)
>  {
> -    int i, ret;
> +    int i, ret = 0;
> +    AVFrame *f;

Unused. Probably a rebase error as it has been removed in
a132614bba247afac30d3a8b1378c40bd7f672bc.

>  
>      av_assert1(ist->nb_filters > 0); /* ensure ret is initialized */
>      for (i = 0; i < ist->nb_filters; i++) {
> @@ -2508,81 +2377,214 @@ fail:
>      return err < 0 ? err : ret;
>  }
>  
> -static int transcode_subtitles(InputStream *ist, AVPacket *pkt, int *got_output,
> +static void subtitle_resend_current(InputStream *ist, int64_t heartbeat_pts)
> +{
> +    AVFrame *frame;
> +    int64_t pts, end_pts;
> +
> +    if (ist->subtitle_heartbeat.recent_sub) {
> +        frame = av_frame_clone(ist->subtitle_heartbeat.recent_sub);

Unchecked allocation.

> +
> +        pts     = heartbeat_pts; //av_rescale_q(frame->subtitle_pts + frame->subtitle_start_time * 1000LL, AV_TIME_BASE_Q, ist->st->time_base);
> +        end_pts = av_rescale_q(frame->subtitle_pts + frame->subtitle_end_time   * 1000LL, AV_TIME_BASE_Q, ist->st->time_base);
> +    }
> +    else {

Put this on the same line as }.

> +        frame = av_frame_alloc();

Is it actually certain that we need a new frame and can't reuse
decoded_frame like the other functions that ultimately call
send_frame_to_filters() do?

> +        if (!frame) {
> +            av_log(ist->dec_ctx, AV_LOG_ERROR, "Unable to alloc frame (out of memory).\n");
> +            return;
> +        }
> +
> +        frame->type = AVMEDIA_TYPE_SUBTITLE;
> +        frame->format = av_get_subtitle_format_from_codecdesc(ist->dec_ctx->codec_descriptor);
> +
> +        av_frame_get_buffer2(frame, 0);
> +
> +        frame->width = ist->subtitle_heartbeat.w;
> +        frame->height = ist->subtitle_heartbeat.h;
> +
> +        pts       = (ist->subtitle_heartbeat.end_pts < INT64_MAX && ist->subtitle_heartbeat.end_pts > 0)
> +                    ? ist->subtitle_heartbeat.end_pts : heartbeat_pts;
> +        end_pts   = INT64_MAX;
> +
> +        frame->subtitle_pts = av_rescale_q(pts, ist->st->time_base, AV_TIME_BASE_Q);
> +        frame->subtitle_end_time = 1000;
> +    }
> +
> +    ////av_log(NULL, AV_LOG_WARNING, "subtitle_heartbeat: call subtitle_resend_current %"PRId64" \n", pts);

Don't add outcommented logging stuff.

> +
> +    frame->pts = pts;
> +    ist->subtitle_heartbeat.last_pts = pts;
> +    ist->subtitle_heartbeat.end_pts = end_pts;
> +
> +    send_frame_to_filters(ist, frame);

frame will leak here (the AVFrame certainly leaks; in some scenarios
(like error conditions) the AVFrame and its buffers/side data/metadata
leaks).

> +}
> +
> +static void subtitle_heartbeat(InputStream *ist, int64_t pts)
> +{
> +    int i;
> +    int64_t pts2;
> +
> +    if (ist->st->codecpar->codec_type != AVMEDIA_TYPE_VIDEO)
> +        return;
> +
> +    /* When a frame is read from a file, examine all subtitle streams in
> +       the same file and send the subtitle frame again. Otherwise, decoded
> +       video frames could be accumulating in the filter graph while a filter
> +       (possibly overlay) is desperately waiting for a subtitle frame. */
> +    for (i = 0; i < nb_input_streams; i++) {
> +        InputStream *ist2 = input_streams[i];
> +        if (!ist2->subtitle_heartbeat.is_active)
> +            continue;
> +        /* subtitles seem to be usually muxed ahead of other streams;
> +           if not, subtracting a larger time here is necessary */
> +        pts2 = av_rescale_q(pts, ist->st->time_base, ist2->st->time_base) - 1;
> +        /* do not send the heartbeat frame if the subtitle is already ahead */
> +        if (pts2 <= ist2->subtitle_heartbeat.last_pts)
> +            continue;
> +        if (pts2 >= ist2->subtitle_heartbeat.end_pts) {
> +            /* if we have hit the end of the current displayed subpicture,
> +               or if we need to initialize the system, update the
> +               overlayed subpicture and its start/end times */
> +            if (ist2->subtitle_heartbeat.recent_sub)
> +                av_frame_free(&ist2->subtitle_heartbeat.recent_sub);
> +
> +            av_log(NULL, AV_LOG_DEBUG, "subtitle_heartbeat: clear + resend - pts: %"PRIi64"\n", pts2 + 1);
> +            subtitle_resend_current(ist2, pts2 + 1);
> +            continue;
> +        }
> +        if (!ist2->subtitle_heartbeat.check_buffer_requests) {
> +            unsigned j, nb_reqs;
> +            for (j = 0, nb_reqs = 0; j < ist2->nb_filters; j++)
> +                nb_reqs += av_buffersrc_get_nb_failed_requests(ist2->filters[j]->filter);
> +            if (nb_reqs) {
> +                av_log(NULL, AV_LOG_DEBUG, "subtitle_heartbeat: resend - pts: %"PRIi64"\n", pts2);
> +                subtitle_resend_current(ist2, pts2);
> +            }
> +        }
> +    }
> +}
> +
> +static InputStream *get_input_stream(OutputStream *ost)
> +{
> +    if (ost->source_index >= 0)
> +        return input_streams[ost->source_index];
> +    return NULL;
> +}
> +
> +static int decode_subtitles(InputStream *ist, AVPacket *pkt, int *got_output,
>                                 int *decode_failed)
>  {
> -    AVSubtitle subtitle;
> -    int free_sub = 1;
> -    int i, ret = avcodec_decode_subtitle2(ist->dec_ctx,
> -                                          &subtitle, got_output, pkt);
> +    AVFrame *decoded_frame;
> +    AVCodecContext *avctx = ist->dec_ctx;
> +    int i = 0, ret = 0, err = 0;
> +    int64_t pts, end_pts;
> +
> +    if (!ist->decoded_frame && !(ist->decoded_frame = av_frame_alloc()))
> +        return AVERROR(ENOMEM);
> +    decoded_frame = ist->decoded_frame;
> +    decoded_frame->type = AVMEDIA_TYPE_SUBTITLE;
> +    decoded_frame->format = av_get_subtitle_format_from_codecdesc(avctx->codec_descriptor);
>  
> -    check_decode_result(NULL, got_output, ret);
> +    if (!ist->subtitle_heartbeat.header && avctx->subtitle_header && avctx->subtitle_header_size > 0) {
> +        ist->subtitle_heartbeat.header = av_buffer_allocz(avctx->subtitle_header_size + 1);
> +        if (!ist->subtitle_heartbeat.header)
> +            return AVERROR(ENOMEM);
> +        memcpy(ist->subtitle_heartbeat.header->data, avctx->subtitle_header, avctx->subtitle_header_size);
> +    }
> +
> +    ret = decode(avctx, decoded_frame, got_output, pkt);
> +
> +    if (ret != AVERROR_EOF)
> +        check_decode_result(NULL, got_output, ret);
>  
>      if (ret < 0 || !*got_output) {
>          *decode_failed = 1;
> -        if (!pkt->size)
> -            sub2video_flush(ist);
> +        if (!pkt->size) {
> +            // Flush
> +            for (i = 0; i < ist->nb_filters; i++) {
> +                ret = av_buffersrc_add_frame(ist->filters[i]->filter, NULL);
> +                if (ret != AVERROR_EOF && ret < 0)
> +                    av_log(NULL, AV_LOG_WARNING, "Flush the frame error.\n");
> +            }
> +        }
>          return ret;
>      }
>  
>      if (ist->fix_sub_duration) {
>          int end = 1;
> -        if (ist->prev_sub.got_output) {
> -            end = av_rescale(subtitle.pts - ist->prev_sub.subtitle.pts,
> +        if (ist->prev_sub.got_output && ist->prev_sub.subtitle) {
> +            end = av_rescale(decoded_frame->subtitle_pts - ist->prev_sub.subtitle->subtitle_pts,
>                               1000, AV_TIME_BASE);
> -            if (end < ist->prev_sub.subtitle.end_display_time) {
> -                av_log(ist->dec_ctx, AV_LOG_DEBUG,
> +            if (end < ist->prev_sub.subtitle->subtitle_end_time) {
> +                av_log(avctx, AV_LOG_DEBUG,
>                         "Subtitle duration reduced from %"PRId32" to %d%s\n",
> -                       ist->prev_sub.subtitle.end_display_time, end,
> +                       ist->prev_sub.subtitle->subtitle_end_time, end,
>                         end <= 0 ? ", dropping it" : "");
> -                ist->prev_sub.subtitle.end_display_time = end;
> +                ist->prev_sub.subtitle->subtitle_end_time = end;
>              }
>          }
>          FFSWAP(int,        *got_output, ist->prev_sub.got_output);
>          FFSWAP(int,        ret,         ist->prev_sub.ret);
> -        FFSWAP(AVSubtitle, subtitle,    ist->prev_sub.subtitle);
> +        FFSWAP(AVFrame*,   decoded_frame, ist->prev_sub.subtitle);
>          if (end <= 0)
> -            goto out;
> +            return end;
>      }
>  
> -    if (!*got_output)
> +    if (!*got_output || !decoded_frame)
>          return ret;
>  
> -    if (ist->sub2video.frame) {
> -        sub2video_update(ist, INT64_MIN, &subtitle);
> -    } else if (ist->nb_filters) {
> -        if (!ist->sub2video.sub_queue)
> -            ist->sub2video.sub_queue = av_fifo_alloc(8 * sizeof(AVSubtitle));
> -        if (!ist->sub2video.sub_queue)
> -            exit_program(1);
> -        if (!av_fifo_space(ist->sub2video.sub_queue)) {
> -            ret = av_fifo_realloc2(ist->sub2video.sub_queue, 2 * av_fifo_size(ist->sub2video.sub_queue));
> -            if (ret < 0)
> -                exit_program(1);
> -        }
> -        av_fifo_generic_write(ist->sub2video.sub_queue, &subtitle, sizeof(subtitle), NULL);
> -        free_sub = 0;
> -    }
> +    decoded_frame->type = AVMEDIA_TYPE_SUBTITLE;
> +    decoded_frame->format = av_get_subtitle_format_from_codecdesc(avctx->codec_descriptor);
>  
> -    if (!subtitle.num_rects)
> -        goto out;
> +    ////if ((ret = av_frame_get_buffer2(decoded_frame, 0)) < 0)
> +    ////    return ret;
>  
> -    ist->frames_decoded++;
> +    if ((ret = av_buffer_replace(&decoded_frame->subtitle_header, ist->subtitle_heartbeat.header)) < 0)
> +        return ret;
>  
> -    for (i = 0; i < nb_output_streams; i++) {
> -        OutputStream *ost = output_streams[i];
> +    pts     = av_rescale_q(decoded_frame->subtitle_pts + decoded_frame->subtitle_start_time * 1000LL,
> +                           AV_TIME_BASE_Q, ist->st->time_base);
> +    end_pts = av_rescale_q(decoded_frame->subtitle_pts + decoded_frame->subtitle_end_time   * 1000LL,
> +                             AV_TIME_BASE_Q, ist->st->time_base);
>  
> -        if (!check_output_constraints(ist, ost) || !ost->encoding_needed
> -            || ost->enc->type != AVMEDIA_TYPE_SUBTITLE)
> -            continue;
> +    ist->subtitle_heartbeat.last_pts = decoded_frame->pts = pts;
> +    ist->subtitle_heartbeat.end_pts = end_pts;
>  
> -        do_subtitle_out(output_files[ost->file_index], ost, &subtitle);
> +    if (ist->nb_filters > 0) {
> +        AVFrame *filter_frame = av_frame_clone(decoded_frame);

If I see this correctly, then the reason that one has to do something
besides send_frame_to_filters() is that we do not add a dummy
filtergraph like is done in line 2645 of ffmpeg_opt.c for audio and video?

> +        if (!filter_frame)
> +            err = AVERROR(ENOMEM);
> +        else

Don't use an else as if this were two ordinary cases: One case is an
error condition.

> +            err = send_frame_to_filters(ist, filter_frame);

filter_frame leaks here.

>      }
>  
> -out:
> -    if (free_sub)
> -        avsubtitle_free(&subtitle);
> -    return ret;
> +    if (err >= 0) {
> +        for (i = 0; i < nb_output_streams; i++) {
> +            OutputStream *ost = output_streams[i];
> +            InputStream *ist_src = get_input_stream(ost);
> +
> +            if (!ist_src || !check_output_constraints(ist, ost) 
> +                || ist_src != ist
> +                || !ost->encoding_needed
> +                || ost->enc->type != AVMEDIA_TYPE_SUBTITLE)
> +                continue;
> +
> +            if (ost->filter && ost->filter->filter->nb_inputs > 0)
> +                continue;
> +
> +            ////if (!ost->pkt && !((ost->pkt = av_packet_alloc())))
> +            ////    exit_program(1);
> +            do_subtitle_out(output_files[ost->file_index], ost, decoded_frame);
> +        }
> +    }
> +
> +    av_frame_free(&ist->subtitle_heartbeat.recent_sub);

You should not constantly free and allocate the AVFrames, but instead
allocate them once (in new_subtitle_stream() or so).

> +    ist->subtitle_heartbeat.recent_sub = av_frame_clone(decoded_frame);
> +
> +
> +    av_frame_unref(decoded_frame);
> +    return err < 0 ? err : ret;
>  }
>  
>  static int send_filter_eof(InputStream *ist)
> @@ -2686,7 +2688,7 @@ static int process_input_packet(InputStream *ist, const AVPacket *pkt, int no_eo
>          case AVMEDIA_TYPE_SUBTITLE:
>              if (repeating)
>                  break;
> -            ret = transcode_subtitles(ist, avpkt, &got_output, &decode_failed);
> +            ret = decode_subtitles(ist, avpkt, &got_output, &decode_failed);
>              if (!pkt && ret >= 0)
>                  ret = AVERROR_EOF;
>              av_packet_unref(avpkt);
> @@ -2977,13 +2979,6 @@ FF_ENABLE_DEPRECATION_WARNINGS
>      return 0;
>  }
>  
> -static InputStream *get_input_stream(OutputStream *ost)
> -{
> -    if (ost->source_index >= 0)
> -        return input_streams[ost->source_index];
> -    return NULL;
> -}
> -
>  static int compare_int64(const void *a, const void *b)
>  {
>      return FFDIFFSIGN(*(const int64_t *)a, *(const int64_t *)b);
> @@ -3462,7 +3457,7 @@ static int init_output_stream_encode(OutputStream *ost, AVFrame *frame)
>          break;
>      case AVMEDIA_TYPE_SUBTITLE:
>          enc_ctx->time_base = AV_TIME_BASE_Q;
> -        if (!enc_ctx->width) {
> +        if (!enc_ctx->width && ost->source_index >= 0) {
>              enc_ctx->width     = input_streams[ost->source_index]->st->codecpar->width;
>              enc_ctx->height    = input_streams[ost->source_index]->st->codecpar->height;
>          }
> @@ -3515,19 +3510,14 @@ static int init_output_stream(OutputStream *ost, AVFrame *frame,
>          }
>  
>          if (ist && ist->dec->type == AVMEDIA_TYPE_SUBTITLE && ost->enc->type == AVMEDIA_TYPE_SUBTITLE) {
> -            int input_props = 0, output_props = 0;
> -            AVCodecDescriptor const *input_descriptor =
> -                avcodec_descriptor_get(dec->codec_id);
> -            AVCodecDescriptor const *output_descriptor =
> -                avcodec_descriptor_get(ost->enc_ctx->codec_id);
> -            if (input_descriptor)
> -                input_props = input_descriptor->props & (AV_CODEC_PROP_TEXT_SUB | AV_CODEC_PROP_BITMAP_SUB);
> -            if (output_descriptor)
> -                output_props = output_descriptor->props & (AV_CODEC_PROP_TEXT_SUB | AV_CODEC_PROP_BITMAP_SUB);
> -            if (input_props && output_props && input_props != output_props) {
> -                snprintf(error, error_len,
> -                         "Subtitle encoding currently only possible from text to text "
> -                         "or bitmap to bitmap");
> +            AVCodecDescriptor const *input_descriptor     = avcodec_descriptor_get(dec->codec_id);
> +            AVCodecDescriptor const *output_descriptor    = avcodec_descriptor_get(ost->enc_ctx->codec_id);
> +            const enum AVSubtitleType in_subtitle_format  = output_descriptor ? av_get_subtitle_format_from_codecdesc(input_descriptor) : AV_SUBTITLE_FMT_UNKNOWN;
> +            const enum AVSubtitleType out_subtitle_format = output_descriptor ? av_get_subtitle_format_from_codecdesc(output_descriptor) : AV_SUBTITLE_FMT_UNKNOWN;
> +
> +            if (ist->nb_filters == 0 && in_subtitle_format != AV_SUBTITLE_FMT_UNKNOWN && out_subtitle_format != AV_SUBTITLE_FMT_UNKNOWN
> +                && in_subtitle_format != out_subtitle_format) {
> +                snprintf(error, error_len, "Subtitle encoding is only possible from text to text or bitmap to bitmap");
>                  return AVERROR_INVALIDDATA;
>              }
>          }
> @@ -4528,7 +4518,7 @@ static int process_input(int file_index)
>                 av_ts2timestr(input_files[ist->file_index]->ts_offset, &AV_TIME_BASE_Q));
>      }
>  
> -    sub2video_heartbeat(ist, pkt->pts);
> +    subtitle_heartbeat(ist, pkt->pts);
>  
>      process_input_packet(ist, pkt, 0);
>  
> @@ -4740,6 +4730,7 @@ static int transcode(void)
>      /* at the end of stream, we must flush the decoder buffers */
>      for (i = 0; i < nb_input_streams; i++) {
>          ist = input_streams[i];
> +        ist->subtitle_heartbeat.is_active = 0;
>          if (!input_files[ist->file_index]->eof_reached) {
>              process_input_packet(ist, NULL, 0);
>          }
> diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
> index 1728010f56..8a7080834a 100644
> --- a/fftools/ffmpeg.h
> +++ b/fftools/ffmpeg.h
> @@ -349,17 +349,18 @@ typedef struct InputStream {
>      struct { /* previous decoded subtitle and related variables */
>          int got_output;
>          int ret;
> -        AVSubtitle subtitle;
> +        AVFrame *subtitle;
>      } prev_sub;
>  
> -    struct sub2video {
> +    struct subtitle_heartbeat {
> +        int is_active;
> +        int check_buffer_requests;
>          int64_t last_pts;
>          int64_t end_pts;
> -        AVFifoBuffer *sub_queue;    ///< queue of AVSubtitle* before filter init
> -        AVFrame *frame;
> +        AVFrame *recent_sub;
>          int w, h;
> -        unsigned int initialize; ///< marks if sub2video_update should force an initialization
> -    } sub2video;
> +        AVBufferRef *header;
> +    } subtitle_heartbeat;
>  
>      /* decoded data from this stream goes into all those filters
>       * currently video and audio only */
> @@ -659,8 +660,6 @@ int filtergraph_is_simple(FilterGraph *fg);
>  int init_simple_filtergraph(InputStream *ist, OutputStream *ost);
>  int init_complex_filtergraph(FilterGraph *fg);
>  
> -void sub2video_update(InputStream *ist, int64_t heartbeat_pts, AVSubtitle *sub);
> -
>  int ifilter_parameters_from_frame(InputFilter *ifilter, const AVFrame *frame);
>  
>  int ffmpeg_parse_options(int argc, char **argv);
> diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c
> index 452b689d62..1299529d36 100644
> --- a/fftools/ffmpeg_filter.c
> +++ b/fftools/ffmpeg_filter.c
> @@ -22,6 +22,8 @@
>  
>  #include "ffmpeg.h"
>  
> +#include <libavutil/ass_split_internal.h>
> +

ass_split_internal.h is not an installed header, hence the <> form for
headers is inappropriate. Moreover, see below.

>  #include "libavfilter/avfilter.h"
>  #include "libavfilter/buffersink.h"
>  #include "libavfilter/buffersrc.h"
> @@ -30,11 +32,9 @@
>  #include "libavutil/avstring.h"
>  #include "libavutil/bprint.h"
>  #include "libavutil/channel_layout.h"
> -#include "libavutil/display.h"
>  #include "libavutil/opt.h"
>  #include "libavutil/pixdesc.h"
>  #include "libavutil/pixfmt.h"
> -#include "libavutil/imgutils.h"
>  #include "libavutil/samplefmt.h"
>  
>  // FIXME: YUV420P etc. are actually supported with full color range,
> @@ -221,9 +221,8 @@ static void init_input_filter(FilterGraph *fg, AVFilterInOut *in)
>      enum AVMediaType type = avfilter_pad_get_type(in->filter_ctx->input_pads, in->pad_idx);
>      int i;
>  
> -    // TODO: support other filter types
> -    if (type != AVMEDIA_TYPE_VIDEO && type != AVMEDIA_TYPE_AUDIO) {
> -        av_log(NULL, AV_LOG_FATAL, "Only video and audio filters supported "
> +    if (type != AVMEDIA_TYPE_VIDEO && type != AVMEDIA_TYPE_AUDIO && type != AVMEDIA_TYPE_SUBTITLE) {
> +        av_log(NULL, AV_LOG_FATAL, "Only video, audio and subtitle filters supported "
>                 "currently.\n");
>          exit_program(1);
>      }
> @@ -244,8 +243,9 @@ static void init_input_filter(FilterGraph *fg, AVFilterInOut *in)
>          for (i = 0; i < s->nb_streams; i++) {
>              enum AVMediaType stream_type = s->streams[i]->codecpar->codec_type;
>              if (stream_type != type &&
> -                !(stream_type == AVMEDIA_TYPE_SUBTITLE &&
> -                  type == AVMEDIA_TYPE_VIDEO /* sub2video hack */))
> +                // in the followng case we auto-insert the graphicsub2video conversion filter 
> +                // for retaining compatibility with the previous sub2video hack
> +                !(stream_type == AVMEDIA_TYPE_SUBTITLE && type == AVMEDIA_TYPE_VIDEO))
>                  continue;
>              if (check_stream_specifier(s, s->streams[i], *p == ':' ? p + 1 : p) == 1) {
>                  st = s->streams[i];
> @@ -294,6 +294,13 @@ static void init_input_filter(FilterGraph *fg, AVFilterInOut *in)
>      fg->inputs[fg->nb_inputs - 1]->type = ist->st->codecpar->codec_type;
>      fg->inputs[fg->nb_inputs - 1]->name = describe_filter_link(fg, in, 1);
>  
> +    if (ist->st->codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE && ist->dec_ctx) {
> +        const AVCodecDescriptor *codec_descriptor = ist->dec_ctx->codec_descriptor;
> +        if (!codec_descriptor)
> +            codec_descriptor = avcodec_descriptor_get(ist->dec_ctx->codec_id);
> +        fg->inputs[fg->nb_inputs - 1]->format = av_get_subtitle_format_from_codecdesc(codec_descriptor);
> +    }
> +
>      fg->inputs[fg->nb_inputs - 1]->frame_queue = av_fifo_alloc(8 * sizeof(AVFrame*));
>      if (!fg->inputs[fg->nb_inputs - 1]->frame_queue)
>          exit_program(1);
> @@ -416,6 +423,39 @@ static int insert_filter(AVFilterContext **last_filter, int *pad_idx,
>      return 0;
>  }
>  
> +static int configure_output_subtitle_filter(FilterGraph *fg, OutputFilter *ofilter, AVFilterInOut *out)
> +{
> +    OutputStream *ost = ofilter->ost;
> +    AVFilterContext *last_filter = out->filter_ctx;
> +    int pad_idx = out->pad_idx;
> +    int ret;
> +    char name[255];
> +
> +    snprintf(name, sizeof(name), "out_%d_%d", ost->file_index, ost->index);
> +    ret = avfilter_graph_create_filter(&ofilter->filter,
> +                                       avfilter_get_by_name("sbuffersink"),
> +                                       name, NULL, NULL, fg->graph);
> +
> +    if (ret < 0) {
> +        av_log(NULL, AV_LOG_ERROR, "Unable to create filter sbuffersink\n");
> +        return ret;
> +    }
> +
> +    ////snprintf(name, sizeof(name), "trim_out_%d_%d",
> +    ////         ost->file_index, ost->index);
> +    ////ret = insert_trim(of->start_time, of->recording_time,
> +    ////                  &last_filter, &pad_idx, name);
> +    ////if (ret < 0)
> +    ////    return ret;
> +
> +    ////ost->st->codecpar->codec_tag = MKTAG('a', 's', 's', 's');
> +
> +    if ((ret = avfilter_link(last_filter, pad_idx, ofilter->filter, 0)) < 0)
> +        return ret;
> +
> +    return 0;
> +}
> +
>  static int configure_output_video_filter(FilterGraph *fg, OutputFilter *ofilter, AVFilterInOut *out)
>  {
>      char *pix_fmts;
> @@ -594,7 +634,8 @@ static int configure_output_audio_filter(FilterGraph *fg, OutputFilter *ofilter,
>          int i;
>  
>          for (i=0; i<of->ctx->nb_streams; i++)
> -            if (of->ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO)
> +            if (of->ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO ||
> +                of->ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE)
>                  break;
>  
>          if (i<of->ctx->nb_streams) {
> @@ -628,6 +669,7 @@ static int configure_output_filter(FilterGraph *fg, OutputFilter *ofilter,
>      switch (avfilter_pad_get_type(out->filter_ctx->output_pads, out->pad_idx)) {
>      case AVMEDIA_TYPE_VIDEO: return configure_output_video_filter(fg, ofilter, out);
>      case AVMEDIA_TYPE_AUDIO: return configure_output_audio_filter(fg, ofilter, out);
> +    case AVMEDIA_TYPE_SUBTITLE: return configure_output_subtitle_filter(fg, ofilter, out);
>      default: av_assert0(0); return 0;
>      }
>  }
> @@ -647,51 +689,112 @@ void check_filter_outputs(void)
>      }
>  }
>  
> -static int sub2video_prepare(InputStream *ist, InputFilter *ifilter)
> +static int configure_input_subtitle_filter(FilterGraph *fg, InputFilter *ifilter,
> +                                        AVFilterInOut *in)
>  {
> -    AVFormatContext *avf = input_files[ist->file_index]->ctx;
> -    int i, w, h;
> +    AVFilterContext *last_filter;
> +    const AVFilter *buffer_filt = avfilter_get_by_name("sbuffer");
> +    InputStream *ist = ifilter->ist;
> +    AVBPrint args;
> +    char name[255];
> +    int ret, pad_idx = 0;
> +    int w, h;
> +    AVBufferSrcParameters *par = av_buffersrc_parameters_alloc();
> +    enum AVMediaType media_type;
> +
> +    if (!par)
> +        return AVERROR(ENOMEM);
> +    memset(par, 0, sizeof(*par));

Unnecessary, as av_buffersrc_parameters_alloc() already initializes *par.

> +    par->format = AV_PIX_FMT_NONE;
> +
> +    if (ist->dec_ctx->codec_type == AVMEDIA_TYPE_AUDIO) {
> +        av_log(NULL, AV_LOG_ERROR, "Cannot connect subtitle filter to audio input\n");
> +        ret = AVERROR(EINVAL);
> +        goto fail;
> +    }
> +
> +    if (ist->dec_ctx->codec_type == AVMEDIA_TYPE_VIDEO) {
> +        av_log(NULL, AV_LOG_ERROR, "Cannot connect subtitle filter to video input\n");
> +        ret = AVERROR(EINVAL);
> +        goto fail;
> +    }
> +
> +    if (!ist->subtitle_heartbeat.header && ist->dec_ctx->subtitle_header && ist->dec_ctx->subtitle_header_size > 0) {
> +        ist->subtitle_heartbeat.header = av_buffer_allocz(ist->dec_ctx->subtitle_header_size + 1);
> +        if (!ist->subtitle_heartbeat.header)
> +            return AVERROR(ENOMEM);
> +        memcpy(ist->subtitle_heartbeat.header->data, ist->dec_ctx->subtitle_header, ist->dec_ctx->subtitle_header_size);
> +    }
> +
> +    ist->subtitle_heartbeat.is_active = 1;
>  
> -    /* Compute the size of the canvas for the subtitles stream.
> -       If the subtitles codecpar has set a size, use it. Otherwise use the
> -       maximum dimensions of the video streams in the same file. */
>      w = ifilter->width;
>      h = ifilter->height;
> +
>      if (!(w && h)) {
> -        for (i = 0; i < avf->nb_streams; i++) {
> -            if (avf->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
> -                w = FFMAX(w, avf->streams[i]->codecpar->width);
> -                h = FFMAX(h, avf->streams[i]->codecpar->height);
> -            }
> -        }
> -        if (!(w && h)) {
> -            w = FFMAX(w, 720);
> -            h = FFMAX(h, 576);
> -        }
> -        av_log(avf, AV_LOG_INFO, "sub2video: using %dx%d canvas\n", w, h);
> +        w = ist->dec_ctx->width;
> +        h = ist->dec_ctx->height;
>      }
> -    ist->sub2video.w = ifilter->width  = w;
> -    ist->sub2video.h = ifilter->height = h;
>  
> -    ifilter->width  = ist->dec_ctx->width  ? ist->dec_ctx->width  : ist->sub2video.w;
> -    ifilter->height = ist->dec_ctx->height ? ist->dec_ctx->height : ist->sub2video.h;
> +    if (!(w && h) && ist->dec_ctx->subtitle_header) {
> +        ASSSplitContext *ass_ctx = avpriv_ass_split((char *)ist->dec_ctx->subtitle_header);

avpriv functions must not be used in fftools. And what makes you so
certain that subtitle_header will only be used by ass subtitles?
Furthermore, missing check.
(Maybe ass subtitle based codecs should set AVCodecContext.width and
height based upon this play_res_x/y? This would be easy to implement in
ff_ass_subtitle_header_full()? But I don't know where
ASS_DEFAULT_PLAYRES[XY] comes from and I expect this to be the common
case given that only movtextdec and ass itself uses something different.)

> +        ASS *ass = (ASS *)ass_ctx;
> +        w = ass->script_info.play_res_x;
> +        h = ass->script_info.play_res_y;
> +        avpriv_ass_split_free(ass_ctx);
> +    }
>  
> -    /* rectangles are AV_PIX_FMT_PAL8, but we have no guarantee that the
> -       palettes for all rectangles are identical or compatible */
> -    ifilter->format = AV_PIX_FMT_RGB32;
> +    ist->subtitle_heartbeat.w = w;
> +    ist->subtitle_heartbeat.h = h;
> +    av_log(ifilter, AV_LOG_INFO, "subtitle input filter: decoding size %dx%d\n", ist->subtitle_heartbeat.w, ist->subtitle_heartbeat.h);
>  
> -    ist->sub2video.frame = av_frame_alloc();
> -    if (!ist->sub2video.frame)
> -        return AVERROR(ENOMEM);
> -    ist->sub2video.last_pts = INT64_MIN;
> -    ist->sub2video.end_pts  = INT64_MIN;
> +    ifilter->width = w;
> +    ifilter->height = h;
> +    ist->dec_ctx->width = w;
> +    ist->dec_ctx->height = h;
>  
> -    /* sub2video structure has been (re-)initialized.
> -       Mark it as such so that the system will be
> -       initialized with the first received heartbeat. */
> -    ist->sub2video.initialize = 1;
> +    ist->subtitle_heartbeat.last_pts = INT64_MIN;
> +
> +    snprintf(name, sizeof(name), "graph %d subtitle input from stream %d:%d", fg->index,
> +             ist->file_index, ist->st->index);
> +
> +
> +    av_bprint_init(&args, 0, AV_BPRINT_SIZE_AUTOMATIC);
> +    av_bprintf(&args,
> +             "subtitle_type=%d:width=%d:height=%d:time_base=%d/%d:",
> +             ifilter->format, ifilter->width, ifilter->height,
> +             ist->st->time_base.num, ist->st->time_base.den);
> +    if ((ret = avfilter_graph_create_filter(&ifilter->filter, buffer_filt, name,
> +                                            args.str, NULL, fg->graph)) < 0)
> +        goto fail;
> +
> +    par->hw_frames_ctx = ifilter->hw_frames_ctx;
> +    par->format = ifilter->format;
> +    par->width = ifilter->width;
> +    par->height = ifilter->height;
> +
> +    ret = av_buffersrc_parameters_set(ifilter->filter, par);
> +    if (ret < 0)
> +        goto fail;
> +    av_freep(&par);
> +    last_filter = ifilter->filter;
> +
> +    media_type = avfilter_pad_get_type(in->filter_ctx->input_pads, in->pad_idx);
> +    if (media_type == AVMEDIA_TYPE_VIDEO) {
> +        av_log(NULL, AV_LOG_INFO, "Auto-inserting graphicsub2video filter\n");
> +        ret = insert_filter(&last_filter, &pad_idx, "graphicsub2video", NULL);
> +        if (ret < 0)
> +            return ret;
> +    }
> +
> +    if ((ret = avfilter_link(last_filter, 0, in->filter_ctx, in->pad_idx)) < 0)
> +        return ret;
>  
>      return 0;
> +fail:
> +    av_freep(&par);
> +
> +    return ret;
>  }
>  
>  static int configure_input_video_filter(FilterGraph *fg, InputFilter *ifilter,
> @@ -710,8 +813,15 @@ static int configure_input_video_filter(FilterGraph *fg, InputFilter *ifilter,
>      char name[255];
>      int ret, pad_idx = 0;
>      int64_t tsoffset = 0;
> -    AVBufferSrcParameters *par = av_buffersrc_parameters_alloc();
> +    AVBufferSrcParameters *par;
> +
> +    if (ist->dec_ctx->codec_type == AVMEDIA_TYPE_SUBTITLE) {
> +        // Automatically insert conversion filter to retain compatibility
> +        // with sub2video command lines
> +        return configure_input_subtitle_filter(fg, ifilter, in);
> +    }
>  
> +    par = av_buffersrc_parameters_alloc();
>      if (!par)
>          return AVERROR(ENOMEM);
>      memset(par, 0, sizeof(*par));
> @@ -726,12 +836,6 @@ static int configure_input_video_filter(FilterGraph *fg, InputFilter *ifilter,
>      if (!fr.num)
>          fr = av_guess_frame_rate(input_files[ist->file_index]->ctx, ist->st, NULL);
>  
> -    if (ist->dec_ctx->codec_type == AVMEDIA_TYPE_SUBTITLE) {
> -        ret = sub2video_prepare(ist, ifilter);
> -        if (ret < 0)
> -            goto fail;
> -    }
> -
>      sar = ifilter->sample_aspect_ratio;
>      if(!sar.den)
>          sar = (AVRational){0,1};
> @@ -743,7 +847,7 @@ static int configure_input_video_filter(FilterGraph *fg, InputFilter *ifilter,
>               tb.num, tb.den, sar.num, sar.den);
>      if (fr.num && fr.den)
>          av_bprintf(&args, ":frame_rate=%d/%d", fr.num, fr.den);
> -    snprintf(name, sizeof(name), "graph %d input from stream %d:%d", fg->index,
> +    snprintf(name, sizeof(name), "graph %d video input from stream %d:%d", fg->index,
>               ist->file_index, ist->st->index);
>  
>  
> @@ -949,6 +1053,7 @@ static int configure_input_filter(FilterGraph *fg, InputFilter *ifilter,
>      switch (avfilter_pad_get_type(in->filter_ctx->input_pads, in->pad_idx)) {
>      case AVMEDIA_TYPE_VIDEO: return configure_input_video_filter(fg, ifilter, in);
>      case AVMEDIA_TYPE_AUDIO: return configure_input_audio_filter(fg, ifilter, in);
> +    case AVMEDIA_TYPE_SUBTITLE: return configure_input_subtitle_filter(fg, ifilter, in);
>      default: av_assert0(0); return 0;
>      }
>  }
> @@ -1116,19 +1221,6 @@ int configure_filtergraph(FilterGraph *fg)
>          }
>      }
>  
> -    /* process queued up subtitle packets */
> -    for (i = 0; i < fg->nb_inputs; i++) {
> -        InputStream *ist = fg->inputs[i]->ist;
> -        if (ist->sub2video.sub_queue && ist->sub2video.frame) {
> -            while (av_fifo_size(ist->sub2video.sub_queue)) {
> -                AVSubtitle tmp;
> -                av_fifo_generic_read(ist->sub2video.sub_queue, &tmp, sizeof(tmp), NULL);
> -                sub2video_update(ist, INT64_MIN, &tmp);
> -                avsubtitle_free(&tmp);
> -            }
> -        }
> -    }
> -
>      return 0;
>  
>  fail:
> @@ -1151,6 +1243,7 @@ int ifilter_parameters_from_frame(InputFilter *ifilter, const AVFrame *frame)
>      ifilter->sample_rate         = frame->sample_rate;
>      ifilter->channels            = frame->channels;
>      ifilter->channel_layout      = frame->channel_layout;
> +    ifilter->type                = frame->type;
>  
>      av_freep(&ifilter->displaymatrix);
>      sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DISPLAYMATRIX);
> diff --git a/fftools/ffmpeg_hw.c b/fftools/ffmpeg_hw.c
> index 14e702bd92..be69d54aaf 100644
> --- a/fftools/ffmpeg_hw.c
> +++ b/fftools/ffmpeg_hw.c
> @@ -449,7 +449,7 @@ int hw_device_setup_for_encode(OutputStream *ost)
>      AVBufferRef *frames_ref = NULL;
>      int i;
>  
> -    if (ost->filter) {
> +    if (ost->filter && ost->filter->filter) {

I don't think that your patches make it necessary to add this (or have
you already added hardware accelerated subtitle encoding?), so it is
either already necessary in master and should be sent as a separate
commit or it is unnecessary and should be dropped.

>          frames_ref = av_buffersink_get_hw_frames_ctx(ost->filter->filter);
>          if (frames_ref &&
>              ((AVHWFramesContext*)frames_ref->data)->format ==
> diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c
> index e55b584fd4..d443a5b8c8 100644
> --- a/fftools/ffmpeg_opt.c
> +++ b/fftools/ffmpeg_opt.c
> @@ -2207,8 +2207,9 @@ static void init_output_filter(OutputFilter *ofilter, OptionsContext *o,
>      switch (ofilter->type) {
>      case AVMEDIA_TYPE_VIDEO: ost = new_video_stream(o, oc, -1); break;
>      case AVMEDIA_TYPE_AUDIO: ost = new_audio_stream(o, oc, -1); break;
> +    case AVMEDIA_TYPE_SUBTITLE: ost = new_subtitle_stream(o, oc, -1); break;
>      default:
> -        av_log(NULL, AV_LOG_FATAL, "Only video and audio filters are supported "
> +        av_log(NULL, AV_LOG_FATAL, "Only video, audio and subtitle filters are supported "
>                 "currently.\n");
>          exit_program(1);
>      }
Soft Works Nov. 27, 2021, 7:18 a.m. UTC | #2 
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of Andreas
> Rheinhardt
> Sent: Friday, November 26, 2021 2:02 PM
> To: ffmpeg-devel@ffmpeg.org
> Subject: Re: [FFmpeg-devel] [PATCH v16 08/16] fftools/ffmpeg: Replace
> sub2video with subtitle frame filtering
> 
> > -    int subtitle_out_max_size = 1024 * 1024;
> > +    const int subtitle_out_max_size = 1024 * 1024;
> >      int subtitle_out_size, nb, i;
> >      AVCodecContext *enc;
> >      AVPacket *pkt = ost->pkt;
> > +    AVSubtitle out_sub = { 0 };
> 
> You are adding some stuff here which is removed in patch 16. These
> patches should be merged.

Done. I had to move the encoding API commit to an earlier position for this.


> > +        pts     = heartbeat_pts; //av_rescale_q(frame->subtitle_pts +
> frame->subtitle_start_time * 1000LL, AV_TIME_BASE_Q, ist->st->time_base);
> > +        end_pts = av_rescale_q(frame->subtitle_pts + frame-
> >subtitle_end_time   * 1000LL, AV_TIME_BASE_Q, ist->st->time_base);
> > +    }
> > +    else {
> 
> Put this on the same line as }.
> 
> > +        frame = av_frame_alloc();
> 
> Is it actually certain that we need a new frame and can't reuse
> decoded_frame like the other functions that ultimately call
> send_frame_to_filters() do?

This is typically only happening at the start when no subtitle frames have
been decoded yet. Also those are empty frames, nothing like video frames
with a large buffer allocated.

> >
> > -        do_subtitle_out(output_files[ost->file_index], ost, &subtitle);
> > +    if (ist->nb_filters > 0) {
> > +        AVFrame *filter_frame = av_frame_clone(decoded_frame);
> 
> If I see this correctly, then the reason that one has to do something
> besides send_frame_to_filters() is that we do not add a dummy
> filtergraph like is done in line 2645 of ffmpeg_opt.c for audio and video?

IIRC this is to avoid the heartbeat frames arriving at the encoder when no 
filtering is done (or a source subtitle stream is both going through filtering
but at the same time encoded directly as an output stream.

> > +    if (!(w && h) && ist->dec_ctx->subtitle_header) {
> > +        ASSSplitContext *ass_ctx = avpriv_ass_split((char *)ist->dec_ctx-
> >subtitle_header);
> 
> avpriv functions must not be used in fftools. 

I wanted to make this part of the public API, but you an/or Hendrik objected
IIRC. I had made some suggestions for how this could be done, but it remained
unresponded (I had asked multiple times about this).

> And what makes you so
> certain that subtitle_header will only be used by ass subtitles?

Because ASS is the defined internal format for text subtitles.

- Each text subtitle decoder outputs ASS subtitles
- Each text subtitle encoder gets ASS subtitles as input
- Text subtitle filters work on ASS subtitle data
(there's a hardly used exception: AV_SUBTITLE_FMT_TEXT, but that doesn’t use any
header)

> Furthermore, missing check.
> (Maybe ass subtitle based codecs should set AVCodecContext.width and
> height based upon this play_res_x/y?

Breaks the decoder API. Anyway, the full header needs to be parsed by certain filters.
Also the other ass functions needs to be available for filters (e.g. for parsing dialogs).

Due to the fact that ASS is not just an encoding output or decoding input, but 
internally used as transport format, those ass functions need to be available outside 
of libavcodec.
I'm totally open for suggestions regarding _how_ this should be done.


> >      av_freep(&ifilter->displaymatrix);
> >      sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DISPLAYMATRIX);
> > diff --git a/fftools/ffmpeg_hw.c b/fftools/ffmpeg_hw.c
> > index 14e702bd92..be69d54aaf 100644
> > --- a/fftools/ffmpeg_hw.c
> > +++ b/fftools/ffmpeg_hw.c
> > @@ -449,7 +449,7 @@ int hw_device_setup_for_encode(OutputStream *ost)
> >      AVBufferRef *frames_ref = NULL;
> >      int i;
> >
> > -    if (ost->filter) {
> > +    if (ost->filter && ost->filter->filter) {
> 
> I don't think that your patches make it necessary to add this (or have
> you already added hardware accelerated subtitle encoding?), so it is
> either already necessary in master and should be sent as a separate
> commit or it is unnecessary and should be dropped.

Using ffmpeg with hw acceleration is my primary use case, that's why I'm 
rather sure that it isn't required before this patchset.

While working on this patchset I've also tested various scenarios with hw
acceleration, so it might be possible that I've run into this in combination
with hw acceleration. Anyway - why should adding this check be unnecessary 
or even wrong?

When ost->filter->filter is NULL, then av_buffersink_get_hw_frames_ctx()
will crash.


Kind regards,
softworkz
Soft Works Nov. 27, 2021, 7:23 a.m. UTC | #3 
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of Soft Works
> Sent: Saturday, November 27, 2021 8:19 AM
> To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org>
> Subject: Re: [FFmpeg-devel] [PATCH v16 08/16] fftools/ffmpeg: Replace
> sub2video with subtitle frame filtering
> 
> 
> 
> > -----Original Message-----
> > From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of Andreas
> > Rheinhardt
> > Sent: Friday, November 26, 2021 2:02 PM
> > To: ffmpeg-devel@ffmpeg.org
> > Subject: Re: [FFmpeg-devel] [PATCH v16 08/16] fftools/ffmpeg: Replace
> > sub2video with subtitle frame filtering
> >
> > > -    int subtitle_out_max_size = 1024 * 1024;
> > > +    const int subtitle_out_max_size = 1024 * 1024;
> > >      int subtitle_out_size, nb, i;
> > >      AVCodecContext *enc;
> > >      AVPacket *pkt = ost->pkt;
> > > +    AVSubtitle out_sub = { 0 };
> >
> > You are adding some stuff here which is removed in patch 16. These
> > patches should be merged.
> 
> Done. I had to move the encoding API commit to an earlier position for this.
> 
> 
> > > +        pts     = heartbeat_pts; //av_rescale_q(frame->subtitle_pts +
> > frame->subtitle_start_time * 1000LL, AV_TIME_BASE_Q, ist->st->time_base);
> > > +        end_pts = av_rescale_q(frame->subtitle_pts + frame-
> > >subtitle_end_time   * 1000LL, AV_TIME_BASE_Q, ist->st->time_base);
> > > +    }
> > > +    else {
> >
> > Put this on the same line as }.
> >
> > > +        frame = av_frame_alloc();
> >
> > Is it actually certain that we need a new frame and can't reuse
> > decoded_frame like the other functions that ultimately call
> > send_frame_to_filters() do?
> 
> This is typically only happening at the start when no subtitle frames have
> been decoded yet. Also those are empty frames, nothing like video frames
> with a large buffer allocated.
> 
> > >
> > > -        do_subtitle_out(output_files[ost->file_index], ost, &subtitle);
> > > +    if (ist->nb_filters > 0) {
> > > +        AVFrame *filter_frame = av_frame_clone(decoded_frame);
> >
> > If I see this correctly, then the reason that one has to do something
> > besides send_frame_to_filters() is that we do not add a dummy
> > filtergraph like is done in line 2645 of ffmpeg_opt.c for audio and video?
> 
> IIRC this is to avoid the heartbeat frames arriving at the encoder when no
> filtering is done (or a source subtitle stream is both going through
> filtering
> but at the same time encoded directly as an output stream.
> 
> > > +    if (!(w && h) && ist->dec_ctx->subtitle_header) {
> > > +        ASSSplitContext *ass_ctx = avpriv_ass_split((char *)ist-
> >dec_ctx-
> > >subtitle_header);
> >
> > avpriv functions must not be used in fftools.
> 
> I wanted to make this part of the public API, but you an/or Hendrik objected
> IIRC. I had made some suggestions for how this could be done, but it remained
> unresponded (I had asked multiple times about this).
> 
> > And what makes you so
> > certain that subtitle_header will only be used by ass subtitles?
> 
> Because ASS is the defined internal format for text subtitles.
> 
> - Each text subtitle decoder outputs ASS subtitles
> - Each text subtitle encoder gets ASS subtitles as input
> - Text subtitle filters work on ASS subtitle data
> (there's a hardly used exception: AV_SUBTITLE_FMT_TEXT, but that doesn’t use
> any
> header)
> 
> > Furthermore, missing check.
> > (Maybe ass subtitle based codecs should set AVCodecContext.width and
> > height based upon this play_res_x/y?
> 
> Breaks the decoder API. Anyway, the full header needs to be parsed by certain
> filters.
> Also the other ass functions needs to be available for filters (e.g. for
> parsing dialogs).
> 
> Due to the fact that ASS is not just an encoding output or decoding input,
> but
> internally used as transport format, those ass functions need to be available
> outside
> of libavcodec.
> I'm totally open for suggestions regarding _how_ this should be done.
> 
> 
> > >      av_freep(&ifilter->displaymatrix);
> > >      sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DISPLAYMATRIX);
> > > diff --git a/fftools/ffmpeg_hw.c b/fftools/ffmpeg_hw.c
> > > index 14e702bd92..be69d54aaf 100644
> > > --- a/fftools/ffmpeg_hw.c
> > > +++ b/fftools/ffmpeg_hw.c
> > > @@ -449,7 +449,7 @@ int hw_device_setup_for_encode(OutputStream *ost)
> > >      AVBufferRef *frames_ref = NULL;
> > >      int i;
> > >
> > > -    if (ost->filter) {
> > > +    if (ost->filter && ost->filter->filter) {
> >
> > I don't think that your patches make it necessary to add this (or have
> > you already added hardware accelerated subtitle encoding?), so it is
> > either already necessary in master and should be sent as a separate
> > commit or it is unnecessary and should be dropped.
> 
> Using ffmpeg with hw acceleration is my primary use case, that's why I'm
> rather sure that it isn't required before this patchset.
> 
> While working on this patchset I've also tested various scenarios with hw
> acceleration, so it might be possible that I've run into this in combination
> with hw acceleration. Anyway - why should adding this check be unnecessary
> or even wrong?
> 
> When ost->filter->filter is NULL, then av_buffersink_get_hw_frames_ctx()
> will crash.
> 
> 
> Kind regards,
> softworkz

PS: Forgot to mention - everything else applied as suggested
Anton Khirnov Nov. 27, 2021, 9:04 a.m. UTC | #4 
Quoting Soft Works (2021-11-27 08:18:37)
> > -----Original Message-----
> > From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of Andreas
> > Rheinhardt
> > Sent: Friday, November 26, 2021 2:02 PM
> > To: ffmpeg-devel@ffmpeg.org
> > Subject: Re: [FFmpeg-devel] [PATCH v16 08/16] fftools/ffmpeg: Replace
> > sub2video with subtitle frame filtering
> 
> > Furthermore, missing check.
> > (Maybe ass subtitle based codecs should set AVCodecContext.width and
> > height based upon this play_res_x/y?
> 
> Breaks the decoder API.

Why? I'd think width/height are currently unused for subtitles, so you
can repurpose them without breaking anything.
Soft Works Nov. 27, 2021, 9:20 a.m. UTC | #5 
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of Anton
> Khirnov
> Sent: Saturday, November 27, 2021 10:04 AM
> To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org>
> Subject: Re: [FFmpeg-devel] [PATCH v16 08/16] fftools/ffmpeg: Replace
> sub2video with subtitle frame filtering
> 
> Quoting Soft Works (2021-11-27 08:18:37)
> > > -----Original Message-----
> > > From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of Andreas
> > > Rheinhardt
> > > Sent: Friday, November 26, 2021 2:02 PM
> > > To: ffmpeg-devel@ffmpeg.org
> > > Subject: Re: [FFmpeg-devel] [PATCH v16 08/16] fftools/ffmpeg: Replace
> > > sub2video with subtitle frame filtering
> >
> > > Furthermore, missing check.
> > > (Maybe ass subtitle based codecs should set AVCodecContext.width and
> > > height based upon this play_res_x/y?
> >
> > Breaks the decoder API.
> 
> Why? I'd think width/height are currently unused for subtitles, so you
> can repurpose them without breaking anything.

It breaks the API because every decoder would be required to do this, and
would need to be changed.
Andreas Rheinhardt Nov. 27, 2021, 9:29 a.m. UTC | #6 
Soft Works:
> 
> 
>> -----Original Message-----
>> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of Anton
>> Khirnov
>> Sent: Saturday, November 27, 2021 10:04 AM
>> To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org>
>> Subject: Re: [FFmpeg-devel] [PATCH v16 08/16] fftools/ffmpeg: Replace
>> sub2video with subtitle frame filtering
>>
>> Quoting Soft Works (2021-11-27 08:18:37)
>>>> -----Original Message-----
>>>> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of Andreas
>>>> Rheinhardt
>>>> Sent: Friday, November 26, 2021 2:02 PM
>>>> To: ffmpeg-devel@ffmpeg.org
>>>> Subject: Re: [FFmpeg-devel] [PATCH v16 08/16] fftools/ffmpeg: Replace
>>>> sub2video with subtitle frame filtering
>>>
>>>> Furthermore, missing check.
>>>> (Maybe ass subtitle based codecs should set AVCodecContext.width and
>>>> height based upon this play_res_x/y?
>>>
>>> Breaks the decoder API.
>>
>> Why? I'd think width/height are currently unused for subtitles, so you
>> can repurpose them without breaking anything.
> 
> It breaks the API because every decoder would be required to do this, and
> would need to be changed.
> 

1. If by "every decoder" you mean every decoder in lavc, then this does
not mean that this would be an API break; it just means that it would be
more work.
2. Why would "every decoder" need to be changed? We do not need to
promise to set these fields all the time; we should not even set these
fields all the time, but only if there is really something meaningful to
report (e.g. I don't consider this default value (where does this number
even come from?) meaningful).

- Andreas
Soft Works Nov. 27, 2021, 9:36 a.m. UTC | #7 
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of Andreas
> Rheinhardt
> Sent: Saturday, November 27, 2021 10:29 AM
> To: ffmpeg-devel@ffmpeg.org
> Subject: Re: [FFmpeg-devel] [PATCH v16 08/16] fftools/ffmpeg: Replace
> sub2video with subtitle frame filtering
> 
> Soft Works:
> >
> >
> >> -----Original Message-----
> >> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of Anton
> >> Khirnov
> >> Sent: Saturday, November 27, 2021 10:04 AM
> >> To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org>
> >> Subject: Re: [FFmpeg-devel] [PATCH v16 08/16] fftools/ffmpeg: Replace
> >> sub2video with subtitle frame filtering
> >>
> >> Quoting Soft Works (2021-11-27 08:18:37)
> >>>> -----Original Message-----
> >>>> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
> Andreas
> >>>> Rheinhardt
> >>>> Sent: Friday, November 26, 2021 2:02 PM
> >>>> To: ffmpeg-devel@ffmpeg.org
> >>>> Subject: Re: [FFmpeg-devel] [PATCH v16 08/16] fftools/ffmpeg: Replace
> >>>> sub2video with subtitle frame filtering
> >>>
> >>>> Furthermore, missing check.
> >>>> (Maybe ass subtitle based codecs should set AVCodecContext.width and
> >>>> height based upon this play_res_x/y?
> >>>
> >>> Breaks the decoder API.
> >>
> >> Why? I'd think width/height are currently unused for subtitles, so you
> >> can repurpose them without breaking anything.
> >
> > It breaks the API because every decoder would be required to do this, and
> > would need to be changed.
> >
> 
> 1. If by "every decoder" you mean every decoder in lavc, then this does
> not mean that this would be an API break; it just means that it would be
> more work.
> 2. Why would "every decoder" need to be changed? We do not need to
> promise to set these fields all the time; we should not even set these
> fields all the time, but only if there is really something meaningful to
> report (e.g. I don't consider this default value (where does this number
> even come from?) meaningful).

It's a moot discussion anyway. Anton's idea was to do this to get right
of the avpriv_ass... functions, as he had seen that it's mostly used to get
the PlayResX/Y values. But it won't work out. Those functions are needed, 
because ASS is the internal text subtitle format. Filters read and manipulate
that ass data => those functions are needed outside of libavcodec.

softworkz
diff mbox series

Patch

diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c
index 3761ea0c38..c697c12777 100644
--- a/fftools/ffmpeg.c
+++ b/fftools/ffmpeg.c
@@ -169,163 +169,6 @@  static int restore_tty;
 static void free_input_threads(void);
 #endif
 
-/* sub2video hack:
-   Convert subtitles to video with alpha to insert them in filter graphs.
-   This is a temporary solution until libavfilter gets real subtitles support.
- */
-
-static int sub2video_get_blank_frame(InputStream *ist)
-{
-    int ret;
-    AVFrame *frame = ist->sub2video.frame;
-
-    av_frame_unref(frame);
-    ist->sub2video.frame->width  = ist->dec_ctx->width  ? ist->dec_ctx->width  : ist->sub2video.w;
-    ist->sub2video.frame->height = ist->dec_ctx->height ? ist->dec_ctx->height : ist->sub2video.h;
-    ist->sub2video.frame->format = AV_PIX_FMT_RGB32;
-    if ((ret = av_frame_get_buffer(frame, 0)) < 0)
-        return ret;
-    memset(frame->data[0], 0, frame->height * frame->linesize[0]);
-    return 0;
-}
-
-static void sub2video_copy_rect(uint8_t *dst, int dst_linesize, int w, int h,
-                                AVSubtitleRect *r)
-{
-    uint32_t *pal, *dst2;
-    uint8_t *src, *src2;
-    int x, y;
-
-    if (r->type != SUBTITLE_BITMAP) {
-        av_log(NULL, AV_LOG_WARNING, "sub2video: non-bitmap subtitle\n");
-        return;
-    }
-    if (r->x < 0 || r->x + r->w > w || r->y < 0 || r->y + r->h > h) {
-        av_log(NULL, AV_LOG_WARNING, "sub2video: rectangle (%d %d %d %d) overflowing %d %d\n",
-            r->x, r->y, r->w, r->h, w, h
-        );
-        return;
-    }
-
-    dst += r->y * dst_linesize + r->x * 4;
-    src = r->data[0];
-    pal = (uint32_t *)r->data[1];
-    for (y = 0; y < r->h; y++) {
-        dst2 = (uint32_t *)dst;
-        src2 = src;
-        for (x = 0; x < r->w; x++)
-            *(dst2++) = pal[*(src2++)];
-        dst += dst_linesize;
-        src += r->linesize[0];
-    }
-}
-
-static void sub2video_push_ref(InputStream *ist, int64_t pts)
-{
-    AVFrame *frame = ist->sub2video.frame;
-    int i;
-    int ret;
-
-    av_assert1(frame->data[0]);
-    ist->sub2video.last_pts = frame->pts = pts;
-    for (i = 0; i < ist->nb_filters; i++) {
-        ret = av_buffersrc_add_frame_flags(ist->filters[i]->filter, frame,
-                                           AV_BUFFERSRC_FLAG_KEEP_REF |
-                                           AV_BUFFERSRC_FLAG_PUSH);
-        if (ret != AVERROR_EOF && ret < 0)
-            av_log(NULL, AV_LOG_WARNING, "Error while add the frame to buffer source(%s).\n",
-                   av_err2str(ret));
-    }
-}
-
-void sub2video_update(InputStream *ist, int64_t heartbeat_pts, AVSubtitle *sub)
-{
-    AVFrame *frame = ist->sub2video.frame;
-    int8_t *dst;
-    int     dst_linesize;
-    int num_rects, i;
-    int64_t pts, end_pts;
-
-    if (!frame)
-        return;
-    if (sub) {
-        pts       = av_rescale_q(sub->pts + sub->start_display_time * 1000LL,
-                                 AV_TIME_BASE_Q, ist->st->time_base);
-        end_pts   = av_rescale_q(sub->pts + sub->end_display_time   * 1000LL,
-                                 AV_TIME_BASE_Q, ist->st->time_base);
-        num_rects = sub->num_rects;
-    } else {
-        /* If we are initializing the system, utilize current heartbeat
-           PTS as the start time, and show until the following subpicture
-           is received. Otherwise, utilize the previous subpicture's end time
-           as the fall-back value. */
-        pts       = ist->sub2video.initialize ?
-                    heartbeat_pts : ist->sub2video.end_pts;
-        end_pts   = INT64_MAX;
-        num_rects = 0;
-    }
-    if (sub2video_get_blank_frame(ist) < 0) {
-        av_log(ist->dec_ctx, AV_LOG_ERROR,
-               "Impossible to get a blank canvas.\n");
-        return;
-    }
-    dst          = frame->data    [0];
-    dst_linesize = frame->linesize[0];
-    for (i = 0; i < num_rects; i++)
-        sub2video_copy_rect(dst, dst_linesize, frame->width, frame->height, sub->rects[i]);
-    sub2video_push_ref(ist, pts);
-    ist->sub2video.end_pts = end_pts;
-    ist->sub2video.initialize = 0;
-}
-
-static void sub2video_heartbeat(InputStream *ist, int64_t pts)
-{
-    InputFile *infile = input_files[ist->file_index];
-    int i, j, nb_reqs;
-    int64_t pts2;
-
-    /* When a frame is read from a file, examine all sub2video streams in
-       the same file and send the sub2video frame again. Otherwise, decoded
-       video frames could be accumulating in the filter graph while a filter
-       (possibly overlay) is desperately waiting for a subtitle frame. */
-    for (i = 0; i < infile->nb_streams; i++) {
-        InputStream *ist2 = input_streams[infile->ist_index + i];
-        if (!ist2->sub2video.frame)
-            continue;
-        /* subtitles seem to be usually muxed ahead of other streams;
-           if not, subtracting a larger time here is necessary */
-        pts2 = av_rescale_q(pts, ist->st->time_base, ist2->st->time_base) - 1;
-        /* do not send the heartbeat frame if the subtitle is already ahead */
-        if (pts2 <= ist2->sub2video.last_pts)
-            continue;
-        if (pts2 >= ist2->sub2video.end_pts || ist2->sub2video.initialize)
-            /* if we have hit the end of the current displayed subpicture,
-               or if we need to initialize the system, update the
-               overlayed subpicture and its start/end times */
-            sub2video_update(ist2, pts2 + 1, NULL);
-        for (j = 0, nb_reqs = 0; j < ist2->nb_filters; j++)
-            nb_reqs += av_buffersrc_get_nb_failed_requests(ist2->filters[j]->filter);
-        if (nb_reqs)
-            sub2video_push_ref(ist2, pts2);
-    }
-}
-
-static void sub2video_flush(InputStream *ist)
-{
-    int i;
-    int ret;
-
-    if (ist->sub2video.end_pts < INT64_MAX)
-        sub2video_update(ist, INT64_MAX, NULL);
-    for (i = 0; i < ist->nb_filters; i++) {
-        ret = av_buffersrc_add_frame(ist->filters[i]->filter, NULL);
-        if (ret != AVERROR_EOF && ret < 0)
-            av_log(NULL, AV_LOG_WARNING, "Flush the frame error.\n");
-    }
-}
-
-/* end of sub2video hack */
-
 static void term_exit_sigsafe(void)
 {
 #if HAVE_TERMIOS_H
@@ -526,7 +369,6 @@  static void ffmpeg_cleanup(int ret)
         avfilter_graph_free(&fg->graph);
         for (j = 0; j < fg->nb_inputs; j++) {
             InputFilter *ifilter = fg->inputs[j];
-            struct InputStream *ist = ifilter->ist;
 
             while (av_fifo_size(ifilter->frame_queue)) {
                 AVFrame *frame;
@@ -536,15 +378,6 @@  static void ffmpeg_cleanup(int ret)
             }
             av_fifo_freep(&ifilter->frame_queue);
             av_freep(&ifilter->displaymatrix);
-            if (ist->sub2video.sub_queue) {
-                while (av_fifo_size(ist->sub2video.sub_queue)) {
-                    AVSubtitle sub;
-                    av_fifo_generic_read(ist->sub2video.sub_queue,
-                                         &sub, sizeof(sub), NULL);
-                    avsubtitle_free(&sub);
-                }
-                av_fifo_freep(&ist->sub2video.sub_queue);
-            }
             av_buffer_unref(&ifilter->hw_frames_ctx);
             av_freep(&ifilter->name);
             av_freep(&fg->inputs[j]);
@@ -635,12 +468,14 @@  static void ffmpeg_cleanup(int ret)
         av_frame_free(&ist->decoded_frame);
         av_packet_free(&ist->pkt);
         av_dict_free(&ist->decoder_opts);
-        avsubtitle_free(&ist->prev_sub.subtitle);
-        av_frame_free(&ist->sub2video.frame);
+        av_frame_free(&ist->prev_sub.subtitle);
         av_freep(&ist->filters);
         av_freep(&ist->hwaccel_device);
         av_freep(&ist->dts_buffer);
 
+        av_frame_free(&ist->subtitle_heartbeat.recent_sub);
+        av_buffer_unref(&ist->subtitle_heartbeat.header);
+
         avcodec_free_context(&ist->dec_ctx);
 
         av_freep(&input_streams[i]);
@@ -1058,17 +893,21 @@  error:
     exit_program(1);
 }
 
-static void do_subtitle_out(OutputFile *of,
-                            OutputStream *ost,
-                            AVSubtitle *sub)
+static void do_subtitle_out(OutputFile *of, OutputStream *ost, AVFrame *frame)
 {
-    int subtitle_out_max_size = 1024 * 1024;
+    const int subtitle_out_max_size = 1024 * 1024;
     int subtitle_out_size, nb, i;
     AVCodecContext *enc;
     AVPacket *pkt = ost->pkt;
+    AVSubtitle out_sub = { 0 };
     int64_t pts;
 
-    if (sub->pts == AV_NOPTS_VALUE) {
+    if (!frame)
+        return;
+
+    av_log(NULL, AV_LOG_TRACE, "do_subtitle_out: sub->pts: %"PRId64"  frame->pts: %"PRId64"\n", frame->subtitle_pts, frame->pts);
+
+    if (frame->subtitle_pts == AV_NOPTS_VALUE) {
         av_log(NULL, AV_LOG_ERROR, "Subtitle packets must have a pts\n");
         if (exit_on_error)
             exit_program(1);
@@ -1094,51 +933,58 @@  static void do_subtitle_out(OutputFile *of,
         nb = 1;
 
     /* shift timestamp to honor -ss and make check_recording_time() work with -t */
-    pts = sub->pts;
+    pts = frame->subtitle_pts;
     if (output_files[ost->file_index]->start_time != AV_NOPTS_VALUE)
         pts -= output_files[ost->file_index]->start_time;
+
+    ost->sync_opts = av_rescale_q(pts, AV_TIME_BASE_Q, enc->time_base);
+    if (!check_recording_time(ost))
+        return;
+
+    frame->subtitle_pts = pts;
+    // subtitle_start_time is required to be 0
+    frame->subtitle_pts               += av_rescale_q(frame->subtitle_start_time, (AVRational){ 1, 1000 }, AV_TIME_BASE_Q);
+    frame->subtitle_end_time  -= frame->subtitle_start_time;
+    frame->subtitle_start_time = 0;
+
+    av_frame_get_subtitle(&out_sub, frame);
+
     for (i = 0; i < nb; i++) {
-        unsigned save_num_rects = sub->num_rects;
+        const unsigned save_num_rects = out_sub.num_rects;
 
-        ost->sync_opts = av_rescale_q(pts, AV_TIME_BASE_Q, enc->time_base);
-        if (!check_recording_time(ost))
-            return;
+        ost->frames_encoded++;
 
-        sub->pts = pts;
-        // start_display_time is required to be 0
-        sub->pts               += av_rescale_q(sub->start_display_time, (AVRational){ 1, 1000 }, AV_TIME_BASE_Q);
-        sub->end_display_time  -= sub->start_display_time;
-        sub->start_display_time = 0;
         if (i == 1)
-            sub->num_rects = 0;
+            out_sub.num_rects = 0;
 
-        ost->frames_encoded++;
+        subtitle_out_size = avcodec_encode_subtitle(enc, subtitle_out, subtitle_out_max_size, &out_sub);
 
-        subtitle_out_size = avcodec_encode_subtitle(enc, subtitle_out,
-                                                    subtitle_out_max_size, sub);
         if (i == 1)
-            sub->num_rects = save_num_rects;
+            out_sub.num_rects = save_num_rects;
+
         if (subtitle_out_size < 0) {
             av_log(NULL, AV_LOG_FATAL, "Subtitle encoding failed\n");
             exit_program(1);
         }
 
-        av_packet_unref(pkt);
+        //av_packet_unref(pkt);
         pkt->data = subtitle_out;
         pkt->size = subtitle_out_size;
-        pkt->pts  = av_rescale_q(sub->pts, AV_TIME_BASE_Q, ost->mux_timebase);
-        pkt->duration = av_rescale_q(sub->end_display_time, (AVRational){ 1, 1000 }, ost->mux_timebase);
+        pkt->pts  = av_rescale_q(frame->subtitle_pts, AV_TIME_BASE_Q, ost->mux_timebase);
+        pkt->duration = av_rescale_q(frame->subtitle_end_time, (AVRational){ 1, 1000 }, ost->mux_timebase);
         if (enc->codec_id == AV_CODEC_ID_DVB_SUBTITLE) {
             /* XXX: the pts correction is handled here. Maybe handling
                it in the codec would be better */
             if (i == 0)
-                pkt->pts += av_rescale_q(sub->start_display_time, (AVRational){ 1, 1000 }, ost->mux_timebase);
+                pkt->pts += av_rescale_q(frame->subtitle_start_time, (AVRational){ 1, 1000 }, ost->mux_timebase);
             else
-                pkt->pts += av_rescale_q(sub->end_display_time, (AVRational){ 1, 1000 }, ost->mux_timebase);
+                pkt->pts += av_rescale_q(frame->subtitle_end_time, (AVRational){ 1, 1000 }, ost->mux_timebase);
         }
         pkt->dts = pkt->pts;
         output_packet(of, pkt, ost, 0);
     }
+
+    avsubtitle_free(&out_sub);
 }
 
 static void do_video_out(OutputFile *of,
@@ -1568,8 +1414,26 @@  static int reap_filters(int flush)
                 }
                 do_audio_out(of, ost, filtered_frame);
                 break;
+            case AVMEDIA_TYPE_SUBTITLE:
+
+                if (filtered_frame->format == AV_SUBTITLE_FMT_ASS && !enc->subtitle_header 
+                    && filtered_frame->subtitle_header) {
+                    const char *subtitle_header = (char *)filtered_frame->subtitle_header->data;
+                    enc->subtitle_header = (uint8_t *)av_strdup(subtitle_header);
+                    if (!enc->subtitle_header)
+                        return AVERROR(ENOMEM);
+                    enc->subtitle_header_size = strlen(subtitle_header);
+                }
+
+                if ((ost->enc_ctx->width == 0 || ost->enc_ctx->height == 0)
+                    && filter->inputs[0]->w > 0 && filter->inputs[0]->h > 0 ) {
+                    ost->enc_ctx->width = filter->inputs[0]->w;
+                    ost->enc_ctx->height = filter->inputs[0]->h;
+                }
+                    
+                do_subtitle_out(of, ost, filtered_frame);
+                break;
             default:
-                // TODO support subtitle filters
                 av_assert0(0);
             }
 
@@ -1972,6 +1836,9 @@  static void flush_encoders(void)
             AVPacket *pkt = ost->pkt;
             int pkt_size;
 
+            if (!pkt)
+                break;
+
             switch (enc->codec_type) {
             case AVMEDIA_TYPE_AUDIO:
                 desc   = "audio";
@@ -2164,7 +2031,8 @@  static int ifilter_has_all_input_formats(FilterGraph *fg)
     int i;
     for (i = 0; i < fg->nb_inputs; i++) {
         if (fg->inputs[i]->format < 0 && (fg->inputs[i]->type == AVMEDIA_TYPE_AUDIO ||
-                                          fg->inputs[i]->type == AVMEDIA_TYPE_VIDEO))
+                                          fg->inputs[i]->type == AVMEDIA_TYPE_VIDEO ||
+                                          fg->inputs[i]->type == AVMEDIA_TYPE_SUBTITLE))
             return 0;
     }
     return 1;
@@ -2269,7 +2137,7 @@  static int ifilter_send_eof(InputFilter *ifilter, int64_t pts)
         // the filtergraph was never configured
         if (ifilter->format < 0)
             ifilter_parameters_from_codecpar(ifilter, ifilter->ist->st->codecpar);
-        if (ifilter->format < 0 && (ifilter->type == AVMEDIA_TYPE_AUDIO || ifilter->type == AVMEDIA_TYPE_VIDEO)) {
+        if (ifilter->format < 0 && (ifilter->type == AVMEDIA_TYPE_AUDIO || ifilter->type == AVMEDIA_TYPE_VIDEO || ifilter->type == AVMEDIA_TYPE_SUBTITLE)) {
             av_log(NULL, AV_LOG_ERROR, "Cannot determine format of input stream %d:%d after EOF\n", ifilter->ist->file_index, ifilter->ist->st->index);
             return AVERROR_INVALIDDATA;
         }
@@ -2307,7 +2175,8 @@  static int decode(AVCodecContext *avctx, AVFrame *frame, int *got_frame, AVPacke
 
 static int send_frame_to_filters(InputStream *ist, AVFrame *decoded_frame)
 {
-    int i, ret;
+    int i, ret = 0;
+    AVFrame *f;
 
     av_assert1(ist->nb_filters > 0); /* ensure ret is initialized */
     for (i = 0; i < ist->nb_filters; i++) {
@@ -2508,81 +2377,214 @@  fail:
     return err < 0 ? err : ret;
 }
 
-static int transcode_subtitles(InputStream *ist, AVPacket *pkt, int *got_output,
+static void subtitle_resend_current(InputStream *ist, int64_t heartbeat_pts)
+{
+    AVFrame *frame;
+    int64_t pts, end_pts;
+
+    if (ist->subtitle_heartbeat.recent_sub) {
+        frame = av_frame_clone(ist->subtitle_heartbeat.recent_sub);
+
+        pts     = heartbeat_pts; //av_rescale_q(frame->subtitle_pts + frame->subtitle_start_time * 1000LL, AV_TIME_BASE_Q, ist->st->time_base);
+        end_pts = av_rescale_q(frame->subtitle_pts + frame->subtitle_end_time   * 1000LL, AV_TIME_BASE_Q, ist->st->time_base);
+    }
+    else {
+        frame = av_frame_alloc();
+        if (!frame) {
+            av_log(ist->dec_ctx, AV_LOG_ERROR, "Unable to alloc frame (out of memory).\n");
+            return;
+        }
+
+        frame->type = AVMEDIA_TYPE_SUBTITLE;
+        frame->format = av_get_subtitle_format_from_codecdesc(ist->dec_ctx->codec_descriptor);
+
+        av_frame_get_buffer2(frame, 0);
+
+        frame->width = ist->subtitle_heartbeat.w;
+        frame->height = ist->subtitle_heartbeat.h;
+
+        pts       = (ist->subtitle_heartbeat.end_pts < INT64_MAX && ist->subtitle_heartbeat.end_pts > 0)
+                    ? ist->subtitle_heartbeat.end_pts : heartbeat_pts;
+        end_pts   = INT64_MAX;
+
+        frame->subtitle_pts = av_rescale_q(pts, ist->st->time_base, AV_TIME_BASE_Q);
+        frame->subtitle_end_time = 1000;
+    }
+
+    ////av_log(NULL, AV_LOG_WARNING, "subtitle_heartbeat: call subtitle_resend_current %"PRId64" \n", pts);
+
+    frame->pts = pts;
+    ist->subtitle_heartbeat.last_pts = pts;
+    ist->subtitle_heartbeat.end_pts = end_pts;
+
+    send_frame_to_filters(ist, frame);
+}
+
+static void subtitle_heartbeat(InputStream *ist, int64_t pts)
+{
+    int i;
+    int64_t pts2;
+
+    if (ist->st->codecpar->codec_type != AVMEDIA_TYPE_VIDEO)
+        return;
+
+    /* When a frame is read from a file, examine all subtitle streams in
+       the same file and send the subtitle frame again. Otherwise, decoded
+       video frames could be accumulating in the filter graph while a filter
+       (possibly overlay) is desperately waiting for a subtitle frame. */
+    for (i = 0; i < nb_input_streams; i++) {
+        InputStream *ist2 = input_streams[i];
+        if (!ist2->subtitle_heartbeat.is_active)
+            continue;
+        /* subtitles seem to be usually muxed ahead of other streams;
+           if not, subtracting a larger time here is necessary */
+        pts2 = av_rescale_q(pts, ist->st->time_base, ist2->st->time_base) - 1;
+        /* do not send the heartbeat frame if the subtitle is already ahead */
+        if (pts2 <= ist2->subtitle_heartbeat.last_pts)
+            continue;
+        if (pts2 >= ist2->subtitle_heartbeat.end_pts) {
+            /* if we have hit the end of the current displayed subpicture,
+               or if we need to initialize the system, update the
+               overlayed subpicture and its start/end times */
+            if (ist2->subtitle_heartbeat.recent_sub)
+                av_frame_free(&ist2->subtitle_heartbeat.recent_sub);
+
+            av_log(NULL, AV_LOG_DEBUG, "subtitle_heartbeat: clear + resend - pts: %"PRIi64"\n", pts2 + 1);
+            subtitle_resend_current(ist2, pts2 + 1);
+            continue;
+        }
+        if (!ist2->subtitle_heartbeat.check_buffer_requests) {
+            unsigned j, nb_reqs;
+            for (j = 0, nb_reqs = 0; j < ist2->nb_filters; j++)
+                nb_reqs += av_buffersrc_get_nb_failed_requests(ist2->filters[j]->filter);
+            if (nb_reqs) {
+                av_log(NULL, AV_LOG_DEBUG, "subtitle_heartbeat: resend - pts: %"PRIi64"\n", pts2);
+                subtitle_resend_current(ist2, pts2);
+            }
+        }
+    }
+}
+
+static InputStream *get_input_stream(OutputStream *ost)
+{
+    if (ost->source_index >= 0)
+        return input_streams[ost->source_index];
+    return NULL;
+}
+
+static int decode_subtitles(InputStream *ist, AVPacket *pkt, int *got_output,
                                int *decode_failed)
 {
-    AVSubtitle subtitle;
-    int free_sub = 1;
-    int i, ret = avcodec_decode_subtitle2(ist->dec_ctx,
-                                          &subtitle, got_output, pkt);
+    AVFrame *decoded_frame;
+    AVCodecContext *avctx = ist->dec_ctx;
+    int i = 0, ret = 0, err = 0;
+    int64_t pts, end_pts;
+
+    if (!ist->decoded_frame && !(ist->decoded_frame = av_frame_alloc()))
+        return AVERROR(ENOMEM);
+    decoded_frame = ist->decoded_frame;
+    decoded_frame->type = AVMEDIA_TYPE_SUBTITLE;
+    decoded_frame->format = av_get_subtitle_format_from_codecdesc(avctx->codec_descriptor);
 
-    check_decode_result(NULL, got_output, ret);
+    if (!ist->subtitle_heartbeat.header && avctx->subtitle_header && avctx->subtitle_header_size > 0) {
+        ist->subtitle_heartbeat.header = av_buffer_allocz(avctx->subtitle_header_size + 1);
+        if (!ist->subtitle_heartbeat.header)
+            return AVERROR(ENOMEM);
+        memcpy(ist->subtitle_heartbeat.header->data, avctx->subtitle_header, avctx->subtitle_header_size);
+    }
+
+    ret = decode(avctx, decoded_frame, got_output, pkt);
+
+    if (ret != AVERROR_EOF)
+        check_decode_result(NULL, got_output, ret);
 
     if (ret < 0 || !*got_output) {
         *decode_failed = 1;
-        if (!pkt->size)
-            sub2video_flush(ist);
+        if (!pkt->size) {
+            // Flush
+            for (i = 0; i < ist->nb_filters; i++) {
+                ret = av_buffersrc_add_frame(ist->filters[i]->filter, NULL);
+                if (ret != AVERROR_EOF && ret < 0)
+                    av_log(NULL, AV_LOG_WARNING, "Flush the frame error.\n");
+            }
+        }
         return ret;
     }
 
     if (ist->fix_sub_duration) {
         int end = 1;
-        if (ist->prev_sub.got_output) {
-            end = av_rescale(subtitle.pts - ist->prev_sub.subtitle.pts,
+        if (ist->prev_sub.got_output && ist->prev_sub.subtitle) {
+            end = av_rescale(decoded_frame->subtitle_pts - ist->prev_sub.subtitle->subtitle_pts,
                              1000, AV_TIME_BASE);
-            if (end < ist->prev_sub.subtitle.end_display_time) {
-                av_log(ist->dec_ctx, AV_LOG_DEBUG,
+            if (end < ist->prev_sub.subtitle->subtitle_end_time) {
+                av_log(avctx, AV_LOG_DEBUG,
                        "Subtitle duration reduced from %"PRId32" to %d%s\n",
-                       ist->prev_sub.subtitle.end_display_time, end,
+                       ist->prev_sub.subtitle->subtitle_end_time, end,
                        end <= 0 ? ", dropping it" : "");
-                ist->prev_sub.subtitle.end_display_time = end;
+                ist->prev_sub.subtitle->subtitle_end_time = end;
             }
         }
         FFSWAP(int,        *got_output, ist->prev_sub.got_output);
         FFSWAP(int,        ret,         ist->prev_sub.ret);
-        FFSWAP(AVSubtitle, subtitle,    ist->prev_sub.subtitle);
+        FFSWAP(AVFrame*,   decoded_frame, ist->prev_sub.subtitle);
         if (end <= 0)
-            goto out;
+            return end;
     }
 
-    if (!*got_output)
+    if (!*got_output || !decoded_frame)
         return ret;
 
-    if (ist->sub2video.frame) {
-        sub2video_update(ist, INT64_MIN, &subtitle);
-    } else if (ist->nb_filters) {
-        if (!ist->sub2video.sub_queue)
-            ist->sub2video.sub_queue = av_fifo_alloc(8 * sizeof(AVSubtitle));
-        if (!ist->sub2video.sub_queue)
-            exit_program(1);
-        if (!av_fifo_space(ist->sub2video.sub_queue)) {
-            ret = av_fifo_realloc2(ist->sub2video.sub_queue, 2 * av_fifo_size(ist->sub2video.sub_queue));
-            if (ret < 0)
-                exit_program(1);
-        }
-        av_fifo_generic_write(ist->sub2video.sub_queue, &subtitle, sizeof(subtitle), NULL);
-        free_sub = 0;
-    }
+    decoded_frame->type = AVMEDIA_TYPE_SUBTITLE;
+    decoded_frame->format = av_get_subtitle_format_from_codecdesc(avctx->codec_descriptor);
 
-    if (!subtitle.num_rects)
-        goto out;
+    ////if ((ret = av_frame_get_buffer2(decoded_frame, 0)) < 0)
+    ////    return ret;
 
-    ist->frames_decoded++;
+    if ((ret = av_buffer_replace(&decoded_frame->subtitle_header, ist->subtitle_heartbeat.header)) < 0)
+        return ret;
 
-    for (i = 0; i < nb_output_streams; i++) {
-        OutputStream *ost = output_streams[i];
+    pts     = av_rescale_q(decoded_frame->subtitle_pts + decoded_frame->subtitle_start_time * 1000LL,
+                           AV_TIME_BASE_Q, ist->st->time_base);
+    end_pts = av_rescale_q(decoded_frame->subtitle_pts + decoded_frame->subtitle_end_time   * 1000LL,
+                             AV_TIME_BASE_Q, ist->st->time_base);
 
-        if (!check_output_constraints(ist, ost) || !ost->encoding_needed
-            || ost->enc->type != AVMEDIA_TYPE_SUBTITLE)
-            continue;
+    ist->subtitle_heartbeat.last_pts = decoded_frame->pts = pts;
+    ist->subtitle_heartbeat.end_pts = end_pts;
 
-        do_subtitle_out(output_files[ost->file_index], ost, &subtitle);
+    if (ist->nb_filters > 0) {
+        AVFrame *filter_frame = av_frame_clone(decoded_frame);
+        if (!filter_frame)
+            err = AVERROR(ENOMEM);
+        else
+            err = send_frame_to_filters(ist, filter_frame);
     }
 
-out:
-    if (free_sub)
-        avsubtitle_free(&subtitle);
-    return ret;
+    if (err >= 0) {
+        for (i = 0; i < nb_output_streams; i++) {
+            OutputStream *ost = output_streams[i];
+            InputStream *ist_src = get_input_stream(ost);
+
+            if (!ist_src || !check_output_constraints(ist, ost) 
+                || ist_src != ist
+                || !ost->encoding_needed
+                || ost->enc->type != AVMEDIA_TYPE_SUBTITLE)
+                continue;
+
+            if (ost->filter && ost->filter->filter->nb_inputs > 0)
+                continue;
+
+            ////if (!ost->pkt && !((ost->pkt = av_packet_alloc())))
+            ////    exit_program(1);
+            do_subtitle_out(output_files[ost->file_index], ost, decoded_frame);
+        }
+    }
+
+    av_frame_free(&ist->subtitle_heartbeat.recent_sub);
+    ist->subtitle_heartbeat.recent_sub = av_frame_clone(decoded_frame);
+
+
+    av_frame_unref(decoded_frame);
+    return err < 0 ? err : ret;
 }
 
 static int send_filter_eof(InputStream *ist)
@@ -2686,7 +2688,7 @@  static int process_input_packet(InputStream *ist, const AVPacket *pkt, int no_eo
         case AVMEDIA_TYPE_SUBTITLE:
             if (repeating)
                 break;
-            ret = transcode_subtitles(ist, avpkt, &got_output, &decode_failed);
+            ret = decode_subtitles(ist, avpkt, &got_output, &decode_failed);
             if (!pkt && ret >= 0)
                 ret = AVERROR_EOF;
             av_packet_unref(avpkt);
@@ -2977,13 +2979,6 @@  FF_ENABLE_DEPRECATION_WARNINGS
     return 0;
 }
 
-static InputStream *get_input_stream(OutputStream *ost)
-{
-    if (ost->source_index >= 0)
-        return input_streams[ost->source_index];
-    return NULL;
-}
-
 static int compare_int64(const void *a, const void *b)
 {
     return FFDIFFSIGN(*(const int64_t *)a, *(const int64_t *)b);
@@ -3462,7 +3457,7 @@  static int init_output_stream_encode(OutputStream *ost, AVFrame *frame)
         break;
     case AVMEDIA_TYPE_SUBTITLE:
         enc_ctx->time_base = AV_TIME_BASE_Q;
-        if (!enc_ctx->width) {
+        if (!enc_ctx->width && ost->source_index >= 0) {
             enc_ctx->width     = input_streams[ost->source_index]->st->codecpar->width;
             enc_ctx->height    = input_streams[ost->source_index]->st->codecpar->height;
         }
@@ -3515,19 +3510,14 @@  static int init_output_stream(OutputStream *ost, AVFrame *frame,
         }
 
         if (ist && ist->dec->type == AVMEDIA_TYPE_SUBTITLE && ost->enc->type == AVMEDIA_TYPE_SUBTITLE) {
-            int input_props = 0, output_props = 0;
-            AVCodecDescriptor const *input_descriptor =
-                avcodec_descriptor_get(dec->codec_id);
-            AVCodecDescriptor const *output_descriptor =
-                avcodec_descriptor_get(ost->enc_ctx->codec_id);
-            if (input_descriptor)
-                input_props = input_descriptor->props & (AV_CODEC_PROP_TEXT_SUB | AV_CODEC_PROP_BITMAP_SUB);
-            if (output_descriptor)
-                output_props = output_descriptor->props & (AV_CODEC_PROP_TEXT_SUB | AV_CODEC_PROP_BITMAP_SUB);
-            if (input_props && output_props && input_props != output_props) {
-                snprintf(error, error_len,
-                         "Subtitle encoding currently only possible from text to text "
-                         "or bitmap to bitmap");
+            AVCodecDescriptor const *input_descriptor     = avcodec_descriptor_get(dec->codec_id);
+            AVCodecDescriptor const *output_descriptor    = avcodec_descriptor_get(ost->enc_ctx->codec_id);
+            const enum AVSubtitleType in_subtitle_format  = output_descriptor ? av_get_subtitle_format_from_codecdesc(input_descriptor) : AV_SUBTITLE_FMT_UNKNOWN;
+            const enum AVSubtitleType out_subtitle_format = output_descriptor ? av_get_subtitle_format_from_codecdesc(output_descriptor) : AV_SUBTITLE_FMT_UNKNOWN;
+
+            if (ist->nb_filters == 0 && in_subtitle_format != AV_SUBTITLE_FMT_UNKNOWN && out_subtitle_format != AV_SUBTITLE_FMT_UNKNOWN
+                && in_subtitle_format != out_subtitle_format) {
+                snprintf(error, error_len, "Subtitle encoding is only possible from text to text or bitmap to bitmap");
                 return AVERROR_INVALIDDATA;
             }
         }
@@ -4528,7 +4518,7 @@  static int process_input(int file_index)
                av_ts2timestr(input_files[ist->file_index]->ts_offset, &AV_TIME_BASE_Q));
     }
 
-    sub2video_heartbeat(ist, pkt->pts);
+    subtitle_heartbeat(ist, pkt->pts);
 
     process_input_packet(ist, pkt, 0);
 
@@ -4740,6 +4730,7 @@  static int transcode(void)
     /* at the end of stream, we must flush the decoder buffers */
     for (i = 0; i < nb_input_streams; i++) {
         ist = input_streams[i];
+        ist->subtitle_heartbeat.is_active = 0;
         if (!input_files[ist->file_index]->eof_reached) {
             process_input_packet(ist, NULL, 0);
         }
diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
index 1728010f56..8a7080834a 100644
--- a/fftools/ffmpeg.h
+++ b/fftools/ffmpeg.h
@@ -349,17 +349,18 @@  typedef struct InputStream {
     struct { /* previous decoded subtitle and related variables */
         int got_output;
         int ret;
-        AVSubtitle subtitle;
+        AVFrame *subtitle;
     } prev_sub;
 
-    struct sub2video {
+    struct subtitle_heartbeat {
+        int is_active;
+        int check_buffer_requests;
         int64_t last_pts;
         int64_t end_pts;
-        AVFifoBuffer *sub_queue;    ///< queue of AVSubtitle* before filter init
-        AVFrame *frame;
+        AVFrame *recent_sub;
         int w, h;
-        unsigned int initialize; ///< marks if sub2video_update should force an initialization
-    } sub2video;
+        AVBufferRef *header;
+    } subtitle_heartbeat;
 
     /* decoded data from this stream goes into all those filters
      * currently video and audio only */
@@ -659,8 +660,6 @@  int filtergraph_is_simple(FilterGraph *fg);
 int init_simple_filtergraph(InputStream *ist, OutputStream *ost);
 int init_complex_filtergraph(FilterGraph *fg);
 
-void sub2video_update(InputStream *ist, int64_t heartbeat_pts, AVSubtitle *sub);
-
 int ifilter_parameters_from_frame(InputFilter *ifilter, const AVFrame *frame);
 
 int ffmpeg_parse_options(int argc, char **argv);
diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c
index 452b689d62..1299529d36 100644
--- a/fftools/ffmpeg_filter.c
+++ b/fftools/ffmpeg_filter.c
@@ -22,6 +22,8 @@ 
 
 #include "ffmpeg.h"
 
+#include <libavutil/ass_split_internal.h>
+
 #include "libavfilter/avfilter.h"
 #include "libavfilter/buffersink.h"
 #include "libavfilter/buffersrc.h"
@@ -30,11 +32,9 @@ 
 #include "libavutil/avstring.h"
 #include "libavutil/bprint.h"
 #include "libavutil/channel_layout.h"
-#include "libavutil/display.h"
 #include "libavutil/opt.h"
 #include "libavutil/pixdesc.h"
 #include "libavutil/pixfmt.h"
-#include "libavutil/imgutils.h"
 #include "libavutil/samplefmt.h"
 
 // FIXME: YUV420P etc. are actually supported with full color range,
@@ -221,9 +221,8 @@  static void init_input_filter(FilterGraph *fg, AVFilterInOut *in)
     enum AVMediaType type = avfilter_pad_get_type(in->filter_ctx->input_pads, in->pad_idx);
     int i;
 
-    // TODO: support other filter types
-    if (type != AVMEDIA_TYPE_VIDEO && type != AVMEDIA_TYPE_AUDIO) {
-        av_log(NULL, AV_LOG_FATAL, "Only video and audio filters supported "
+    if (type != AVMEDIA_TYPE_VIDEO && type != AVMEDIA_TYPE_AUDIO && type != AVMEDIA_TYPE_SUBTITLE) {
+        av_log(NULL, AV_LOG_FATAL, "Only video, audio and subtitle filters supported "
                "currently.\n");
         exit_program(1);
     }
@@ -244,8 +243,9 @@  static void init_input_filter(FilterGraph *fg, AVFilterInOut *in)
         for (i = 0; i < s->nb_streams; i++) {
             enum AVMediaType stream_type = s->streams[i]->codecpar->codec_type;
             if (stream_type != type &&
-                !(stream_type == AVMEDIA_TYPE_SUBTITLE &&
-                  type == AVMEDIA_TYPE_VIDEO /* sub2video hack */))
+                // in the followng case we auto-insert the graphicsub2video conversion filter 
+                // for retaining compatibility with the previous sub2video hack
+                !(stream_type == AVMEDIA_TYPE_SUBTITLE && type == AVMEDIA_TYPE_VIDEO))
                 continue;
             if (check_stream_specifier(s, s->streams[i], *p == ':' ? p + 1 : p) == 1) {
                 st = s->streams[i];
@@ -294,6 +294,13 @@  static void init_input_filter(FilterGraph *fg, AVFilterInOut *in)
     fg->inputs[fg->nb_inputs - 1]->type = ist->st->codecpar->codec_type;
     fg->inputs[fg->nb_inputs - 1]->name = describe_filter_link(fg, in, 1);
 
+    if (ist->st->codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE && ist->dec_ctx) {
+        const AVCodecDescriptor *codec_descriptor = ist->dec_ctx->codec_descriptor;
+        if (!codec_descriptor)
+            codec_descriptor = avcodec_descriptor_get(ist->dec_ctx->codec_id);
+        fg->inputs[fg->nb_inputs - 1]->format = av_get_subtitle_format_from_codecdesc(codec_descriptor);
+    }
+
     fg->inputs[fg->nb_inputs - 1]->frame_queue = av_fifo_alloc(8 * sizeof(AVFrame*));
     if (!fg->inputs[fg->nb_inputs - 1]->frame_queue)
         exit_program(1);
@@ -416,6 +423,39 @@  static int insert_filter(AVFilterContext **last_filter, int *pad_idx,
     return 0;
 }
 
+static int configure_output_subtitle_filter(FilterGraph *fg, OutputFilter *ofilter, AVFilterInOut *out)
+{
+    OutputStream *ost = ofilter->ost;
+    AVFilterContext *last_filter = out->filter_ctx;
+    int pad_idx = out->pad_idx;
+    int ret;
+    char name[255];
+
+    snprintf(name, sizeof(name), "out_%d_%d", ost->file_index, ost->index);
+    ret = avfilter_graph_create_filter(&ofilter->filter,
+                                       avfilter_get_by_name("sbuffersink"),
+                                       name, NULL, NULL, fg->graph);
+
+    if (ret < 0) {
+        av_log(NULL, AV_LOG_ERROR, "Unable to create filter sbuffersink\n");
+        return ret;
+    }
+
+    ////snprintf(name, sizeof(name), "trim_out_%d_%d",
+    ////         ost->file_index, ost->index);
+    ////ret = insert_trim(of->start_time, of->recording_time,
+    ////                  &last_filter, &pad_idx, name);
+    ////if (ret < 0)
+    ////    return ret;
+
+    ////ost->st->codecpar->codec_tag = MKTAG('a', 's', 's', 's');
+
+    if ((ret = avfilter_link(last_filter, pad_idx, ofilter->filter, 0)) < 0)
+        return ret;
+
+    return 0;
+}
+
 static int configure_output_video_filter(FilterGraph *fg, OutputFilter *ofilter, AVFilterInOut *out)
 {
     char *pix_fmts;
@@ -594,7 +634,8 @@  static int configure_output_audio_filter(FilterGraph *fg, OutputFilter *ofilter,
         int i;
 
         for (i=0; i<of->ctx->nb_streams; i++)
-            if (of->ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO)
+            if (of->ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO ||
+                of->ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE)
                 break;
 
         if (i<of->ctx->nb_streams) {
@@ -628,6 +669,7 @@  static int configure_output_filter(FilterGraph *fg, OutputFilter *ofilter,
     switch (avfilter_pad_get_type(out->filter_ctx->output_pads, out->pad_idx)) {
     case AVMEDIA_TYPE_VIDEO: return configure_output_video_filter(fg, ofilter, out);
     case AVMEDIA_TYPE_AUDIO: return configure_output_audio_filter(fg, ofilter, out);
+    case AVMEDIA_TYPE_SUBTITLE: return configure_output_subtitle_filter(fg, ofilter, out);
     default: av_assert0(0); return 0;
     }
 }
@@ -647,51 +689,112 @@  void check_filter_outputs(void)
     }
 }
 
-static int sub2video_prepare(InputStream *ist, InputFilter *ifilter)
+static int configure_input_subtitle_filter(FilterGraph *fg, InputFilter *ifilter,
+                                        AVFilterInOut *in)
 {
-    AVFormatContext *avf = input_files[ist->file_index]->ctx;
-    int i, w, h;
+    AVFilterContext *last_filter;
+    const AVFilter *buffer_filt = avfilter_get_by_name("sbuffer");
+    InputStream *ist = ifilter->ist;
+    AVBPrint args;
+    char name[255];
+    int ret, pad_idx = 0;
+    int w, h;
+    AVBufferSrcParameters *par = av_buffersrc_parameters_alloc();
+    enum AVMediaType media_type;
+
+    if (!par)
+        return AVERROR(ENOMEM);
+    memset(par, 0, sizeof(*par));
+    par->format = AV_PIX_FMT_NONE;
+
+    if (ist->dec_ctx->codec_type == AVMEDIA_TYPE_AUDIO) {
+        av_log(NULL, AV_LOG_ERROR, "Cannot connect subtitle filter to audio input\n");
+        ret = AVERROR(EINVAL);
+        goto fail;
+    }
+
+    if (ist->dec_ctx->codec_type == AVMEDIA_TYPE_VIDEO) {
+        av_log(NULL, AV_LOG_ERROR, "Cannot connect subtitle filter to video input\n");
+        ret = AVERROR(EINVAL);
+        goto fail;
+    }
+
+    if (!ist->subtitle_heartbeat.header && ist->dec_ctx->subtitle_header && ist->dec_ctx->subtitle_header_size > 0) {
+        ist->subtitle_heartbeat.header = av_buffer_allocz(ist->dec_ctx->subtitle_header_size + 1);
+        if (!ist->subtitle_heartbeat.header)
+            return AVERROR(ENOMEM);
+        memcpy(ist->subtitle_heartbeat.header->data, ist->dec_ctx->subtitle_header, ist->dec_ctx->subtitle_header_size);
+    }
+
+    ist->subtitle_heartbeat.is_active = 1;
 
-    /* Compute the size of the canvas for the subtitles stream.
-       If the subtitles codecpar has set a size, use it. Otherwise use the
-       maximum dimensions of the video streams in the same file. */
     w = ifilter->width;
     h = ifilter->height;
+
     if (!(w && h)) {
-        for (i = 0; i < avf->nb_streams; i++) {
-            if (avf->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
-                w = FFMAX(w, avf->streams[i]->codecpar->width);
-                h = FFMAX(h, avf->streams[i]->codecpar->height);
-            }
-        }
-        if (!(w && h)) {
-            w = FFMAX(w, 720);
-            h = FFMAX(h, 576);
-        }
-        av_log(avf, AV_LOG_INFO, "sub2video: using %dx%d canvas\n", w, h);
+        w = ist->dec_ctx->width;
+        h = ist->dec_ctx->height;
     }
-    ist->sub2video.w = ifilter->width  = w;
-    ist->sub2video.h = ifilter->height = h;
 
-    ifilter->width  = ist->dec_ctx->width  ? ist->dec_ctx->width  : ist->sub2video.w;
-    ifilter->height = ist->dec_ctx->height ? ist->dec_ctx->height : ist->sub2video.h;
+    if (!(w && h) && ist->dec_ctx->subtitle_header) {
+        ASSSplitContext *ass_ctx = avpriv_ass_split((char *)ist->dec_ctx->subtitle_header);
+        ASS *ass = (ASS *)ass_ctx;
+        w = ass->script_info.play_res_x;
+        h = ass->script_info.play_res_y;
+        avpriv_ass_split_free(ass_ctx);
+    }
 
-    /* rectangles are AV_PIX_FMT_PAL8, but we have no guarantee that the
-       palettes for all rectangles are identical or compatible */
-    ifilter->format = AV_PIX_FMT_RGB32;
+    ist->subtitle_heartbeat.w = w;
+    ist->subtitle_heartbeat.h = h;
+    av_log(ifilter, AV_LOG_INFO, "subtitle input filter: decoding size %dx%d\n", ist->subtitle_heartbeat.w, ist->subtitle_heartbeat.h);
 
-    ist->sub2video.frame = av_frame_alloc();
-    if (!ist->sub2video.frame)
-        return AVERROR(ENOMEM);
-    ist->sub2video.last_pts = INT64_MIN;
-    ist->sub2video.end_pts  = INT64_MIN;
+    ifilter->width = w;
+    ifilter->height = h;
+    ist->dec_ctx->width = w;
+    ist->dec_ctx->height = h;
 
-    /* sub2video structure has been (re-)initialized.
-       Mark it as such so that the system will be
-       initialized with the first received heartbeat. */
-    ist->sub2video.initialize = 1;
+    ist->subtitle_heartbeat.last_pts = INT64_MIN;
+
+    snprintf(name, sizeof(name), "graph %d subtitle input from stream %d:%d", fg->index,
+             ist->file_index, ist->st->index);
+
+
+    av_bprint_init(&args, 0, AV_BPRINT_SIZE_AUTOMATIC);
+    av_bprintf(&args,
+             "subtitle_type=%d:width=%d:height=%d:time_base=%d/%d:",
+             ifilter->format, ifilter->width, ifilter->height,
+             ist->st->time_base.num, ist->st->time_base.den);
+    if ((ret = avfilter_graph_create_filter(&ifilter->filter, buffer_filt, name,
+                                            args.str, NULL, fg->graph)) < 0)
+        goto fail;
+
+    par->hw_frames_ctx = ifilter->hw_frames_ctx;
+    par->format = ifilter->format;
+    par->width = ifilter->width;
+    par->height = ifilter->height;
+
+    ret = av_buffersrc_parameters_set(ifilter->filter, par);
+    if (ret < 0)
+        goto fail;
+    av_freep(&par);
+    last_filter = ifilter->filter;
+
+    media_type = avfilter_pad_get_type(in->filter_ctx->input_pads, in->pad_idx);
+    if (media_type == AVMEDIA_TYPE_VIDEO) {
+        av_log(NULL, AV_LOG_INFO, "Auto-inserting graphicsub2video filter\n");
+        ret = insert_filter(&last_filter, &pad_idx, "graphicsub2video", NULL);
+        if (ret < 0)
+            return ret;
+    }
+
+    if ((ret = avfilter_link(last_filter, 0, in->filter_ctx, in->pad_idx)) < 0)
+        return ret;
 
     return 0;
+fail:
+    av_freep(&par);
+
+    return ret;
 }
 
 static int configure_input_video_filter(FilterGraph *fg, InputFilter *ifilter,
@@ -710,8 +813,15 @@  static int configure_input_video_filter(FilterGraph *fg, InputFilter *ifilter,
     char name[255];
     int ret, pad_idx = 0;
     int64_t tsoffset = 0;
-    AVBufferSrcParameters *par = av_buffersrc_parameters_alloc();
+    AVBufferSrcParameters *par;
+
+    if (ist->dec_ctx->codec_type == AVMEDIA_TYPE_SUBTITLE) {
+        // Automatically insert conversion filter to retain compatibility
+        // with sub2video command lines
+        return configure_input_subtitle_filter(fg, ifilter, in);
+    }
 
+    par = av_buffersrc_parameters_alloc();
     if (!par)
         return AVERROR(ENOMEM);
     memset(par, 0, sizeof(*par));
@@ -726,12 +836,6 @@  static int configure_input_video_filter(FilterGraph *fg, InputFilter *ifilter,
     if (!fr.num)
         fr = av_guess_frame_rate(input_files[ist->file_index]->ctx, ist->st, NULL);
 
-    if (ist->dec_ctx->codec_type == AVMEDIA_TYPE_SUBTITLE) {
-        ret = sub2video_prepare(ist, ifilter);
-        if (ret < 0)
-            goto fail;
-    }
-
     sar = ifilter->sample_aspect_ratio;
     if(!sar.den)
         sar = (AVRational){0,1};
@@ -743,7 +847,7 @@  static int configure_input_video_filter(FilterGraph *fg, InputFilter *ifilter,
              tb.num, tb.den, sar.num, sar.den);
     if (fr.num && fr.den)
         av_bprintf(&args, ":frame_rate=%d/%d", fr.num, fr.den);
-    snprintf(name, sizeof(name), "graph %d input from stream %d:%d", fg->index,
+    snprintf(name, sizeof(name), "graph %d video input from stream %d:%d", fg->index,
              ist->file_index, ist->st->index);
 
 
@@ -949,6 +1053,7 @@  static int configure_input_filter(FilterGraph *fg, InputFilter *ifilter,
     switch (avfilter_pad_get_type(in->filter_ctx->input_pads, in->pad_idx)) {
     case AVMEDIA_TYPE_VIDEO: return configure_input_video_filter(fg, ifilter, in);
     case AVMEDIA_TYPE_AUDIO: return configure_input_audio_filter(fg, ifilter, in);
+    case AVMEDIA_TYPE_SUBTITLE: return configure_input_subtitle_filter(fg, ifilter, in);
     default: av_assert0(0); return 0;
     }
 }
@@ -1116,19 +1221,6 @@  int configure_filtergraph(FilterGraph *fg)
         }
     }
 
-    /* process queued up subtitle packets */
-    for (i = 0; i < fg->nb_inputs; i++) {
-        InputStream *ist = fg->inputs[i]->ist;
-        if (ist->sub2video.sub_queue && ist->sub2video.frame) {
-            while (av_fifo_size(ist->sub2video.sub_queue)) {
-                AVSubtitle tmp;
-                av_fifo_generic_read(ist->sub2video.sub_queue, &tmp, sizeof(tmp), NULL);
-                sub2video_update(ist, INT64_MIN, &tmp);
-                avsubtitle_free(&tmp);
-            }
-        }
-    }
-
     return 0;
 
 fail:
@@ -1151,6 +1243,7 @@  int ifilter_parameters_from_frame(InputFilter *ifilter, const AVFrame *frame)
     ifilter->sample_rate         = frame->sample_rate;
     ifilter->channels            = frame->channels;
     ifilter->channel_layout      = frame->channel_layout;
+    ifilter->type                = frame->type;
 
     av_freep(&ifilter->displaymatrix);
     sd = av_frame_get_side_data(frame, AV_FRAME_DATA_DISPLAYMATRIX);
diff --git a/fftools/ffmpeg_hw.c b/fftools/ffmpeg_hw.c
index 14e702bd92..be69d54aaf 100644
--- a/fftools/ffmpeg_hw.c
+++ b/fftools/ffmpeg_hw.c
@@ -449,7 +449,7 @@  int hw_device_setup_for_encode(OutputStream *ost)
     AVBufferRef *frames_ref = NULL;
     int i;
 
-    if (ost->filter) {
+    if (ost->filter && ost->filter->filter) {
         frames_ref = av_buffersink_get_hw_frames_ctx(ost->filter->filter);
         if (frames_ref &&
             ((AVHWFramesContext*)frames_ref->data)->format ==
diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c
index e55b584fd4..d443a5b8c8 100644
--- a/fftools/ffmpeg_opt.c
+++ b/fftools/ffmpeg_opt.c
@@ -2207,8 +2207,9 @@  static void init_output_filter(OutputFilter *ofilter, OptionsContext *o,
     switch (ofilter->type) {
     case AVMEDIA_TYPE_VIDEO: ost = new_video_stream(o, oc, -1); break;
     case AVMEDIA_TYPE_AUDIO: ost = new_audio_stream(o, oc, -1); break;
+    case AVMEDIA_TYPE_SUBTITLE: ost = new_subtitle_stream(o, oc, -1); break;
     default:
-        av_log(NULL, AV_LOG_FATAL, "Only video and audio filters are supported "
+        av_log(NULL, AV_LOG_FATAL, "Only video, audio and subtitle filters are supported "
                "currently.\n");
         exit_program(1);
     }
diff --git a/tests/ref/fate/filter-overlay-dvdsub-2397 b/tests/ref/fate/filter-overlay-dvdsub-2397
index 483e5fa4e0..3bf198d6ad 100644
--- a/tests/ref/fate/filter-overlay-dvdsub-2397
+++ b/tests/ref/fate/filter-overlay-dvdsub-2397
@@ -490,368 +490,367 @@ 
 1,       3877,       3877,       10,     2013, 0x95a39f9c
 1,       3887,       3887,       10,     2013, 0x4f7ea123
 1,       3897,       3897,       10,     2013, 0x9efb9ba1
-0,        117,        117,        1,   518400, 0xbf8523da
+0,        117,        117,        1,   518400, 0x61e0f688
 1,       3907,       3907,       10,     2013, 0xf395b2cd
 1,       3917,       3917,       10,     2013, 0x261a881e
 1,       3927,       3927,       10,     2013, 0x7f2d9f72
 1,       3937,       3937,       10,     2013, 0x0105b38d
-0,        118,        118,        1,   518400, 0x41890ed6
+0,        118,        118,        1,   518400, 0xa47de755
 1,       3952,       3952,       10,     2013, 0x0e5db67e
 1,       3962,       3962,       10,     2013, 0xfc9baf97
-0,        119,        119,        1,   518400, 0x588534fc
+0,        119,        119,        1,   518400, 0x52d52e73
 1,       3972,       3972,       10,     2013, 0x8e02a1b1
 1,       3982,       3982,       10,     2013, 0x6eecaac8
 1,       3992,       3992,       10,     2013, 0xf5558f0c
 1,       4002,       4002,       10,     2013, 0x512ba99b
-0,        120,        120,        1,   518400, 0x2145ebc1
+0,        120,        120,        1,   518400, 0xc732e546
 1,       4012,       4012,       10,     2013, 0x932b9932
 1,       4022,       4022,       10,     2013, 0xc01ea987
-0,        121,        121,        1,   518400, 0x28bca595
+0,        121,        121,        1,   518400, 0xc36f9f14
 1,       4038,       4038,       10,     2013, 0x10879cf7
 1,       4048,       4048,       10,     2013, 0x90679338
 1,       4058,       4058,       10,     2013, 0x077d8a9e
 1,       4068,       4068,       10,     2013, 0x969fa57c
-0,        122,        122,        1,   518400, 0x77dc951e
+0,        122,        122,        1,   518400, 0x78428e8b
 1,       4078,       4078,       10,     2013, 0xe049ab07
 1,       4088,       4088,       10,     2013, 0xf535b3b3
 1,       4098,       4098,       10,     2013, 0xfe76bd37
-0,        123,        123,        1,   518400, 0xe8924c17
+0,        123,        123,        1,   518400, 0xf0f8458d
 1,       4108,       4108,       10,     2013, 0xde79ad8c
 1,       4123,       4123,       10,     2013, 0xe89b9c47
 1,       4133,       4133,       10,     2013, 0xc570b0f0
-0,        124,        124,        1,   518400, 0xadb4cccc
+0,        124,        124,        1,   518400, 0x7083c653
 1,       4143,       4143,       10,     2013, 0xee709cd9
 1,       4153,       4153,       10,     2013, 0xcfe5afab
 1,       4163,       4163,       10,     2013, 0x98ff8ce4
-0,        125,        125,        1,   518400, 0x1d7b56ac
+0,        125,        125,        1,   518400, 0xa105502c
 1,       4173,       4173,       10,     2013, 0x9d19b44c
 1,       4183,       4183,       10,     2013, 0x4349917a
 1,       4193,       4193,       10,     2013, 0xbf54a59a
-0,        126,        126,        1,   518400, 0xad5739a4
+0,        126,        126,        1,   518400, 0xd411331a
 1,       4208,       4208,       10,     2013, 0xc4a399e0
 1,       4218,       4218,       10,     2013, 0x1bf58ff0
 1,       4228,       4228,       10,     2013, 0x3518ac56
-0,        127,        127,        1,   518400, 0x2733d35a
+0,        127,        127,        1,   518400, 0x83b0ccdb
 1,       4238,       4238,       10,     2013, 0xcd38c1de
 1,       4248,       4248,       10,     2013, 0xbe7d9c4d
 1,       4258,       4258,       10,     2013, 0xe113a306
 1,       4268,       4268,       10,     2013, 0x083197ea
-0,        128,        128,        1,   518400, 0x78e76da2
+0,        128,        128,        1,   518400, 0xa9be671a
 1,       4278,       4278,       10,     2013, 0x1929b1eb
 1,       4294,       4294,       10,     2013, 0x5d6ea5af
 1,       4304,       4304,       10,     2013, 0x05519d53
-0,        129,        129,        1,   518400, 0x6c076013
+0,        129,        129,        1,   518400, 0xaeb75983
 1,       4314,       4314,       10,     2013, 0x5773b380
 1,       4324,       4324,       10,     2013, 0xaa70a8f5
 1,       4334,       4334,       10,     2013, 0x990db0ec
-0,        130,        130,        1,   518400, 0x7854f2b1
+0,        130,        130,        1,   518400, 0x81f8ec13
 1,       4344,       4344,       10,     2013, 0x91d3a623
 1,       4354,       4354,       10,     2013, 0xc91f9824
 1,       4364,       4364,       10,     2013, 0x1d058abf
-0,        131,        131,        1,   518400, 0xd2ae1ecd
+0,        131,        131,        1,   518400, 0x8aaa1839
 1,       4379,       4379,       10,     2013, 0x8de1b8d5
 1,       4389,       4389,       10,     2013, 0x7872b06b
 1,       4399,       4399,       10,     2013, 0xa084c203
-0,        132,        132,        1,   518400, 0xf5eab38d
+0,        132,        132,        1,   518400, 0xc98bacf5
 1,       4409,       4409,       10,     2013, 0xff90ae8d
 1,       4419,       4419,       10,     2013, 0x61dead8e
 1,       4429,       4429,       10,     2013, 0xee76b284
-0,        133,        133,        1,   518400, 0x994d3e9c
+0,        133,        133,        1,   518400, 0x31083804
 1,       4439,       4439,       10,     2013, 0xe888af7f
 1,       4449,       4449,       10,     2013, 0x5d57b115
 1,       4464,       4464,       10,     2013, 0xcdbfb1d0
-0,        134,        134,        1,   518400, 0x95ab705a
+0,        134,        134,        1,   518400, 0x540a69dc
 1,       4474,       4474,       10,     2013, 0x2e28a952
 1,       4484,       4484,       10,     2013, 0x4795a994
 1,       4494,       4494,       10,     2013, 0x7e7ea304
 1,       4504,       4504,       10,     2013, 0x9502c1e1
-0,        135,        135,        1,   518400, 0x3c83c5ce
+0,        135,        135,        1,   518400, 0x80d3bf46
 1,       4514,       4514,       10,     2013, 0xf7c78ab2
 1,       4524,       4524,       10,     2013, 0x24049816
 1,       4534,       4534,       10,     2013, 0x52089dcf
-0,        136,        136,        1,   518400, 0xfa22c508
+0,        136,        136,        1,   518400, 0x2967be7f
 1,       4550,       4550,       10,     2013, 0x2150a0b1
 1,       4560,       4560,       10,     2013, 0x3c2e9b93
 1,       4570,       4570,       10,     2013, 0x491f932b
-0,        137,        137,        1,   518400, 0xddda1712
+0,        137,        137,        1,   518400, 0x5a3b1092
 1,       4580,       4580,       10,     2013, 0x31359cf8
 1,       4590,       4590,       10,     2013, 0x1b00ac3f
 1,       4600,       4600,       10,     2013, 0x8d7ab3cb
-0,        138,        138,        1,   518400, 0x985a3b93
+0,        138,        138,        1,   518400, 0x8741350b
 1,       4610,       4610,       10,     2013, 0xb2c2a4de
 1,       4620,       4620,       10,     2013, 0x80a4abf2
 1,       4635,       4635,       10,     2013, 0x0701a4ee
-0,        139,        139,        1,   518400, 0xea63c5e7
+0,        139,        139,        1,   518400, 0xd5a9bf60
 1,       4645,       4645,       10,     2013, 0xdc1ba5bc
 1,       4655,       4655,       10,     2013, 0x6083a8a4
 1,       4665,       4665,       10,     2013, 0x6226ad45
-0,        140,        140,        1,   518400, 0xef64983d
+0,        140,        140,        1,   518400, 0xc05f91ba
 1,       4675,       4675,       10,     2013, 0x2732a205
 1,       4685,       4685,       10,     2013, 0x0f62a0d3
 1,       4695,       4695,       10,     2013, 0xc1799249
-0,        141,        141,        1,   518400, 0x747bb193
+0,        141,        141,        1,   518400, 0x3fdaab0b
 1,       4705,       4705,       10,     2013, 0xbccfa9c8
 1,       4720,       4720,       10,     2013, 0xded096e7
 1,       4730,       4730,       10,     2013, 0x7f0daf43
-0,        142,        142,        1,   518400, 0xb8748862
+0,        142,        142,        1,   518400, 0xab7281d9
 1,       4740,       4740,       10,     2013, 0xc47ea682
 1,       4750,       4750,       10,     2013, 0x5a72b07a
 1,       4760,       4760,       10,     2013, 0x386faa8c
 1,       4770,       4770,       10,     2013, 0xf9919a91
-0,        143,        143,        1,   518400, 0xaab55a5f
+0,        143,        143,        1,   518400, 0xc80053d6
 1,       4780,       4780,       10,     2013, 0x4908897e
 1,       4790,       4790,       10,     2013, 0x4882b594
-0,        144,        144,        1,   518400, 0x7b468add
+0,        144,        144,        1,   518400, 0x6526845c
 1,       4806,       4806,       10,     2013, 0x113e98d1
 1,       4816,       4816,       10,     2013, 0x5098b30d
 1,       4826,       4826,       10,     2013, 0x0ef7b857
 1,       4836,       4836,       10,     2013, 0x216ea176
-0,        145,        145,        1,   518400, 0xf2078707
+0,        145,        145,        1,   518400, 0x1b788089
 1,       4846,       4846,       10,     2013, 0xf906944a
 1,       4856,       4856,       10,     2013, 0xee9b92fb
 1,       4866,       4866,       10,     2013, 0xd6029209
-0,        146,        146,        1,   518400, 0x6a2d931e
+0,        146,        146,        1,   518400, 0xfa8e8ca9
 1,       4876,       4876,       10,     2013, 0x2256a12e
 1,       4891,       4891,       10,     2013, 0x89de8e4a
 1,       4901,       4901,       10,     2013, 0x0bf0a584
-0,        147,        147,        1,   518400, 0xbbe3c417
+0,        147,        147,        1,   518400, 0xb278bda1
 1,       4911,       4911,       10,     2013, 0x6a5ebd58
 1,       4921,       4921,       10,     2013, 0x3edd9aa4
 1,       4931,       4931,       10,     2013, 0xbd66ac26
-0,        148,        148,        1,   518400, 0x6294e449
+0,        148,        148,        1,   518400, 0xb0c3ddca
 1,       4941,       4941,       10,     2013, 0x313896ea
 1,       4951,       4951,       10,     2013, 0x6b83a6a0
 1,       4961,       4961,       10,     2013, 0x9aafb109
-0,        149,        149,        1,   518400, 0xa05721e7
+0,        149,        149,        1,   518400, 0x10351b53
 1,       4976,       4976,       10,     2013, 0x5192a85a
 1,       4986,       4986,       10,     2013, 0x1f919f79
 1,       4996,       4996,       10,     2013, 0xc0799c40
-0,        150,        150,        1,   518400, 0x37749183
+0,        150,        150,        1,   518400, 0xc1408aee
 1,       5006,       5006,       10,     2013, 0x2988bcd8
 1,       5016,       5016,       10,     2013, 0x1482913a
 1,       5026,       5026,       10,     2013, 0x74da9a94
 1,       5036,       5036,       10,     2013, 0x763eb709
-0,        151,        151,        1,   518400, 0xf9d9dca0
+0,        151,        151,        1,   518400, 0xf016d615
 1,       5046,       5046,       10,     2013, 0x1285b405
 1,       5062,       5062,       10,     2013, 0xb6ab9dfc
-0,        152,        152,        1,   518400, 0x5f8ccf08
+0,        152,        152,        1,   518400, 0xa768c892
 1,       5072,       5072,       10,     2013, 0xe4c8bf19
 1,       5082,       5082,       10,     2013, 0xabbbade8
 1,       5092,       5092,       10,     2013, 0xf8b69d89
 1,       5102,       5102,       10,     2013, 0xce04a866
-0,        153,        153,        1,   518400, 0x7303f77b
+0,        153,        153,        1,   518400, 0x11c3f11e
 1,       5112,       5112,       10,     2013, 0x07528abf
 1,       5122,       5122,       10,     2013, 0x74fb98bf
 1,       5132,       5132,       10,     2013, 0x579fb1c9
-0,        154,        154,        1,   518400, 0x22b0513f
+0,        154,        154,        1,   518400, 0xcd9a4ac4
 1,       5147,       5147,       10,     2013, 0x7ddea2ed
 1,       5157,       5157,       10,     2013, 0x296caa2c
 1,       5167,       5167,       10,     2013, 0x346d9c4f
-0,        155,        155,        1,   518400, 0x330485d2
+0,        155,        155,        1,   518400, 0x4ade7f5e
 1,       5177,       5177,       10,     2013, 0x3e1fba15
 1,       5187,       5187,       10,     2013, 0x48a2908f
 1,       5197,       5197,       10,     2013, 0xc1938d09
-0,        156,        156,        1,   518400, 0x7f83daea
+0,        156,        156,        1,   518400, 0x655dd46b
 1,       5207,       5207,       10,     2013, 0x0e96a060
 1,       5217,       5217,       10,     2013, 0x7b6a9e06
 1,       5232,       5232,       10,     2013, 0x5b779d28
-0,        157,        157,        1,   518400, 0xee19f2df
+0,        157,        157,        1,   518400, 0x5ab5ec61
 1,       5242,       5242,       10,     2013, 0xf600aca1
 1,       5252,       5252,       10,     2013, 0x3a6c9e68
 1,       5262,       5262,       10,     2013, 0x0c8dc1b0
-0,        158,        158,        1,   518400, 0xb71b1c77
+0,        158,        158,        1,   518400, 0x45dc15e6
 1,       5272,       5272,       10,     2013, 0x26beb245
 1,       5282,       5282,       10,     2013, 0x2bc09557
 1,       5292,       5292,       10,     2013, 0x27fc8845
 1,       5302,       5302,       10,     2013, 0x1025aa47
-0,        159,        159,        1,   518400, 0xbffc1856
+0,        159,        159,        1,   518400, 0x201911d3
 1,       5318,       5318,       10,     2013, 0xc2e69baa
 1,       5328,       5328,       10,     2013, 0xdb249b92
 1,       5338,       5338,       10,     2013, 0x6ccda29e
-0,        160,        160,        1,   518400, 0xabc125aa
+0,        160,        160,        1,   518400, 0x0fbc1f46
 1,       5348,       5348,       10,     2013, 0xeaf6a1cf
 1,       5358,       5358,       10,     2013, 0x509ba397
 1,       5368,       5368,       10,     2013, 0xfaf8a2df
-0,        161,        161,        1,   518400, 0x5ee467f8
+0,        161,        161,        1,   518400, 0x7e316179
 1,       5378,       5378,       10,     2013, 0x41388f28
 1,       5388,       5388,       10,     2013, 0xfe5eab39
 1,       5403,       5403,       10,     2013, 0xd5ffa066
-0,        162,        162,        1,   518400, 0x6c2cf168
+0,        162,        162,        1,   518400, 0x73bbeaed
 1,       5413,       5413,       10,     2013, 0x6813a30a
 1,       5423,       5423,       10,     2013, 0x9be89718
 1,       5433,       5433,       10,     2013, 0xaec3a27b
-0,        163,        163,        1,   518400, 0x63996b26
+0,        163,        163,        1,   518400, 0x3a7c648a
 1,       5446,       5446,       10,     2013, 0x579a983e
 1,       5456,       5456,       10,     2013, 0x98cea21f
 1,       5466,       5466,       10,     2013, 0xca77a58a
-0,        164,        164,        1,   518400, 0xb34d789a
+0,        164,        164,        1,   518400, 0x9f707209
 1,       5476,       5476,       10,     2013, 0xcbc3b1ee
 1,       5486,       5486,       10,     2013, 0xf3bb8f07
 1,       5496,       5496,       10,     2013, 0x6aeebd92
-0,        165,        165,        1,   518400, 0xf49c030f
+0,        165,        165,        1,   518400, 0x9f25fc5c
 1,       5506,       5506,       10,     2013, 0xe955a449
 1,       5516,       5516,       10,     2013, 0x9436aa5b
 1,       5531,       5531,       10,     2013, 0x4f0a8f9f
-0,        166,        166,        1,   518400, 0x092dc41a
+0,        166,        166,        1,   518400, 0x2ed8bd75
 1,       5541,       5541,       10,     2013, 0x3551b22d
 1,       5551,       5551,       10,     2013, 0x0959a3d4
 1,       5561,       5561,       10,     2013, 0x2ed5a11b
 1,       5571,       5571,       10,     2013, 0x8f52a5c3
-0,        167,        167,        1,   518400, 0x4134c577
+0,        167,        167,        1,   518400, 0xb493becb
 1,       5581,       5581,       10,     2013, 0x6552978d
 1,       5591,       5591,       10,     2013, 0x7dcca0c1
 1,       5601,       5601,       10,     2013, 0xbcd4a3c9
-0,        168,        168,        1,   518400, 0x261de1ed
+0,        168,        168,        1,   518400, 0x7df6db57
 1,       5616,       5616,       10,     2013, 0xfe41a8d8
 1,       5626,       5626,       10,     2013, 0xc85aae14
 1,       5636,       5636,       10,     2013, 0x1185b346
-0,        169,        169,        1,   518400, 0xcbc8566a
+0,        169,        169,        1,   518400, 0x1cb94fca
 1,       5646,       5646,       10,     2013, 0xf7429a0d
 1,       5656,       5656,       10,     2013, 0x48c2a160
 1,       5666,       5666,       10,     2013, 0x9d85a85d
-0,        170,        170,        1,   518400, 0x407a5c76
+0,        170,        170,        1,   518400, 0x70db55d8
 1,       5676,       5676,       10,     2013, 0xbbe89fe9
 1,       5686,       5686,       10,     2013, 0xea429fe2
 1,       5702,       5702,       10,     2013, 0x221ca1d4
-0,        171,        171,        1,   518400, 0x1ed73bb2
+0,        171,        171,        1,   518400, 0xc1d9351b
 1,       5712,       5712,       10,     2013, 0x394b925b
 1,       5722,       5722,       10,     2013, 0x556dc26f
 1,       5732,       5732,       10,     2013, 0xce21a5e1
-0,        172,        172,        1,   518400, 0x8467ddb5
+0,        172,        172,        1,   518400, 0xa4b0d717
 1,       5742,       5742,       10,     2013, 0xbc87c0a8
 1,       5752,       5752,       10,     2013, 0xbac4ac07
 1,       5762,       5762,       10,     2013, 0xdeefa4aa
 1,       5772,       5772,       10,     2013, 0x1f15b362
-0,        173,        173,        1,   518400, 0x0523dc73
+0,        173,        173,        1,   518400, 0x3730d5e9
 1,       5787,       5787,       10,     2013, 0x6406b7b2
 1,       5797,       5797,       10,     2013, 0x8030a03d
-0,        174,        174,        1,   518400, 0x81f5e895
+0,        174,        174,        1,   518400, 0x9673e1ec
 1,       5807,       5807,       10,     2013, 0x0373a5b1
 1,       5817,       5817,       10,     2013, 0x34ef93da
 1,       5827,       5827,       10,     2013, 0x94c198fe
 1,       5837,       5837,       10,     2013, 0xfefcabad
-0,        175,        175,        1,   518400, 0xfc74608d
+0,        175,        175,        1,   518400, 0x877959d5
 1,       5847,       5847,       10,     2013, 0x8755b3ec
 1,       5857,       5857,       10,     2013, 0xe436a6fd
 1,       5872,       5872,       10,     2013, 0x9cf5a11e
-0,        176,        176,        1,   518400, 0xc4e0dae0
+0,        176,        176,        1,   518400, 0x04f3d421
 1,       5882,       5882,       10,     2013, 0x03b8a98c
 1,       5892,       5892,       10,     2013, 0x6216a138
 1,       5902,       5902,       10,     2013, 0xd87b9f12
-0,        177,        177,        1,   518400, 0x98367f5b
+0,        177,        177,        1,   518400, 0x4f3078bc
 1,       5912,       5912,       10,     2013, 0x4ce99653
 1,       5922,       5922,       10,     2013, 0x6c2ea9e2
 1,       5932,       5932,       10,     2013, 0x918cae4c
-0,        178,        178,        1,   518400, 0x0f1a869d
+0,        178,        178,        1,   518400, 0x8a127ff8
 1,       5942,       5942,       10,     2013, 0xd19fa5f2
 1,       5958,       5958,       10,     2013, 0x0bdda7c6
 1,       5968,       5968,       10,     2013, 0x0f9ab0ca
-0,        179,        179,        1,   518400, 0x45b6ccf2
+0,        179,        179,        1,   518400, 0x5864c64f
 1,       5978,       5978,       10,     2013, 0x410a92b1
 1,       5988,       5988,       10,     2013, 0xcfbe9d1c
 1,       5998,       5998,       10,     2013, 0x59ed9d15
-0,        180,        180,        1,   518400, 0x5f9ccb77
+0,        180,        180,        1,   518400, 0xdaccc4c0
 1,       6008,       6008,       10,     2013, 0x4e129e27
 1,       6018,       6018,       10,     2013, 0x7bb9ac0a
 1,       6028,       6028,       10,     2013, 0x826ca82b
-0,        181,        181,        1,   518400, 0x5f15ea31
+0,        181,        181,        1,   518400, 0xd999e376
 1,       6043,       6043,       10,     2013, 0x9ad5a74b
 1,       6053,       6053,       10,     2013, 0x6c5f969a
 1,       6063,       6063,       10,     2013, 0x8479a0e5
-0,        182,        182,        1,   518400, 0x86369f27
+0,        182,        182,        1,   518400, 0x8af39876
 1,       6073,       6073,       10,     2013, 0x165298ef
 1,       6083,       6083,       10,     2013, 0xdcadb4a1
 1,       6093,       6093,       10,     2013, 0xa90e987c
 1,       6103,       6103,       10,     2013, 0x1ac5b510
-0,        183,        183,        1,   518400, 0x2e27f9fa
+0,        183,        183,        1,   518400, 0x5e72f33d
 1,       6113,       6113,       10,     2013, 0x66728d85
 1,       6128,       6128,       10,     2013, 0xe4859fc5
 1,       6138,       6138,       10,     2013, 0x9901786e
-0,        184,        184,        1,   518400, 0xc029a44d
+0,        184,        184,        1,   518400, 0x14af9d92
 1,       6148,       6148,       10,     2013, 0x6aebb406
 1,       6158,       6158,       10,     2013, 0x7d13a2cc
 1,       6168,       6168,       10,     2013, 0x99b7a8cc
-0,        185,        185,        1,   518400, 0xebee33b0
+0,        185,        185,        1,   518400, 0x50b82d10
 1,       6178,       6178,       10,     2013, 0x80b8a624
 1,       6188,       6188,       10,     2013, 0xbb6aa271
 1,       6198,       6198,       10,     2013, 0x17af9e4a
-0,        186,        186,        1,   518400, 0x19e5494f
+0,        186,        186,        1,   518400, 0xc068429c
 1,       6214,       6214,       10,     2013, 0xfaf0a8f1
 1,       6224,       6224,       10,     2013, 0xd6849b93
 1,       6234,       6234,       10,     2013, 0xe9829669
-0,        187,        187,        1,   518400, 0xf697bd7c
+0,        187,        187,        1,   518400, 0x8934b6d1
 1,       6244,       6244,       10,     2013, 0x7ec98944
 1,       6254,       6254,       10,     2013, 0x2b2099a4
 1,       6264,       6264,       10,     2013, 0x1033a82f
-0,        188,        188,        1,   518400, 0x82569002
+0,        188,        188,        1,   518400, 0x11d08947
 1,       6274,       6274,       10,     2013, 0x5ec88990
 1,       6284,       6284,       10,     2013, 0xd2a19b3d
 1,       6299,       6299,       10,     2013, 0xa377b268
-0,        189,        189,        1,   518400, 0xfcb6d707
+0,        189,        189,        1,   518400, 0x8a27d041
 1,       6309,       6309,       10,     2013, 0xfa859901
 1,       6319,       6319,       10,     2013, 0x1713955a
 1,       6329,       6329,       10,     2013, 0x70aab0da
 1,       6339,       6339,       10,     2013, 0xcdaea422
-0,        190,        190,        1,   518400, 0x82a9662b
+0,        190,        190,        1,   518400, 0xab265f7d
 1,       6349,       6349,       10,     2013, 0x65c3bf80
 1,       6359,       6359,       10,     2013, 0x1d75a55f
 1,       6369,       6369,       10,     2013, 0xa5bea4de
-0,        191,        191,        1,   518400, 0x212e16ee
+0,        191,        191,        1,   518400, 0xff491040
 1,       6384,       6384,       10,     2013, 0x184db71c
 1,       6394,       6394,       10,     2013, 0x99858ec8
 1,       6404,       6404,       10,     2013, 0xb8f2aee5
-0,        192,        192,        1,   518400, 0x2ca34dca
+0,        192,        192,        1,   518400, 0x822b4704
 1,       6414,       6414,       10,     2013, 0x4435b2ef
 1,       6424,       6424,       10,     2013, 0x8acfa6c7
 1,       6434,       6434,       10,     2013, 0x42b4c01f
-0,        193,        193,        1,   518400, 0xe9ebe0a5
+0,        193,        193,        1,   518400, 0x4523d9f4
 1,       6444,       6444,       10,     2013, 0x6e308c13
 1,       6454,       6454,       10,     2013, 0x8227a0f6
 1,       6470,       6470,       10,     2013, 0x6f12a7a2
-0,        194,        194,        1,   518400, 0x4e6b6917
+0,        194,        194,        1,   518400, 0xfc3c626e
 1,       6480,       6480,       10,     2013, 0x785392be
 1,       6490,       6490,       10,     2013, 0x81849c2b
 1,       6500,       6500,       10,     2013, 0x5cf2af65
-0,        195,        195,        1,   518400, 0x7dcf20ab
+0,        195,        195,        1,   518400, 0x237319e5
 1,       6510,       6510,       10,     2013, 0x0c6ca6b4
 1,       6520,       6520,       10,     2013, 0x412fab9f
 1,       6530,       6530,       10,     2013, 0x08e792b4
-0,        196,        196,        1,   518400, 0xf30fac97
+0,        196,        196,        1,   518400, 0x892ca5d8
 1,       6540,       6540,       10,     2013, 0x407aace3
 1,       6555,       6555,       10,     2013, 0xd26bac16
 1,       6565,       6565,       10,     2013, 0xac8bb295
-0,        197,        197,        1,   518400, 0xcb9fc692
+0,        197,        197,        1,   518400, 0xc4c0bfc7
 1,       6575,       6575,       10,     2013, 0xddd1949c
 1,       6585,       6585,       10,     2013, 0x6b26b868
 1,       6595,       6595,       10,     2013, 0x5eaba587
 1,       6605,       6605,       10,     2013, 0xef0793b9
-0,        198,        198,        1,   518400, 0x5d05601e
+0,        198,        198,        1,   518400, 0x57c85956
 1,       6615,       6615,       10,     2013, 0xdef19bd6
 1,       6625,       6625,       10,     2013, 0xca98a635
-0,        199,        199,        1,   518400, 0x456c1417
+0,        199,        199,        1,   518400, 0xd6300d46
 1,       6640,       6640,       10,     2013, 0x06269a5a
 1,       6650,       6650,       10,     2013, 0x32cb9952
 1,       6660,       6660,       10,     2013, 0xf01fa95a
 1,       6670,       6670,       10,     2013, 0xefab9e55
-0,        200,        200,        1,   518400, 0x9a0fd1ad
+0,        200,        200,        1,   518400, 0xd3dacaec
 1,       6680,       6680,       10,     2013, 0x55a3b63a
 1,       6690,       6690,       10,     2013, 0xcd36a553
 1,       6700,       6700,       10,     2013, 0x2ec19877
-0,        201,        201,        1,   518400, 0x55db9716
+0,        201,        201,        1,   518400, 0x65429052
 1,       6710,       6710,       10,     2013, 0xc18b924c
 1,       6726,       6726,       10,     2013, 0xf132b04c
 1,       6736,       6736,       10,     2013, 0x7975a44d
-0,        202,        202,        1,   518400, 0x1f0d40d6
+0,        202,        202,        1,   518400, 0xec803a15
 1,       6746,       6746,       10,     2013, 0x2aaf94cb
 1,       6756,       6756,       10,     2013, 0x58cfa60f
 1,       6766,       6766,       10,     2013, 0x9757a658
-0,        203,        203,        1,   518400, 0x73695c82
+0,        203,        203,        1,   518400, 0x7a9a55c9
 1,       6776,       6776,       10,     2013, 0x67ebc0d5
 1,       6786,       6786,       10,     2013, 0x3c50a70e
 1,       6796,       6796,       10,     2013, 0x9c5799c6
-0,        204,        204,        1,   518400, 0xb0f10812
+0,        204,        204,        1,   518400, 0xcac30160
 1,       6811,       6811,       10,     2013, 0x018d85b2
 1,       6821,       6821,       10,     2013, 0x5367a956
-0,        205,        205,        1,   518400, 0xdec18505
-0,        208,        208,        1,   518400, 0xb147b947
-0,        240,        240,        1,   518400, 0x9d2e3977
+0,        205,        205,        1,   518400, 0x7e187e4f
+0,        208,        208,        1,   518400, 0x0be0b2a2
diff --git a/tests/ref/fate/sub-dvb b/tests/ref/fate/sub-dvb
index cbd1801d64..8f33c75d70 100644
--- a/tests/ref/fate/sub-dvb
+++ b/tests/ref/fate/sub-dvb
@@ -1,75 +1,93 @@ 
 #tb 0: 1/1000000
 #media_type 0: subtitle
 #codec_id 0: dvb_subtitle
-0,   15600000,   15600000,   159000,     1168, 0xd0f89d82
-0,   15759000,   15759000,   159000,       14, 0x064900eb
-0,   15760000,   15760000,   239000,     1544, 0xe60f1751
-0,   15999000,   15999000,   239000,       14, 0x0729010b
-0,   16000000,   16000000,   339000,     1658, 0xbe343093
-0,   16339000,   16339000,   339000,       14, 0x0809012b
-0,   16340000,   16340000,   599000,     2343, 0xc68f07ef
-0,   16939000,   16939000,   599000,       14, 0x08e9014b
-0,   16940000,   16940000,   459000,     2568, 0x0ee657b1
-0,   17399000,   17399000,   459000,       14, 0x09c9016b
-0,   17400000,   17400000,   359000,     3422, 0xba5b63ce
-0,   17759000,   17759000,   359000,       14, 0x0aa9018b
-0,   17760000,   17760000,   219000,     5078, 0x95b19902
-0,   17979000,   17979000,   219000,       14, 0x0b8901ab
-0,   17980000,   17980000,   959000,     5808, 0xc9717b89
-0,   18939000,   18939000,   959000,       14, 0x0c6901cb
-0,   18940000,   18940000,   219000,     6015, 0x0becbfac
-0,   19159000,   19159000,   219000,       14, 0x064900eb
-0,   19160000,   19160000,   259000,     6519, 0xfcd24d26
-0,   19419000,   19419000,   259000,       14, 0x0729010b
-0,   19420000,   19420000,    99000,     7061, 0xf0320408
-0,   19519000,   19519000,    99000,       14, 0x0809012b
-0,   19520000,   19520000,   219000,     4773, 0x66c93074
-0,   19739000,   19739000,   219000,       14, 0x08e9014b
-0,   19740000,   19740000,   219000,     5546, 0x06052c81
-0,   19959000,   19959000,   219000,       14, 0x09c9016b
-0,   19960000,   19960000,   239000,     5754, 0x904f7325
-0,   20199000,   20199000,   239000,       14, 0x0aa9018b
-0,   20200000,   20200000,   139000,     6099, 0xe30cde07
-0,   20339000,   20339000,   139000,       14, 0x0b8901ab
-0,   20340000,   20340000,   799000,     6839, 0x770fcb6c
-0,   21139000,   21139000,   799000,       14, 0x0c6901cb
-0,   21140000,   21140000,   239000,     4744, 0xa91e1b41
-0,   21379000,   21379000,   239000,       14, 0x064900eb
-0,   21380000,   21380000,   339000,     5824, 0xcf6d782b
-0,   21719000,   21719000,   339000,       14, 0x0729010b
-0,   21720000,   21720000,  1439000,     6212, 0xabf8f7cf
-0,   23159000,   23159000,  1439000,       14, 0x0809012b
-0,   23160000,   23160000,  1319000,     7082, 0xd7ca10f2
-0,   24479000,   24479000,  1319000,       14, 0x08e9014b
-0,   24480000,   24480000,   219000,     5345, 0x12b2cae0
-0,   24699000,   24699000,   219000,       14, 0x09c9016b
-0,   24700000,   24700000,   219000,     5765, 0xc7d46192
-0,   24919000,   24919000,   219000,       14, 0x0aa9018b
-0,   24920000,   24920000,   599000,     6557, 0xcb995d30
-0,   25519000,   25519000,   599000,       14, 0x0b8901ab
-0,   25520000,   25520000,   219000,     7091, 0xe6ea0559
-0,   25739000,   25739000,   219000,       14, 0x0c6901cb
-0,   25740000,   25740000,   239000,     7305, 0xb66c404e
-0,   25979000,   25979000,   239000,       14, 0x064900eb
-0,   25980000,   25980000,   359000,     7590, 0x0cc2a481
-0,   26339000,   26339000,   359000,       14, 0x0729010b
-0,   26340000,   26340000,   219000,     4629, 0xe18cfea8
-0,   26559000,   26559000,   219000,       14, 0x0809012b
-0,   26560000,   26560000,   719000,     4785, 0x82043fc0
-0,   27279000,   27279000,   719000,       14, 0x08e9014b
-0,   27280000,   27280000,   459000,     6061, 0xbde7d245
-0,   27739000,   27739000,   459000,       14, 0x09c9016b
-0,   27740000,   27740000,   239000,     6301, 0x92d01a51
-0,   27979000,   27979000,   239000,       14, 0x0aa9018b
-0,   27980000,   27980000,    99000,     6736, 0xbd25a134
-0,   28079000,   28079000,    99000,       14, 0x0b8901ab
-0,   28080000,   28080000,   219000,     7214, 0x7ef93c13
-0,   28299000,   28299000,   219000,       14, 0x0c6901cb
-0,   28300000,   28300000,   239000,     7366, 0x5bed7fcd
-0,   28539000,   28539000,   239000,       14, 0x064900eb
-0,   28540000,   28540000,   599000,     4564, 0x7f4c014b
-0,   29139000,   29139000,   599000,       14, 0x0729010b
-0,   29140000,   29140000,   219000,     4637, 0x682626b7
-0,   29359000,   29359000,   219000,       14, 0x0809012b
-0,   29360000,   29360000,  1679000,     5358, 0x29e30c48
-0,   31039000,   31039000,  1679000,       14, 0x08e9014b
+0,          0,          0,   279000,       14, 0x05d900db
+0,     279000,     279000,   279000,       14, 0x064900eb
+0,     280000,     280000,  4999000,       14, 0x06b900fb
+0,    5279000,    5279000,  4999000,       14, 0x0729010b
+0,    5280000,    5280000,  5019000,       14, 0x0799011b
+0,   10299000,   10299000,  5019000,       14, 0x0809012b
+0,   10300000,   10300000,  3599000,       14, 0x0879013b
+0,   13899000,   13899000,  3599000,       14, 0x08e9014b
+0,   13900000,   13900000,   219000,       14, 0x0959015b
+0,   14119000,   14119000,   219000,       14, 0x09c9016b
+0,   14120000,   14120000,  1439000,       14, 0x0a39017b
+0,   15559000,   15559000,  1439000,       14, 0x0aa9018b
+0,   15560000,   15560000,    39000,       14, 0x0b19019b
+0,   15599000,   15599000,    39000,       14, 0x0b8901ab
+0,   15600000,   15600000,   159000,     1168, 0xd69da022
+0,   15759000,   15759000,   159000,       14, 0x0c6901cb
+0,   15760000,   15760000,   239000,     1544, 0xc5f116f1
+0,   15999000,   15999000,   239000,       14, 0x064900eb
+0,   16000000,   16000000,   339000,     1658, 0x73563033
+0,   16339000,   16339000,   339000,       14, 0x0729010b
+0,   16340000,   16340000,   599000,     2343, 0x7ac2078f
+0,   16939000,   16939000,   599000,       14, 0x0809012b
+0,   16940000,   16940000,   459000,     2568, 0x6eaa5751
+0,   17399000,   17399000,   459000,       14, 0x08e9014b
+0,   17400000,   17400000,   359000,     3422, 0xd9d0636e
+0,   17759000,   17759000,   359000,       14, 0x09c9016b
+0,   17760000,   17760000,   219000,     5078, 0x722c9862
+0,   17979000,   17979000,   219000,       14, 0x0aa9018b
+0,   17980000,   17980000,   959000,     5808, 0x38dd7ae9
+0,   18939000,   18939000,   959000,       14, 0x0b8901ab
+0,   18940000,   18940000,   219000,     6015, 0xd4d2c40c
+0,   19159000,   19159000,   219000,       14, 0x0c6901cb
+0,   19160000,   19160000,   259000,     6519, 0x08af4c86
+0,   19419000,   19419000,   259000,       14, 0x064900eb
+0,   19420000,   19420000,    99000,     7061, 0xecf10368
+0,   19519000,   19519000,    99000,       14, 0x0729010b
+0,   19520000,   19520000,   219000,     4773, 0xbee42fd4
+0,   19739000,   19739000,   219000,       14, 0x0809012b
+0,   19740000,   19740000,   219000,     5546, 0xdb822be1
+0,   19959000,   19959000,   219000,       14, 0x08e9014b
+0,   19960000,   19960000,   239000,     5754, 0xfdcc7285
+0,   20199000,   20199000,   239000,       14, 0x09c9016b
+0,   20200000,   20200000,   139000,     6099, 0xa409dd67
+0,   20339000,   20339000,   139000,       14, 0x0aa9018b
+0,   20340000,   20340000,   799000,     6839, 0xc5eecacc
+0,   21139000,   21139000,   799000,       14, 0x0b8901ab
+0,   21140000,   21140000,   239000,     4744, 0x4e451fa1
+0,   21379000,   21379000,   239000,       14, 0x0c6901cb
+0,   21380000,   21380000,   339000,     5824, 0x5299778b
+0,   21719000,   21719000,   339000,       14, 0x064900eb
+0,   21720000,   21720000,  1439000,     6212, 0x6d15f72f
+0,   23159000,   23159000,  1439000,       14, 0x0729010b
+0,   23160000,   23160000,  1319000,     7082, 0xe5c91052
+0,   24479000,   24479000,  1319000,       14, 0x0809012b
+0,   24480000,   24480000,   219000,     5345, 0x2e5eca40
+0,   24699000,   24699000,   219000,       14, 0x08e9014b
+0,   24700000,   24700000,   219000,     5765, 0x118060f2
+0,   24919000,   24919000,   219000,       14, 0x09c9016b
+0,   24920000,   24920000,   599000,     6557, 0x89275c90
+0,   25519000,   25519000,   599000,       14, 0x0aa9018b
+0,   25520000,   25520000,   219000,     7091, 0x996904b9
+0,   25739000,   25739000,   219000,       14, 0x0b8901ab
+0,   25740000,   25740000,   239000,     7305, 0xc23e44ae
+0,   25979000,   25979000,   239000,       14, 0x0c6901cb
+0,   25980000,   25980000,   359000,     7590, 0xc5a3a3e1
+0,   26339000,   26339000,   359000,       14, 0x064900eb
+0,   26340000,   26340000,   219000,     4629, 0x7ad6fe08
+0,   26559000,   26559000,   219000,       14, 0x0729010b
+0,   26560000,   26560000,   719000,     4785, 0xcd3f3f20
+0,   27279000,   27279000,   719000,       14, 0x0809012b
+0,   27280000,   27280000,   459000,     6061, 0x8b04d1a5
+0,   27739000,   27739000,   459000,       14, 0x08e9014b
+0,   27740000,   27740000,   239000,     6301, 0xe7de19b1
+0,   27979000,   27979000,   239000,       14, 0x09c9016b
+0,   27980000,   27980000,    99000,     6736, 0x38b3a094
+0,   28079000,   28079000,    99000,       14, 0x0aa9018b
+0,   28080000,   28080000,   219000,     7214, 0x0b783b73
+0,   28299000,   28299000,   219000,       14, 0x0b8901ab
+0,   28300000,   28300000,   239000,     7366, 0x98bf842d
+0,   28539000,   28539000,   239000,       14, 0x0c6901cb
+0,   28540000,   28540000,   599000,     4564, 0x3d9600ab
+0,   29139000,   29139000,   599000,       14, 0x064900eb
+0,   29140000,   29140000,   219000,     4637, 0x01f02617
+0,   29359000,   29359000,   219000,       14, 0x0729010b
+0,   29360000,   29360000,  1679000,     5358, 0x5b0f0ba8
+0,   31039000,   31039000,  1679000,       14, 0x0809012b
+0,   31040000,   31040000,   359000,       14, 0x0879013b
+0,   31399000,   31399000,   359000,       14, 0x08e9014b
+0,   31400000,   31400000,   479000,       14, 0x0959015b
+0,   31879000,   31879000,   479000,       14, 0x09c9016b
diff --git a/tests/ref/fate/sub2video b/tests/ref/fate/sub2video
index 80abe9c905..01b4800967 100644
--- a/tests/ref/fate/sub2video
+++ b/tests/ref/fate/sub2video
@@ -36,151 +36,109 @@ 
 0,         25,         25,        1,   518400, 0x7322e11c
 0,         26,         26,        1,   518400, 0x45af1a84
 0,         27,         27,        1,   518400, 0x7b781071
-0,         28,         28,        1,   518400, 0x4f7c706c
-0,         29,         29,        1,   518400, 0xb227603b
-0,         30,         30,        1,   518400, 0x7b4b89c2
-0,         31,         31,        1,   518400, 0x456da21e
-0,         32,         32,        1,   518400, 0xb691979f
-0,         33,         33,        1,   518400, 0x0dfaa66d
-0,         34,         34,        1,   518400, 0x191a6f23
-0,         35,         35,        1,   518400, 0xa03b2605
-0,         36,         36,        1,   518400, 0xb36aff87
-0,         37,         37,        1,   518400, 0xf5f0bc4a
-0,         38,         38,        1,   518400, 0x863d701a
-0,         39,         39,        1,   518400, 0xd11b4dce
-0,         40,         40,        1,   518400, 0x969236bd
-0,         41,         41,        1,   518400, 0xb60a485c
-0,         42,         42,        1,   518400, 0xe9796621
-0,         43,         43,        1,   518400, 0x3e8fc04b
-0,         44,         44,        1,   518400, 0xac9944e3
-0,         45,         45,        1,   518400, 0x01452b4d
-0,         46,         46,        1,   518400, 0xb384f6d2
-0,         47,         47,        1,   518400, 0xde69683f
-0,         48,         48,        1,   518400, 0x7df08fba
-0,         49,         49,        1,   518400, 0xbab197ea
+0,         28,         28,        1,   518400, 0x08a0ed62
+0,         29,         29,        1,   518400, 0x39b1e7eb
+0,         30,         30,        1,   518400, 0x20a11073
+0,         31,         31,        1,   518400, 0x7e19198d
+0,         32,         32,        1,   518400, 0xeed206eb
+0,         33,         33,        1,   518400, 0xcf451b3d
+0,         34,         34,        1,   518400, 0xa0a2eb3c
+0,         35,         35,        1,   518400, 0xd598a9e3
+0,         36,         36,        1,   518400, 0x10908608
+0,         37,         37,        1,   518400, 0xa3a03b86
+0,         38,         38,        1,   518400, 0x0190e39a
+0,         39,         39,        1,   518400, 0x5d39b978
+0,         40,         40,        1,   518400, 0x5ba29f40
+0,         41,         41,        1,   518400, 0x86c1b3ed
+0,         42,         42,        1,   518400, 0x77b1d91b
+0,         43,         43,        1,   518400, 0xa28d38d6
+0,         44,         44,        1,   518400, 0xc86abb34
+0,         45,         45,        1,   518400, 0xe753a664
+0,         46,         46,        1,   518400, 0xc7ea803c
+0,         47,         47,        1,   518400, 0x797eff2d
+0,         48,         48,        1,   518400, 0xc69c300c
+0,         49,         49,        1,   518400, 0x8d21303f
 1,   15355000,   15355000,  4733000,     2094, 0x3c171425
 0,         77,         77,        1,   518400, 0x902285d9
-0,        100,        100,        1,   518400, 0xbab197ea
 1,   48797000,   48797000,  2560000,     2480, 0x7c0edf21
 0,        244,        244,        1,   518400, 0x7a11c812
-0,        257,        257,        1,   518400, 0xbab197ea
+0,        257,        257,        1,   518400, 0x34cdddee
 1,   51433000,   51433000,  2366000,     3059, 0xc95b8a05
-0,        258,        258,        1,   518400, 0x34cdddee
-0,        269,        269,        1,   518400, 0xbab197ea
 1,   53910000,   53910000,  2696000,     2095, 0x61bb15ed
 0,        270,        270,        1,   518400, 0x4db4ce51
-0,        283,        283,        1,   518400, 0xbab197ea
+0,        283,        283,        1,   518400, 0xe6bc0ea9
 1,   56663000,   56663000,  1262000,     1013, 0xc9ae89b7
-0,        284,        284,        1,   518400, 0xe6bc0ea9
-0,        290,        290,        1,   518400, 0xbab197ea
+0,        290,        290,        1,   518400, 0xa8643af7
 1,   58014000,   58014000,  1661000,      969, 0xe01878f0
-0,        291,        291,        1,   518400, 0xa8643af7
-0,        298,        298,        1,   518400, 0xbab197ea
 1,   67724000,   67724000,  1365000,      844, 0xe7db4fc1
 0,        339,        339,        1,   518400, 0xb1885c67
-0,        345,        345,        1,   518400, 0xbab197ea
 1,   69175000,   69175000,  1558000,      802, 0xf48531ba
 0,        346,        346,        1,   518400, 0x378e3fd0
-0,        354,        354,        1,   518400, 0xbab197ea
+0,        354,        354,        1,   518400, 0xa3782469
 1,   70819000,   70819000,  1865000,     1709, 0xb4d5a1bd
-0,        355,        355,        1,   518400, 0xa3782469
-0,        363,        363,        1,   518400, 0xbab197ea
 1,   72762000,   72762000,  1968000,     2438, 0x99d7bc82
 0,        364,        364,        1,   518400, 0xba23a0d5
-0,        374,        374,        1,   518400, 0xbab197ea
+0,        374,        374,        1,   518400, 0x129de2f8
 1,   74806000,   74806000,  1831000,     2116, 0x96514097
-0,        375,        375,        1,   518400, 0x129de2f8
-0,        383,        383,        1,   518400, 0xbab197ea
 1,   76716000,   76716000,  1262000,     1822, 0xefccc72e
 0,        384,        384,        1,   518400, 0x19772f0f
-0,        390,        390,        1,   518400, 0xbab197ea
+0,        390,        390,        1,   518400, 0x56f54e73
 1,   78051000,   78051000,  1524000,      987, 0x7b927a27
-0,        391,        391,        1,   518400, 0x56f54e73
-0,        398,        398,        1,   518400, 0xbab197ea
+0,        398,        398,        1,   518400, 0x300b5247
 1,   79644000,   79644000,  2662000,     2956, 0x190778f7
-0,        399,        399,        1,   518400, 0x300b5247
 1,   82380000,   82380000,  2764000,     3094, 0xc021b7d3
-0,        412,        412,        1,   518400, 0xbab197ea
-0,        413,        413,        1,   518400, 0x6fd028fa
-0,        426,        426,        1,   518400, 0xbab197ea
+0,        412,        412,        1,   518400, 0x6fd028fa
+0,        426,        426,        1,   518400, 0x01f80e9d
 1,   85225000,   85225000,  2366000,     2585, 0x74d0048f
-0,        427,        427,        1,   518400, 0x01f80e9d
-0,        438,        438,        1,   518400, 0xbab197ea
+0,        438,        438,        1,   518400, 0xb48d90c0
 1,   87652000,   87652000,  1831000,      634, 0x8832fda1
-0,        439,        439,        1,   518400, 0xb48d90c0
-0,        447,        447,        1,   518400, 0xbab197ea
 1,   91531000,   91531000,  2332000,     2080, 0x97a1146f
 0,        458,        458,        1,   518400, 0xcb5a0173
-0,        469,        469,        1,   518400, 0xbab197ea
 1,   95510000,   95510000,  3299000,     2964, 0x8b8f6684
 0,        478,        478,        1,   518400, 0xb8a323e4
-0,        494,        494,        1,   518400, 0xbab197ea
+0,        494,        494,        1,   518400, 0xc43518ba
 1,   98872000,   98872000,  2161000,     1875, 0x9002ef71
-0,        495,        495,        1,   518400, 0xc43518ba
-0,        505,        505,        1,   518400, 0xbab197ea
 1,  101124000,  101124000,  4096000,     3872, 0x20c6ed9c
 0,        506,        506,        1,   518400, 0x04e38692
-0,        526,        526,        1,   518400, 0xbab197ea
 1,  105303000,  105303000,  2730000,     3094, 0xf203a663
 0,        527,        527,        1,   518400, 0x856b0ee5
-0,        540,        540,        1,   518400, 0xbab197ea
 1,  108106000,  108106000,  2059000,     2404, 0x41a7b429
 0,        541,        541,        1,   518400, 0x3e5beee2
-0,        551,        551,        1,   518400, 0xbab197ea
 1,  141556000,  141556000,  1661000,     1088, 0xde20aa20
 0,        708,        708,        1,   518400, 0xb8bc1365
-0,        716,        716,        1,   518400, 0xbab197ea
 0,        817,        817,        1,   518400, 0x83efa32d
 1,  163445000,  163445000,  1331000,      339, 0x8bd186ef
-0,        824,        824,        1,   518400, 0xbab197ea
 0,        840,        840,        1,   518400, 0x03ea0e90
 1,  168049000,  168049000,  1900000,     1312, 0x0bf20e8d
-0,        850,        850,        1,   518400, 0xbab197ea
+0,        850,        850,        1,   518400, 0x8780239e
 1,  170035000,  170035000,  1524000,     1279, 0xb6c2dafe
-0,        851,        851,        1,   518400, 0x8780239e
-0,        858,        858,        1,   518400, 0xbab197ea
 0,        861,        861,        1,   518400, 0x6eb72347
 1,  172203000,  172203000,  1695000,     1826, 0x9a1ac769
-0,        869,        869,        1,   518400, 0xbab197ea
 1,  173947000,  173947000,  1934000,     1474, 0xa9b03cdc
 0,        870,        870,        1,   518400, 0x9c4a3a3d
-0,        879,        879,        1,   518400, 0xbab197ea
 1,  175957000,  175957000,  1763000,     1019, 0x20409355
 0,        880,        880,        1,   518400, 0xc9ebfa89
-0,        889,        889,        1,   518400, 0xbab197ea
 0,        946,        946,        1,   518400, 0xbaf801ef
 1,  189295000,  189295000,  1968000,     1596, 0x408c726e
-0,        956,        956,        1,   518400, 0xbab197ea
 1,  191356000,  191356000,  1228000,     1517, 0xae8c5c2b
 0,        957,        957,        1,   518400, 0x59f4e72f
-0,        963,        963,        1,   518400, 0xbab197ea
+0,        963,        963,        1,   518400, 0x9d5b9d69
 1,  192640000,  192640000,  1763000,     2506, 0xa458d6d4
-0,        964,        964,        1,   518400, 0x9d5b9d69
-0,        972,        972,        1,   518400, 0xbab197ea
 1,  195193000,  195193000,  1092000,     1074, 0x397ba9a8
 0,        976,        976,        1,   518400, 0x923d1ce7
-0,        981,        981,        1,   518400, 0xbab197ea
 1,  196361000,  196361000,  1524000,     1715, 0x695ca41e
 0,        982,        982,        1,   518400, 0x6e652cd2
-0,        989,        989,        1,   518400, 0xbab197ea
 1,  197946000,  197946000,  1160000,      789, 0xc63a189e
 0,        990,        990,        1,   518400, 0x25113966
-0,        996,        996,        1,   518400, 0xbab197ea
+0,        996,        996,        1,   518400, 0x2dc83609
 1,  199230000,  199230000,  1627000,     1846, 0xeea8c599
-0,        997,        997,        1,   518400, 0x2dc83609
-0,       1004,       1004,        1,   518400, 0xbab197ea
 1,  200924000,  200924000,  1763000,      922, 0xd4a87222
 0,       1005,       1005,        1,   518400, 0x90483bc6
-0,       1013,       1013,        1,   518400, 0xbab197ea
 0,       1053,       1053,        1,   518400, 0x3de86ab7
 1,  210600000,  210600000,  1831000,      665, 0x55580135
-0,       1062,       1062,        1,   518400, 0xbab197ea
 1,  214771000,  214771000,  1558000,     1216, 0x50d1f6c5
 0,       1074,       1074,        1,   518400, 0x8c320e68
-0,       1082,       1082,        1,   518400, 0xbab197ea
 0,       1128,       1128,        1,   518400, 0x81e977b2
 1,  225640000,  225640000,  2127000,     2133, 0x670c11a5
-0,       1139,       1139,        1,   518400, 0xbab197ea
+0,       1139,       1139,        1,   518400, 0xb046dd30
 1,  227834000,  227834000,  1262000,     1264, 0xc1d9fc57
-0,       1140,       1140,        1,   518400, 0xb046dd30
-0,       1145,       1145,        1,   518400, 0xbab197ea
diff --git a/tests/ref/fate/sub2video_basic b/tests/ref/fate/sub2video_basic
index 5f72e292c9..41e02c057c 100644
--- a/tests/ref/fate/sub2video_basic
+++ b/tests/ref/fate/sub2video_basic
@@ -2,94 +2,47 @@ 
 #media_type 0: video
 #codec_id 0: rawvideo
 #dimensions 0: 720x480
-#sar 0: 0/1
-0,       3312,       3312,        1,  1382400, 0x00000000
-0,       3312,       3312,        1,  1382400, 0x8c93c2ba
-0,       3436,       3436,        1,  1382400, 0x00000000
-0,       3684,       3684,        1,  1382400, 0xb02e32ca
-0,       3802,       3802,        1,  1382400, 0x00000000
-0,       4520,       4520,        1,  1382400, 0x83b71116
-0,       4584,       4584,        1,  1382400, 0x00000000
-0,       4586,       4586,        1,  1382400, 0x85547fd1
-0,       4645,       4645,        1,  1382400, 0x00000000
-0,       4648,       4648,        1,  1382400, 0x00000000
-0,       4648,       4648,        1,  1382400, 0xb6a8f181
-0,       4715,       4715,        1,  1382400, 0x00000000
-0,       4717,       4717,        1,  1382400, 0xb64d1a2c
-0,       4748,       4748,        1,  1382400, 0x00000000
-0,       4750,       4750,        1,  1382400, 0x7b37ecf3
-0,       4792,       4792,        1,  1382400, 0x00000000
-0,       4993,       4993,        1,  1382400, 0xdc025bd1
-0,       5027,       5027,        1,  1382400, 0x00000000
-0,       5029,       5029,        1,  1382400, 0x688b294d
-0,       5068,       5068,        1,  1382400, 0x00000000
-0,       5070,       5070,        1,  1382400, 0xa2b33d1b
-0,       5117,       5117,        1,  1382400, 0x00000000
-0,       5119,       5119,        1,  1382400, 0xb3e525e3
-0,       5168,       5168,        1,  1382400, 0x00000000
-0,       5170,       5170,        1,  1382400, 0xaa8fbdd7
-0,       5216,       5216,        1,  1382400, 0x00000000
-0,       5218,       5218,        1,  1382400, 0x7b7f26dd
-0,       5249,       5249,        1,  1382400, 0x00000000
-0,       5251,       5251,        1,  1382400, 0x15e2f836
-0,       5289,       5289,        1,  1382400, 0x00000000
-0,       5291,       5291,        1,  1382400, 0x0fee9b0c
-0,       5358,       5358,        1,  1382400, 0x00000000
-0,       5360,       5360,        1,  1382400, 0x89d62791
-0,       5429,       5429,        1,  1382400, 0x00000000
-0,       5431,       5431,        1,  1382400, 0xa6a9fd74
-0,       5490,       5490,        1,  1382400, 0x00000000
-0,       5491,       5491,        1,  1382400, 0x7896178d
-0,       5537,       5537,        1,  1382400, 0x00000000
-0,       5588,       5588,        1,  1382400, 0x01751a52
-0,       5647,       5647,        1,  1382400, 0x00000000
-0,       5688,       5688,        1,  1382400, 0xa3959c6f
-0,       5770,       5770,        1,  1382400, 0x00000000
-0,       5772,       5772,        1,  1382400, 0x3d3ea47b
-0,       5826,       5826,        1,  1382400, 0x00000000
-0,       5828,       5828,        1,  1382400, 0x593f8b24
-0,       5931,       5931,        1,  1382400, 0x00000000
-0,       5933,       5933,        1,  1382400, 0x171f05ba
-0,       6001,       6001,        1,  1382400, 0x00000000
-0,       6003,       6003,        1,  1382400, 0xb014cdf1
-0,       6054,       6054,        1,  1382400, 0x00000000
-0,       6839,       6839,        1,  1382400, 0xd918e667
-0,       6880,       6880,        1,  1382400, 0x00000000
-0,       7386,       7386,        1,  1382400, 0xc9406331
-0,       7419,       7419,        1,  1382400, 0x00000000
-0,       7501,       7501,        1,  1382400, 0xaf08b10d
-0,       7549,       7549,        1,  1382400, 0x00000000
-0,       7551,       7551,        1,  1382400, 0x00000000
-0,       7551,       7551,        1,  1382400, 0x853a9d93
-0,       7589,       7589,        1,  1382400, 0x00000000
-0,       7605,       7605,        1,  1382400, 0x7491a87d
-0,       7647,       7647,        1,  1382400, 0x00000000
-0,       7649,       7649,        1,  1382400, 0xf7383c58
-0,       7697,       7697,        1,  1382400, 0x00000000
-0,       7699,       7699,        1,  1382400, 0xe66be411
-0,       7743,       7743,        1,  1382400, 0x00000000
-0,       8032,       8032,        1,  1382400, 0xd6850362
-0,       8082,       8082,        1,  1382400, 0x00000000
-0,       8084,       8084,        1,  1382400, 0x3e1ed109
-0,       8115,       8115,        1,  1382400, 0x00000000
-0,       8116,       8116,        1,  1382400, 0x39c1b7bd
-0,       8160,       8160,        1,  1382400, 0x00000000
-0,       8180,       8180,        1,  1382400, 0x35b85f2e
-0,       8207,       8207,        1,  1382400, 0x00000000
-0,       8209,       8209,        1,  1382400, 0x00000000
-0,       8209,       8209,        1,  1382400, 0x83f103e5
-0,       8247,       8247,        1,  1382400, 0x00000000
-0,       8249,       8249,        1,  1382400, 0xbc1ca9b3
-0,       8278,       8278,        1,  1382400, 0x00000000
-0,       8281,       8281,        1,  1382400, 0x94d4a51e
-0,       8321,       8321,        1,  1382400, 0x00000000
-0,       8323,       8323,        1,  1382400, 0xf88cdfde
-0,       8367,       8367,        1,  1382400, 0x00000000
-0,       8565,       8565,        1,  1382400, 0xdd51423b
-0,       8611,       8611,        1,  1382400, 0x00000000
-0,       8669,       8669,        1,  1382400, 0x08259fa4
-0,       8708,       8708,        1,  1382400, 0x00000000
-0,       8941,       8941,        1,  1382400, 0x1663fa34
-0,       8994,       8994,        1,  1382400, 0x00000000
-0,       8996,       8996,        1,  1382400, 0xda2ceb55
-0,       9027,       9027,        1,  1382400, 0x00000000
+#sar 0: 1/1
+0,       3312,       3312,        1,  1382400, 0xc637b893
+0,       3684,       3684,        1,  1382400, 0x4c2960ca
+0,       4520,       4520,        1,  1382400, 0x5fa18966
+0,       4586,       4586,        1,  1382400, 0x55f4b7b1
+0,       4648,       4648,        1,  1382400, 0xdfa4cf32
+0,       4717,       4717,        1,  1382400, 0x35023df8
+0,       4750,       4750,        1,  1382400, 0xed933219
+0,       4993,       4993,        1,  1382400, 0x1b26389a
+0,       5029,       5029,        1,  1382400, 0xf0c7028b
+0,       5070,       5070,        1,  1382400, 0x395f521d
+0,       5119,       5119,        1,  1382400, 0x1ea87415
+0,       5170,       5170,        1,  1382400, 0xc6effdc1
+0,       5218,       5218,        1,  1382400, 0xba6846f8
+0,       5251,       5251,        1,  1382400, 0x033c5d5b
+0,       5291,       5291,        1,  1382400, 0xef5abf66
+0,       5360,       5360,        1,  1382400, 0xec747954
+0,       5431,       5431,        1,  1382400, 0xfa34bcaf
+0,       5491,       5491,        1,  1382400, 0x8b7a709b
+0,       5588,       5588,        1,  1382400, 0xc333382f
+0,       5688,       5688,        1,  1382400, 0xabe5dfcf
+0,       5772,       5772,        1,  1382400, 0x56948101
+0,       5828,       5828,        1,  1382400, 0xb747834a
+0,       5933,       5933,        1,  1382400, 0x3448baad
+0,       6003,       6003,        1,  1382400, 0xaabe4f37
+0,       6839,       6839,        1,  1382400, 0x8a48cd6f
+0,       7386,       7386,        1,  1382400, 0x49518c43
+0,       7501,       7501,        1,  1382400, 0x4a72fa21
+0,       7551,       7551,        1,  1382400, 0xa82f7de8
+0,       7605,       7605,        1,  1382400, 0xeba0b5f3
+0,       7649,       7649,        1,  1382400, 0xd6a91770
+0,       7699,       7699,        1,  1382400, 0x222f827c
+0,       8032,       8032,        1,  1382400, 0x3270f4ff
+0,       8084,       8084,        1,  1382400, 0x40813cb3
+0,       8116,       8116,        1,  1382400, 0x9d8fde41
+0,       8180,       8180,        1,  1382400, 0xc6d7a701
+0,       8209,       8209,        1,  1382400, 0x9d45f2dc
+0,       8249,       8249,        1,  1382400, 0x8525ee40
+0,       8281,       8281,        1,  1382400, 0x5b26b98b
+0,       8323,       8323,        1,  1382400, 0x51be311f
+0,       8565,       8565,        1,  1382400, 0x00a4f2a3
+0,       8669,       8669,        1,  1382400, 0x40a445e8
+0,       8941,       8941,        1,  1382400, 0x43ef5128
+0,       8996,       8996,        1,  1382400, 0x3c3e3819
diff --git a/tests/ref/fate/sub2video_time_limited b/tests/ref/fate/sub2video_time_limited
index 9fb6fb06f9..715af02fee 100644
--- a/tests/ref/fate/sub2video_time_limited
+++ b/tests/ref/fate/sub2video_time_limited
@@ -2,7 +2,5 @@ 
 #media_type 0: video
 #codec_id 0: rawvideo
 #dimensions 0: 1920x1080
-#sar 0: 0/1
-0,          2,          2,        1,  8294400, 0x00000000
+#sar 0: 1/1
 0,          2,          2,        1,  8294400, 0xa87c518f
-0,         10,         10,        1,  8294400, 0xa87c518f