diff mbox series

[FFmpeg-devel,20/24] avfilter/subfeed: add subtitle feed filter

Message ID ad07dbf7c2bdf5af839ff3f166203e4220687f0a.1642122814.git.ffmpegagent@gmail.com
State New
Headers show
Series Subtitle Filtering 2022
Related show

Checks

Context Check Description
andriy/make_armv7_RPi4 success Make finished
andriy/make_fate_armv7_RPi4 fail Make fate failed
andriy/make_x86 success Make finished
andriy/make_fate_x86 fail Make fate failed
andriy/makex86 warning New warnings during build
andriy/makearmv7_RPi4 warning New warnings during build
andriy/make_ppc success Make finished
andriy/make_fate_ppc fail Make fate failed
andriy/makeppc warning New warnings during build
andriy/make_aarch64_jetson success Make finished
andriy/make_fate_aarch64_jetson fail Make fate failed
andriy/makeaarch64_jetson warning New warnings during build

Commit Message

ffmpegagent Jan. 14, 2022, 1:13 a.m. UTC
From: softworkz <softworkz@hotmail.com>

Signed-off-by: softworkz <softworkz@hotmail.com>
---
 libavfilter/Makefile     |   1 +
 libavfilter/allfilters.c |   1 +
 libavfilter/sf_subfeed.c | 366 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 368 insertions(+)
 create mode 100644 libavfilter/sf_subfeed.c
diff mbox series

Patch

diff --git a/libavfilter/Makefile b/libavfilter/Makefile
index 0e3e48613e..5711c7770b 100644
--- a/libavfilter/Makefile
+++ b/libavfilter/Makefile
@@ -570,6 +570,7 @@  OBJS-$(CONFIG_TEXTMOD_FILTER)                += sf_textmod.o
 OBJS-$(CONFIG_SPLITCC_FILTER)                += sf_splitcc.o
 OBJS-$(CONFIG_STRIPSTYLES_FILTER)            += sf_stripstyles.o
 OBJS-$(CONFIG_SUBSCALE_FILTER)               += sf_subscale.o
+OBJS-$(CONFIG_SUBFEED_FILTER)                += sf_subfeed.o
 
 # multimedia filters
 OBJS-$(CONFIG_ABITSCOPE_FILTER)              += avf_abitscope.o
diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
index 8e4f2feca3..10b14b7b8e 100644
--- a/libavfilter/allfilters.c
+++ b/libavfilter/allfilters.c
@@ -553,6 +553,7 @@  extern const AVFilter ff_sf_showspeaker;
 extern const AVFilter ff_sf_splitcc;
 extern const AVFilter ff_sf_stripstyles;
 extern const AVFilter ff_sf_subscale;
+extern const AVFilter ff_sf_subfeed;
 extern const AVFilter ff_sf_textmod;
 extern const AVFilter ff_svf_graphicsub2video;
 extern const AVFilter ff_svf_textsub2video;
diff --git a/libavfilter/sf_subfeed.c b/libavfilter/sf_subfeed.c
new file mode 100644
index 0000000000..8a7ac5528b
--- /dev/null
+++ b/libavfilter/sf_subfeed.c
@@ -0,0 +1,366 @@ 
+/*
+ * Copyright (c) 2021 softworkz
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file
+ * subtitle filter for feeding subtitle frames into a filtergraph in a contiguous way
+ *
+ *
+ * also supports
+ *   - duration fixup
+ *     delaying a subtitle event with unknown duration and infer duration from the
+ *     start time of the subsequent subtitle
+ *   - scattering
+ *     splitting a subtitle event with unknown duration into multiple ones with
+ *     a short and fixed duration
+ *
+ */
+
+#include "filters.h"
+#include "libavutil/opt.h"
+#include "subtitles.h"
+#include "libavutil/avassert.h"
+
+enum SubFeedMode {
+    FM_REPEAT,
+    FM_SCATTER,
+    FM_FORWARD,
+};
+
+typedef struct SubFeedContext {
+    const AVClass *class;
+    enum AVSubtitleType format;
+    enum SubFeedMode mode;
+
+    AVRational frame_rate;
+    int fix_durations;
+    int fix_overlap;
+
+    int current_frame_isnew;
+    int eof;
+    int got_first_input;
+    int need_frame;
+    int64_t next_pts_offset;
+    int64_t recent_subtitle_pts;
+
+    int64_t counter;
+
+    /**
+     * Queue of frames waiting to be filtered.
+     */
+    FFFrameQueue fifo;
+
+} SubFeedContext;
+
+static int64_t ms_to_avtb(int64_t ms)
+{
+    return av_rescale_q(ms, (AVRational){ 1, 1000 }, AV_TIME_BASE_Q);
+}
+
+static int64_t avtb_to_ms(int64_t avtb)
+{
+    return av_rescale_q(avtb, AV_TIME_BASE_Q, (AVRational){ 1, 1000 });
+}
+
+static int init(AVFilterContext *ctx)
+{
+    SubFeedContext *s = ctx->priv;
+
+    ff_framequeue_init(&s->fifo, NULL);
+
+    return 0;
+}
+
+static void uninit(AVFilterContext *ctx)
+{
+    SubFeedContext *s = ctx->priv;
+    ff_framequeue_free(&s->fifo);
+}
+
+static int config_input(AVFilterLink *link)
+{
+    ////const subfeedContext *context = link->dst->priv;
+
+    return 0;
+}
+
+static int query_formats(AVFilterContext *ctx)
+{
+    AVFilterFormats *formats;
+    AVFilterLink *inlink0 = ctx->inputs[0];
+    AVFilterLink *outlink0 = ctx->outputs[0];
+    static const enum AVSubtitleType subtitle_fmts[] = { AV_SUBTITLE_FMT_BITMAP, AV_SUBTITLE_FMT_ASS, AV_SUBTITLE_FMT_NB };
+    int ret;
+
+    formats = ff_make_format_list(subtitle_fmts);
+
+    if ((ret = ff_formats_ref(formats, &inlink0->outcfg.formats)) < 0)
+        return ret;
+
+    if ((ret = ff_formats_ref(formats, &outlink0->incfg.formats)) < 0)
+        return ret;
+
+    return 0;
+}
+
+static int config_output(AVFilterLink *outlink)
+{
+    SubFeedContext *s = outlink->src->priv;
+    const AVFilterLink *inlink = outlink->src->inputs[0];
+
+    outlink->time_base = AV_TIME_BASE_Q;
+    outlink->format = inlink->format;
+    outlink->w = inlink->w;
+    outlink->h = inlink->h;
+
+    if (s->mode == FM_FORWARD)
+        outlink->frame_rate = (AVRational) { 1, 0 };
+    else
+        outlink->frame_rate = s->frame_rate;
+
+    return 0;
+}
+
+static int request_frame(AVFilterLink *outlink)
+{
+    SubFeedContext *s = outlink->src->priv;
+    AVFilterLink *inlink = outlink->src->inputs[0];
+    int64_t last_pts = outlink->current_pts;
+    int64_t next_pts;
+    int64_t interval = ms_to_avtb((int64_t)(av_q2d(av_inv_q(outlink->frame_rate)) * 1000));
+    AVFrame *out;
+    int status;
+
+    if (s->mode == FM_FORWARD)
+        return ff_request_frame(inlink);
+
+    s->counter++;
+    if (interval == 0)
+        interval =  ms_to_avtb(200);
+
+    status = ff_outlink_get_status(inlink);
+    if (status == AVERROR_EOF)
+        s->eof = 1;
+
+    if (s->eof)
+        return AVERROR_EOF;
+
+    if (!s->got_first_input && inlink->current_pts != AV_NOPTS_VALUE) {
+
+        s->got_first_input = 1;
+        next_pts = av_rescale_q(inlink->current_pts, inlink->time_base, AV_TIME_BASE_Q);
+        if (next_pts < last_pts)
+            next_pts = last_pts + interval;
+
+    } else if (last_pts == AV_NOPTS_VALUE)
+        next_pts = av_rescale_q(inlink->current_pts, inlink->time_base, AV_TIME_BASE_Q);
+    else
+        next_pts = last_pts + interval;
+
+    if (next_pts == AV_NOPTS_VALUE)
+        next_pts = 0;
+
+    if (s->next_pts_offset) {
+        next_pts -= s->next_pts_offset;
+        s->next_pts_offset = 0;
+    }
+
+retry:
+    if (ff_framequeue_queued_frames(&s->fifo) && !s->current_frame_isnew) {
+
+        const AVFrame *current_frame = ff_framequeue_peek(&s->fifo, 0);
+        const int64_t sub_end_time   = current_frame->subtitle_timing.start_pts + current_frame->subtitle_timing.duration;
+
+        if (next_pts > sub_end_time) {
+            AVFrame *remove_frame = ff_framequeue_take(&s->fifo);
+            av_frame_free(&remove_frame);
+
+            if (ff_framequeue_queued_frames(&s->fifo)) {
+                s->current_frame_isnew = 1;
+                goto retry;
+            }
+        }
+    }
+
+    if (ff_framequeue_queued_frames(&s->fifo)) {
+        AVFrame *current_frame = ff_framequeue_peek(&s->fifo, 0);
+
+        if (current_frame && current_frame->subtitle_timing.start_pts <= next_pts + interval) {
+            if (!s->current_frame_isnew)
+                current_frame->repeat_sub++;
+
+            out = av_frame_clone(current_frame);
+
+            if (!out)
+                return AVERROR(ENOMEM);
+
+            if (!s->current_frame_isnew) {
+                out->pts = next_pts;
+            } else {
+                out->pts = out->subtitle_timing.start_pts;
+
+                if (out->pts < next_pts)
+                    out->pts = next_pts;
+
+                s->next_pts_offset = (out->pts - next_pts) % interval;
+            }
+
+            if (s->mode == FM_SCATTER) {
+                const int64_t sub_end_time  = current_frame->subtitle_timing.start_pts + current_frame->subtitle_timing.duration;
+
+                if (s->current_frame_isnew == 1 && current_frame->subtitle_timing.start_pts < out->pts) {
+                    const int64_t diff = out->pts - current_frame->subtitle_timing.start_pts;
+                    current_frame->subtitle_timing.duration -= diff;
+                }
+
+                out->repeat_sub = 0;
+                out->subtitle_timing.start_pts = out->pts;
+                out->subtitle_timing.duration = interval;
+                av_assert1(out->pts >= next_pts);
+                av_assert1(out->pts < next_pts + interval);
+                av_assert1(out->pts < sub_end_time);
+
+                if (out->pts > next_pts)
+                    out->subtitle_timing.duration -= out->pts - next_pts;
+
+                if (sub_end_time < next_pts + interval) {
+                    const int64_t diff = next_pts + interval - sub_end_time;
+                    av_assert1(diff <= out->subtitle_timing.duration);
+                    out->subtitle_timing.duration -= diff;
+                }
+            }
+
+            s->current_frame_isnew = 0;
+            s->recent_subtitle_pts = out->subtitle_timing.start_pts;
+
+            return ff_filter_frame(outlink, out);
+        }
+    }
+
+    if (ff_framequeue_queued_frames(&s->fifo) == 0) {
+        status = ff_request_frame(inlink);
+        if (status == AVERROR_EOF) {
+            s->eof = 1;
+            return status;
+        }
+
+        if (s->counter > 1 && s->counter % 2)
+            return 0;
+    }
+
+    out = ff_get_subtitles_buffer(outlink, outlink->format);
+    out->pts = next_pts;
+    out->repeat_sub = 1;
+    out->subtitle_timing.start_pts = s->recent_subtitle_pts;
+
+    return ff_filter_frame(outlink, out);
+}
+
+static int filter_frame(AVFilterLink *inlink, AVFrame *frame)
+{
+    AVFilterContext *ctx        = inlink->dst;
+    SubFeedContext *s           = inlink->dst->priv;
+    AVFilterLink *outlink       = inlink->dst->outputs[0];
+    const int64_t index         = (int64_t)ff_framequeue_queued_frames(&s->fifo) - 1;
+    int ret = 0;
+
+    frame->pts = av_rescale_q(frame->pts, inlink->time_base, AV_TIME_BASE_Q);
+
+    if (index < 0) {
+        s->current_frame_isnew = 1;
+    } else if (s->fix_durations || s->fix_overlap) {
+        AVFrame *previous_frame = ff_framequeue_peek(&s->fifo, index);
+        const int64_t pts_diff = frame->subtitle_timing.start_pts - previous_frame->subtitle_timing.start_pts;
+
+        if (s->fix_durations && pts_diff > 0 && previous_frame->subtitle_timing.duration > ms_to_avtb(29000))
+                    previous_frame->subtitle_timing.duration = frame->subtitle_timing.start_pts - previous_frame->subtitle_timing.start_pts;
+
+        if (s->fix_overlap && pts_diff > 0 && previous_frame->subtitle_timing.duration > pts_diff)
+                    previous_frame->subtitle_timing.duration = pts_diff;
+    }
+
+    ff_framequeue_add(&s->fifo, frame);
+
+    if (index > 3)
+        av_log(ctx, AV_LOG_WARNING, "frame queue count: %d\n", (int)index);
+
+    if (s->mode == FM_FORWARD && ff_framequeue_queued_frames(&s->fifo)) {
+
+        AVFrame *first_frame = ff_framequeue_peek(&s->fifo, 0);
+
+        if (s->fix_overlap && ff_framequeue_queued_frames(&s->fifo) < 2)
+            return 0;
+
+        if (s->fix_durations && first_frame->subtitle_timing.duration > ms_to_avtb(29000))
+            return 0;
+
+        first_frame = ff_framequeue_take(&s->fifo);
+        return ff_filter_frame(outlink, first_frame);
+    }
+
+    return ret;
+}
+
+#define OFFSET(x) offsetof(SubFeedContext, x)
+#define FLAGS (AV_OPT_FLAG_SUBTITLE_PARAM | AV_OPT_FLAG_FILTERING_PARAM)
+
+static const AVOption subfeed_options[] = {
+    { "fix_durations", "delay output and determine duration from next frame", OFFSET(fix_durations),   AV_OPT_TYPE_BOOL,   { .i64=1 },                  0, 1, FLAGS, NULL     },
+    { "fix_overlap", "delay output and adjust durations to prevent overlap", OFFSET(fix_overlap),   AV_OPT_TYPE_BOOL,   { .i64=0 },                  0, 1, FLAGS, NULL     },
+    { "mode",       "set feed mode",         OFFSET(mode),      AV_OPT_TYPE_INT,                 { .i64=FM_REPEAT },  FM_REPEAT, FM_FORWARD,  FLAGS, "mode" },
+    {   "repeat",     "repeat recent while valid, send empty otherwise",   0, AV_OPT_TYPE_CONST, { .i64=FM_REPEAT },  0,                  0,  FLAGS, "mode" },
+    {   "scatter",    "subdivide subtitles into 1/framerate segments",     0, AV_OPT_TYPE_CONST, { .i64=FM_SCATTER }, 0,                  0,  FLAGS, "mode" },
+    {   "forward",    "forward only (clears output framerate)",            0, AV_OPT_TYPE_CONST, { .i64=FM_FORWARD }, 0,                  0,  FLAGS, "mode" },
+    { "rate",       "output frame rate",     OFFSET(frame_rate), AV_OPT_TYPE_VIDEO_RATE,         { .str = "5"},       0,         INT_MAX,     FLAGS, NULL },\
+    { "r",          "output frame rate",     OFFSET(frame_rate), AV_OPT_TYPE_VIDEO_RATE,         { .str = "5"},       0,         INT_MAX,     FLAGS, NULL },\
+    { NULL },
+};
+
+AVFILTER_DEFINE_CLASS(subfeed);
+
+static const AVFilterPad inputs[] = {
+    {
+        .name         = "default",
+        .type         = AVMEDIA_TYPE_SUBTITLE,
+        .filter_frame = filter_frame,
+        .config_props = config_input,
+    },
+};
+
+static const AVFilterPad outputs[] = {
+    {
+        .name          = "default",
+        .type          = AVMEDIA_TYPE_SUBTITLE,
+        .request_frame = request_frame,
+        .config_props  = config_output,
+    },
+};
+
+const AVFilter ff_sf_subfeed = {
+    .name           = "subfeed",
+    .description    = NULL_IF_CONFIG_SMALL("Scatter subtitle events by duration"),
+    .init           = init,
+    .uninit         = uninit,
+    .priv_size      = sizeof(SubFeedContext),
+    .priv_class     = &subfeed_class,
+    FILTER_INPUTS(inputs),
+    FILTER_OUTPUTS(outputs),
+    FILTER_QUERY_FUNC(query_formats),
+};