| Message ID | 20190501141316.24367-1-barsnick@gmx.net |
|---|---|
| State | New |
| Headers | show |
On 5/1/2019 11:13 AM, Moritz Barsnick wrote: > Works for video and audio streams. > > Similar to the "-re" option in ffmpeg, but unlike that option does not > only work for input files, and is not only implemented for the > command line tool. This filter is available through the libraries, > and unlike the "realtime" filter also works when using the "copy" > encoder. > > Implementation mostly taken from libavfilter/f_realtime.c. > --- > doc/bitstream_filters.texi | 24 +++++++++++ > libavcodec/Makefile | 1 + > libavcodec/bitstream_filters.c | 1 + > libavcodec/realtime_bsf.c | 93 ++++++++++++++++++++++++++++++++++++++++++ > 4 files changed, 119 insertions(+) > create mode 100644 libavcodec/realtime_bsf.c > > diff --git a/doc/bitstream_filters.texi b/doc/bitstream_filters.texi > index 25bbf8372b..18ddb2e52b 100644 > --- a/doc/bitstream_filters.texi > +++ b/doc/bitstream_filters.texi > @@ -599,6 +599,30 @@ Set Rec709 colorspace for each frame of the file > ffmpeg -i INPUT -c copy -bsf:v prores_metadata=color_primaries=bt709:color_trc=bt709:colorspace=bt709 output.mov > @end example > > +@section realtime > + > +Slow down output processing to match real time approximately. > + > +This bitstream filter will pause the filtering for a variable amount of time > +to match the output rate with the input timestamps. It is similar to the > +@option{re} option to @code{ffmpeg}. > + > +It accepts the following options: > + > +@table @option > +@item limit > +Time limit for the pauses. Any pause longer than that will be considered > +a timestamp discontinuity and reset the timer. Default is 2 seconds. > +@item speed > +Speed factor for processing. The value must be a float larger than zero. > +Values larger than 1.0 will result in faster than realtime processing, > +smaller will slow processing down. The @var{limit} is automatically adapted > +accordingly. Default is 1.0. > + > +A processing speed faster than what is possible without this bitstream > +filter cannot be achieved. > +@end table > + > @section remove_extra > > Remove extradata from packets. > diff --git a/libavcodec/Makefile b/libavcodec/Makefile > index f37135fc07..83f143922f 100644 > --- a/libavcodec/Makefile > +++ b/libavcodec/Makefile > @@ -1094,6 +1094,7 @@ OBJS-$(CONFIG_MPEG2_METADATA_BSF) += mpeg2_metadata_bsf.o > OBJS-$(CONFIG_NOISE_BSF) += noise_bsf.o > OBJS-$(CONFIG_NULL_BSF) += null_bsf.o > OBJS-$(CONFIG_PRORES_METADATA_BSF) += prores_metadata_bsf.o > +OBJS-$(CONFIG_REALTIME_BSF) += realtime_bsf.o > OBJS-$(CONFIG_REMOVE_EXTRADATA_BSF) += remove_extradata_bsf.o > OBJS-$(CONFIG_TEXT2MOVSUB_BSF) += movsub_bsf.o > OBJS-$(CONFIG_TRACE_HEADERS_BSF) += trace_headers_bsf.o > diff --git a/libavcodec/bitstream_filters.c b/libavcodec/bitstream_filters.c > index 463003966a..741a240380 100644 > --- a/libavcodec/bitstream_filters.c > +++ b/libavcodec/bitstream_filters.c > @@ -49,6 +49,7 @@ extern const AVBitStreamFilter ff_mov2textsub_bsf; > extern const AVBitStreamFilter ff_noise_bsf; > extern const AVBitStreamFilter ff_null_bsf; > extern const AVBitStreamFilter ff_prores_metadata_bsf; > +extern const AVBitStreamFilter ff_realtime_bsf; > extern const AVBitStreamFilter ff_remove_extradata_bsf; > extern const AVBitStreamFilter ff_text2movsub_bsf; > extern const AVBitStreamFilter ff_trace_headers_bsf; > diff --git a/libavcodec/realtime_bsf.c b/libavcodec/realtime_bsf.c > new file mode 100644 > index 0000000000..b9abda90a4 > --- /dev/null > +++ b/libavcodec/realtime_bsf.c > @@ -0,0 +1,93 @@ > +/* > + * Realtime filter > + * Copyright (c) 2015 Nicolas George > + * Copyright (c) 2018 Moritz Barsnick > + * > + * This file is part of FFmpeg. > + * > + * FFmpeg is free software; you can redistribute it and/or > + * modify it under the terms of the GNU Lesser General Public > + * License as published by the Free Software Foundation; either > + * version 2.1 of the License, or (at your option) any later version. > + * > + * FFmpeg is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + * Lesser General Public License for more details. > + * > + * You should have received a copy of the GNU Lesser General Public > + * License along with FFmpeg; if not, write to the Free Software > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > + */ > + > +#include "libavutil/time.h" > +#include "libavutil/opt.h" > +#include "avcodec.h" > +#include "bsf.h" > +#include <float.h> > + > +typedef struct RealtimeContext { > + const AVClass *class; > + int64_t delta; > + int64_t limit; > + double speed; > + unsigned inited; > +} RealtimeContext; > + > +static int realtime_filter(AVBSFContext *bsf, AVPacket *pkt) > +{ > + int ret; > + RealtimeContext *ctx = bsf->priv_data; > + > + ret = ff_bsf_get_packet_ref(bsf, pkt); > + if (ret < 0) > + return ret; > + > + if (pkt->pts != AV_NOPTS_VALUE) { > + int64_t pts = av_rescale_q(pkt->pts, bsf->time_base_in, AV_TIME_BASE_Q) / ctx->speed; > + int64_t now = av_gettime_relative(); > + int64_t sleep = pts - now + ctx->delta; > + if (!ctx->inited) { > + ctx->inited = 1; > + sleep = 0; > + ctx->delta = now - pts; If this is meant to be used for input, where seeking can take place, wouldn't a flush() callback to set ctx->inited and ctx->delta back to 0 be needed? > + } > + if (FFABS(sleep) > ctx->limit / ctx->speed) { > + av_log(ctx, AV_LOG_WARNING, > + "time discontinuity detected: %"PRIi64" us, resetting\n", > + sleep); > + sleep = 0; > + ctx->delta = now - pts; > + } > + if (sleep > 0) { > + av_log(ctx, AV_LOG_DEBUG, "sleeping %"PRIi64" us\n", sleep); > + for (; sleep > 600000000; sleep -= 600000000) > + av_usleep(600000000); > + av_usleep(sleep); > + } > + } > + > + return 0; > +} > + > +#define OFFSET(x) offsetof(RealtimeContext, x) > +#define FLAGS (AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_BSF_PARAM) > +static const AVOption options[] = { > + { "limit", "sleep time limit", OFFSET(limit), AV_OPT_TYPE_DURATION, { .i64 = 2000000 }, 0, INT64_MAX, FLAGS }, > + { "speed", "speed factor", OFFSET(speed), AV_OPT_TYPE_DOUBLE, { .dbl = 1.0 }, DBL_MIN, DBL_MAX, FLAGS }, > + { NULL }, > +}; > + > +static const AVClass realtime_class = { > + .class_name = "realtime_bsf", > + .item_name = av_default_item_name, > + .option = options, > + .version = LIBAVUTIL_VERSION_INT, > +}; > + > +const AVBitStreamFilter ff_realtime_bsf = { > + .name = "realtime", > + .priv_data_size = sizeof(RealtimeContext), > + .priv_class = &realtime_class, > + .filter = realtime_filter, > +}; > -- > 2.14.5 > > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". >
On Wed, May 01, 2019 at 12:03:41 -0300, James Almer wrote: > > + if (pkt->pts != AV_NOPTS_VALUE) { > > + int64_t pts = av_rescale_q(pkt->pts, bsf->time_base_in, AV_TIME_BASE_Q) / ctx->speed; > > + int64_t now = av_gettime_relative(); > > + int64_t sleep = pts - now + ctx->delta; > > + if (!ctx->inited) { > > + ctx->inited = 1; > > + sleep = 0; > > + ctx->delta = now - pts; > > If this is meant to be used for input, where seeking can take place, > wouldn't a flush() callback to set ctx->inited and ctx->delta back to 0 > be needed? Interesting point. I suppose you mean backward seeking. Is that a valid use case for a bitstream filter? Would it also apply to the standard filter, where I took this code from? I don't know, but I can try to add that, if required. Can you tell me how to emulate or test seeking from the command line? Would I create a file with backward jumps in PTS (and how)? I also missed to bump libavcodec version, or to at least mention it, BTW. Thanks, Moritz
On 02-05-2019 04:12 PM, Moritz Barsnick wrote: > On Wed, May 01, 2019 at 12:03:41 -0300, James Almer wrote: >>> + if (pkt->pts != AV_NOPTS_VALUE) { >>> + int64_t pts = av_rescale_q(pkt->pts, bsf->time_base_in, AV_TIME_BASE_Q) / ctx->speed; >>> + int64_t now = av_gettime_relative(); >>> + int64_t sleep = pts - now + ctx->delta; >>> + if (!ctx->inited) { >>> + ctx->inited = 1; >>> + sleep = 0; >>> + ctx->delta = now - pts; >> If this is meant to be used for input, where seeking can take place, >> wouldn't a flush() callback to set ctx->inited and ctx->delta back to 0 >> be needed? > Interesting point. I suppose you mean backward seeking. Is that a valid > use case for a bitstream filter? Would it also apply to the standard > filter, where I took this code from? > > I don't know, but I can try to add that, if required. Can you tell me > how to emulate or test seeking from the command line? Would I create a > file with backward jumps in PTS (and how)? Also, you may want to add support for -copyts which -re lacks as well. Another complication is timestamp rollover, independent of copyts. How does the filter behave with non-monotonic timestamps? Gyan P.S. you can craft files to test all of this by generating individual TS files with custom timestamps and then 'cat'ting them together, whether via the concat protocol or shell utility.
On 5/2/2019 7:42 AM, Moritz Barsnick wrote: > On Wed, May 01, 2019 at 12:03:41 -0300, James Almer wrote: >>> + if (pkt->pts != AV_NOPTS_VALUE) { >>> + int64_t pts = av_rescale_q(pkt->pts, bsf->time_base_in, AV_TIME_BASE_Q) / ctx->speed; >>> + int64_t now = av_gettime_relative(); >>> + int64_t sleep = pts - now + ctx->delta; >>> + if (!ctx->inited) { >>> + ctx->inited = 1; >>> + sleep = 0; >>> + ctx->delta = now - pts; >> >> If this is meant to be used for input, where seeking can take place, >> wouldn't a flush() callback to set ctx->inited and ctx->delta back to 0 >> be needed? > > Interesting point. I suppose you mean backward seeking. Is that a valid > use case for a bitstream filter? The flush() callback exists to reset the bsf into its initial state without the need to destroy and recreate the context. For example, some decoders auto-insert certain bitstream filters that get applied by the generic lavc code before they are feed packets, like the vp9 decoder and some hardware based h264/hevc decoders. avcodec_flush_buffers(), which must be called every time seeking needs to be done in a decoding scenario, flushes both the decoder and said bitstream filters. I don't think we'll ever have a decoder auto inserting this bsf, but we don't know what an API user might want to do with bitstream filters in their own scenarios. And if the bsf keeps a state, a flush() callback is required to reset it. > Would it also apply to the standard filter, where I took this code from? There doesn't seem to be any kind of flush mechanism for filters, so i suppose in those you're expected to uninit and reinit the filter every time? > > I don't know, but I can try to add that, if required. Can you tell me > how to emulate or test seeking from the command line? Would I create a > file with backward jumps in PTS (and how)? I guess the simplest way would be to auto insert this bsf into some decoder (AVCodec.bsfs string), then seek using some libavcodec based application that doesn't already decodes at realtime. > > I also missed to bump libavcodec version, or to at least mention it, > BTW. Missing changelog entry and minor version bump, yeah. > > Thanks, > Moritz
On Fri, May 03, 2019 at 23:47:47 -0300, James Almer wrote: > The flush() callback exists to reset the bsf into its initial state > without the need to destroy and recreate the context. > For example, some decoders auto-insert certain bitstream filters that [...] > avcodec_flush_buffers(), which must be called every time seeking needs > to be done in a decoding scenario, flushes both the decoder and said > bitstream filters. Thanks for the explanation! > I guess the simplest way would be to auto insert this bsf into some > decoder (AVCodec.bsfs string), then seek using some libavcodec based > application that doesn't already decodes at realtime. It doesn't sound simple, but I'll try to construct a test and fix the code. Meanwhile, this version of the patch is withdrawn. Thanks, Moritz
On 5/4/2019 12:06 PM, Moritz Barsnick wrote: > On Fri, May 03, 2019 at 23:47:47 -0300, James Almer wrote: >> The flush() callback exists to reset the bsf into its initial state >> without the need to destroy and recreate the context. >> For example, some decoders auto-insert certain bitstream filters that > [...] >> avcodec_flush_buffers(), which must be called every time seeking needs >> to be done in a decoding scenario, flushes both the decoder and said >> bitstream filters. > > Thanks for the explanation! > >> I guess the simplest way would be to auto insert this bsf into some >> decoder (AVCodec.bsfs string), then seek using some libavcodec based >> application that doesn't already decodes at realtime. > > It doesn't sound simple, but I'll try to construct a test and fix the > code. Meanwhile, this version of the patch is withdrawn. As long as the flush() callback leaves the bsf context in the same state as if it was fresh out of an alloc/init call, it should be good and doing what's expected from it. So don't worry much about writing a custom test for it. > > Thanks, > Moritz > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". >
diff --git a/doc/bitstream_filters.texi b/doc/bitstream_filters.texi index 25bbf8372b..18ddb2e52b 100644 --- a/doc/bitstream_filters.texi +++ b/doc/bitstream_filters.texi @@ -599,6 +599,30 @@ Set Rec709 colorspace for each frame of the file ffmpeg -i INPUT -c copy -bsf:v prores_metadata=color_primaries=bt709:color_trc=bt709:colorspace=bt709 output.mov @end example +@section realtime + +Slow down output processing to match real time approximately. + +This bitstream filter will pause the filtering for a variable amount of time +to match the output rate with the input timestamps. It is similar to the +@option{re} option to @code{ffmpeg}. + +It accepts the following options: + +@table @option +@item limit +Time limit for the pauses. Any pause longer than that will be considered +a timestamp discontinuity and reset the timer. Default is 2 seconds. +@item speed +Speed factor for processing. The value must be a float larger than zero. +Values larger than 1.0 will result in faster than realtime processing, +smaller will slow processing down. The @var{limit} is automatically adapted +accordingly. Default is 1.0. + +A processing speed faster than what is possible without this bitstream +filter cannot be achieved. +@end table + @section remove_extra Remove extradata from packets. diff --git a/libavcodec/Makefile b/libavcodec/Makefile index f37135fc07..83f143922f 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -1094,6 +1094,7 @@ OBJS-$(CONFIG_MPEG2_METADATA_BSF) += mpeg2_metadata_bsf.o OBJS-$(CONFIG_NOISE_BSF) += noise_bsf.o OBJS-$(CONFIG_NULL_BSF) += null_bsf.o OBJS-$(CONFIG_PRORES_METADATA_BSF) += prores_metadata_bsf.o +OBJS-$(CONFIG_REALTIME_BSF) += realtime_bsf.o OBJS-$(CONFIG_REMOVE_EXTRADATA_BSF) += remove_extradata_bsf.o OBJS-$(CONFIG_TEXT2MOVSUB_BSF) += movsub_bsf.o OBJS-$(CONFIG_TRACE_HEADERS_BSF) += trace_headers_bsf.o diff --git a/libavcodec/bitstream_filters.c b/libavcodec/bitstream_filters.c index 463003966a..741a240380 100644 --- a/libavcodec/bitstream_filters.c +++ b/libavcodec/bitstream_filters.c @@ -49,6 +49,7 @@ extern const AVBitStreamFilter ff_mov2textsub_bsf; extern const AVBitStreamFilter ff_noise_bsf; extern const AVBitStreamFilter ff_null_bsf; extern const AVBitStreamFilter ff_prores_metadata_bsf; +extern const AVBitStreamFilter ff_realtime_bsf; extern const AVBitStreamFilter ff_remove_extradata_bsf; extern const AVBitStreamFilter ff_text2movsub_bsf; extern const AVBitStreamFilter ff_trace_headers_bsf; diff --git a/libavcodec/realtime_bsf.c b/libavcodec/realtime_bsf.c new file mode 100644 index 0000000000..b9abda90a4 --- /dev/null +++ b/libavcodec/realtime_bsf.c @@ -0,0 +1,93 @@ +/* + * Realtime filter + * Copyright (c) 2015 Nicolas George + * Copyright (c) 2018 Moritz Barsnick + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/time.h" +#include "libavutil/opt.h" +#include "avcodec.h" +#include "bsf.h" +#include <float.h> + +typedef struct RealtimeContext { + const AVClass *class; + int64_t delta; + int64_t limit; + double speed; + unsigned inited; +} RealtimeContext; + +static int realtime_filter(AVBSFContext *bsf, AVPacket *pkt) +{ + int ret; + RealtimeContext *ctx = bsf->priv_data; + + ret = ff_bsf_get_packet_ref(bsf, pkt); + if (ret < 0) + return ret; + + if (pkt->pts != AV_NOPTS_VALUE) { + int64_t pts = av_rescale_q(pkt->pts, bsf->time_base_in, AV_TIME_BASE_Q) / ctx->speed; + int64_t now = av_gettime_relative(); + int64_t sleep = pts - now + ctx->delta; + if (!ctx->inited) { + ctx->inited = 1; + sleep = 0; + ctx->delta = now - pts; + } + if (FFABS(sleep) > ctx->limit / ctx->speed) { + av_log(ctx, AV_LOG_WARNING, + "time discontinuity detected: %"PRIi64" us, resetting\n", + sleep); + sleep = 0; + ctx->delta = now - pts; + } + if (sleep > 0) { + av_log(ctx, AV_LOG_DEBUG, "sleeping %"PRIi64" us\n", sleep); + for (; sleep > 600000000; sleep -= 600000000) + av_usleep(600000000); + av_usleep(sleep); + } + } + + return 0; +} + +#define OFFSET(x) offsetof(RealtimeContext, x) +#define FLAGS (AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_BSF_PARAM) +static const AVOption options[] = { + { "limit", "sleep time limit", OFFSET(limit), AV_OPT_TYPE_DURATION, { .i64 = 2000000 }, 0, INT64_MAX, FLAGS }, + { "speed", "speed factor", OFFSET(speed), AV_OPT_TYPE_DOUBLE, { .dbl = 1.0 }, DBL_MIN, DBL_MAX, FLAGS }, + { NULL }, +}; + +static const AVClass realtime_class = { + .class_name = "realtime_bsf", + .item_name = av_default_item_name, + .option = options, + .version = LIBAVUTIL_VERSION_INT, +}; + +const AVBitStreamFilter ff_realtime_bsf = { + .name = "realtime", + .priv_data_size = sizeof(RealtimeContext), + .priv_class = &realtime_class, + .filter = realtime_filter, +};