[FFmpeg-devel,11/13] avformat/demux: support inserting bitstream filters in demuxing scenarios

Packets will be passed to the bsf immediately after being generated by a
demuxer, and no further data will be read from the input until all packets
have been returned by the bsf.

Signed-off-by: James Almer <jamrial@gmail.com>
---
 libavformat/avformat.c |  47 ++++++++++++
 libavformat/demux.c    | 162 ++++++++++++++++++++++++++++++-----------
 libavformat/internal.h |  13 +++-
 libavformat/mux.c      |  43 -----------
 libavformat/mux.h      |  11 ---
 libavformat/rawenc.c   |   1 +
 6 files changed, 181 insertions(+), 96 deletions(-)

On 2023-11-28 12:13 am, James Almer wrote:
> Packets will be passed to the bsf immediately after being generated by a
> demuxer, and no further data will be read from the input until all packets
> have been returned by the bsf.

Do you plan to add a lib/cli option for user-specified insertions?
Will be useful for something like dts2pts in some cases.

Regards,
Gyan


>
> Signed-off-by: James Almer <jamrial@gmail.com>
> ---
>   libavformat/avformat.c |  47 ++++++++++++
>   libavformat/demux.c    | 162 ++++++++++++++++++++++++++++++-----------
>   libavformat/internal.h |  13 +++-
>   libavformat/mux.c      |  43 -----------
>   libavformat/mux.h      |  11 ---
>   libavformat/rawenc.c   |   1 +
>   6 files changed, 181 insertions(+), 96 deletions(-)
>
> diff --git a/libavformat/avformat.c b/libavformat/avformat.c
> index a02ec965dd..a41c0b391c 100644
> --- a/libavformat/avformat.c
> +++ b/libavformat/avformat.c
> @@ -1033,3 +1033,50 @@ FF_ENABLE_DEPRECATION_WARNINGS
>       *pb = NULL;
>       return ret;
>   }
> +
> +int ff_stream_add_bitstream_filter(AVStream *st, const char *name, const char *args)
> +{
> +    int ret;
> +    const AVBitStreamFilter *bsf;
> +    FFStream *const sti = ffstream(st);
> +    AVBSFContext *bsfc;
> +
> +    av_assert0(!sti->bsfc);
> +
> +    if (name) {
> +        bsf = av_bsf_get_by_name(name);
> +        if (!bsf) {
> +            av_log(NULL, AV_LOG_ERROR, "Unknown bitstream filter '%s'\n", name);
> +            return AVERROR_BSF_NOT_FOUND;
> +        }
> +        ret = av_bsf_alloc(bsf, &bsfc);
> +    } else
> +        ret = av_bsf_get_null_filter(&bsfc);
> +    if (ret < 0)
> +        return ret;
> +
> +    bsfc->time_base_in = st->time_base;
> +    if ((ret = avcodec_parameters_copy(bsfc->par_in, st->codecpar)) < 0) {
> +        av_bsf_free(&bsfc);
> +        return ret;
> +    }
> +
> +    if (args && bsfc->filter->priv_class) {
> +        if ((ret = av_set_options_string(bsfc->priv_data, args, "=", ":")) < 0) {
> +            av_bsf_free(&bsfc);
> +            return ret;
> +        }
> +    }
> +
> +    if ((ret = av_bsf_init(bsfc)) < 0) {
> +        av_bsf_free(&bsfc);
> +        return ret;
> +    }
> +
> +    sti->bsfc = bsfc;
> +
> +    av_log(NULL, AV_LOG_VERBOSE,
> +           "Automatically inserted bitstream filter '%s'; args='%s'\n",
> +           name, args ? args : "");
> +    return 1;
> +}
> diff --git a/libavformat/demux.c b/libavformat/demux.c
> index 6f640b92b1..fb9bf9e4ac 100644
> --- a/libavformat/demux.c
> +++ b/libavformat/demux.c
> @@ -540,6 +540,109 @@ static int update_wrap_reference(AVFormatContext *s, AVStream *st, int stream_in
>       return 1;
>   }
>   
> +static void update_timestamps(AVFormatContext *s, AVStream *st, AVPacket *pkt)
> +{
> +    FFStream *const sti = ffstream(st);
> +
> +    if (update_wrap_reference(s, st, pkt->stream_index, pkt) && sti->pts_wrap_behavior == AV_PTS_WRAP_SUB_OFFSET) {
> +        // correct first time stamps to negative values
> +        if (!is_relative(sti->first_dts))
> +            sti->first_dts = wrap_timestamp(st, sti->first_dts);
> +        if (!is_relative(st->start_time))
> +            st->start_time = wrap_timestamp(st, st->start_time);
> +        if (!is_relative(sti->cur_dts))
> +            sti->cur_dts = wrap_timestamp(st, sti->cur_dts);
> +    }
> +
> +    pkt->dts = wrap_timestamp(st, pkt->dts);
> +    pkt->pts = wrap_timestamp(st, pkt->pts);
> +
> +    force_codec_ids(s, st);
> +
> +    /* TODO: audio: time filter; video: frame reordering (pts != dts) */
> +    if (s->use_wallclock_as_timestamps)
> +        pkt->dts = pkt->pts = av_rescale_q(av_gettime(), AV_TIME_BASE_Q, st->time_base);
> +}
> +
> +static int filter_packet(AVFormatContext *s, AVStream *st, AVPacket *pkt)
> +{
> +    FFFormatContext *const si = ffformatcontext(s);
> +    FFStream *const sti = ffstream(st);
> +    const AVPacket *pkt1;
> +    int err;
> +
> +    if (!sti->bsfc) {
> +        const PacketListEntry *pktl = si->raw_packet_buffer.head;
> +        if (AVPACKET_IS_EMPTY(pkt))
> +            return 0;
> +
> +        update_timestamps(s, st, pkt);
> +
> +        if (!pktl && sti->request_probe <= 0)
> +            return 0;
> +
> +        err = avpriv_packet_list_put(&si->raw_packet_buffer, pkt, NULL, 0);
> +        if (err < 0) {
> +            av_packet_unref(pkt);
> +            return err;
> +        }
> +
> +        pkt1 = &si->raw_packet_buffer.tail->pkt;
> +        si->raw_packet_buffer_size += pkt1->size;
> +
> +        if (sti->request_probe <= 0)
> +            return 0;
> +
> +        return probe_codec(s, s->streams[pkt1->stream_index], pkt1);
> +    }
> +
> +    err = av_bsf_send_packet(sti->bsfc, pkt);
> +    if (err < 0) {
> +        av_log(s, AV_LOG_ERROR,
> +                "Failed to send packet to filter %s for stream %d\n",
> +                sti->bsfc->filter->name, st->index);
> +        return err;
> +    }
> +
> +    do {
> +        AVStream *out_st;
> +        FFStream *out_sti;
> +
> +        err = av_bsf_receive_packet(sti->bsfc, pkt);
> +        if (err < 0) {
> +            if (err == AVERROR(EAGAIN) || err == AVERROR_EOF)
> +                return 0;
> +            av_log(s, AV_LOG_ERROR, "Error applying bitstream filters to an output "
> +                   "packet for stream #%d: %s\n", st->index, av_err2str(err));
> +            if (!(s->error_recognition & AV_EF_EXPLODE) && err != AVERROR(ENOMEM))
> +                continue;
> +            return err;
> +        }
> +        out_st = s->streams[pkt->stream_index];
> +        out_sti = ffstream(out_st);
> +
> +        update_timestamps(s, out_st, pkt);
> +
> +        err = avpriv_packet_list_put(&si->raw_packet_buffer, pkt, NULL, 0);
> +        if (err < 0) {
> +            av_packet_unref(pkt);
> +            return err;
> +        }
> +
> +        pkt1 = &si->raw_packet_buffer.tail->pkt;
> +        si->raw_packet_buffer_size += pkt1->size;
> +
> +        if (out_sti->request_probe <= 0)
> +            continue;
> +
> +        err = probe_codec(s, out_st, pkt1);
> +        if (err < 0)
> +            return err;
> +    } while (1);
> +
> +    return 0;
> +}
> +
>   int ff_read_packet(AVFormatContext *s, AVPacket *pkt)
>   {
>       FFFormatContext *const si = ffformatcontext(s);
> @@ -557,9 +660,6 @@ FF_ENABLE_DEPRECATION_WARNINGS
>   
>       for (;;) {
>           PacketListEntry *pktl = si->raw_packet_buffer.head;
> -        AVStream *st;
> -        FFStream *sti;
> -        const AVPacket *pkt1;
>   
>           if (pktl) {
>               AVStream *const st = s->streams[pktl->pkt.stream_index];
> @@ -582,16 +682,27 @@ FF_ENABLE_DEPRECATION_WARNINGS
>                  We must re-call the demuxer to get the real packet. */
>               if (err == FFERROR_REDO)
>                   continue;
> -            if (!pktl || err == AVERROR(EAGAIN))
> +            if (err == AVERROR(EAGAIN))
>                   return err;
>               for (unsigned i = 0; i < s->nb_streams; i++) {
>                   AVStream *const st  = s->streams[i];
>                   FFStream *const sti = ffstream(st);
> +                int ret;
> +
> +                // Drain buffered packets in the bsf context on eof
> +                if (err == AVERROR_EOF)
> +                    if ((ret = filter_packet(s, st, pkt)) < 0)
> +                        return ret;
> +                pktl = si->raw_packet_buffer.head;
> +                if (!pktl)
> +                    continue;
>                   if (sti->probe_packets || sti->request_probe > 0)
> -                    if ((err = probe_codec(s, st, NULL)) < 0)
> -                        return err;
> +                    if ((ret = probe_codec(s, st, NULL)) < 0)
> +                        return ret;
>                   av_assert0(sti->request_probe <= 0);
>               }
> +            if (!pktl)
> +                return err;
>               continue;
>           }
>   
> @@ -616,42 +727,11 @@ FF_ENABLE_DEPRECATION_WARNINGS
>           av_assert0(pkt->stream_index < (unsigned)s->nb_streams &&
>                      "Invalid stream index.\n");
>   
> -        st  = s->streams[pkt->stream_index];
> -        sti = ffstream(st);
> -
> -        if (update_wrap_reference(s, st, pkt->stream_index, pkt) && sti->pts_wrap_behavior == AV_PTS_WRAP_SUB_OFFSET) {
> -            // correct first time stamps to negative values
> -            if (!is_relative(sti->first_dts))
> -                sti->first_dts = wrap_timestamp(st, sti->first_dts);
> -            if (!is_relative(st->start_time))
> -                st->start_time = wrap_timestamp(st, st->start_time);
> -            if (!is_relative(sti->cur_dts))
> -                sti->cur_dts = wrap_timestamp(st, sti->cur_dts);
> -        }
> -
> -        pkt->dts = wrap_timestamp(st, pkt->dts);
> -        pkt->pts = wrap_timestamp(st, pkt->pts);
> -
> -        force_codec_ids(s, st);
> -
> -        /* TODO: audio: time filter; video: frame reordering (pts != dts) */
> -        if (s->use_wallclock_as_timestamps)
> -            pkt->dts = pkt->pts = av_rescale_q(av_gettime(), AV_TIME_BASE_Q, st->time_base);
> -
> -        if (!pktl && sti->request_probe <= 0)
> -            return 0;
> -
> -        err = avpriv_packet_list_put(&si->raw_packet_buffer,
> -                                     pkt, NULL, 0);
> -        if (err < 0) {
> -            av_packet_unref(pkt);
> -            return err;
> -        }
> -        pkt1 = &si->raw_packet_buffer.tail->pkt;
> -        si->raw_packet_buffer_size += pkt1->size;
> -
> -        if ((err = probe_codec(s, st, pkt1)) < 0)
> +        err = filter_packet(s, s->streams[pkt->stream_index], pkt);
> +        if (err < 0)
>               return err;
> +        if (!AVPACKET_IS_EMPTY(pkt))
> +            return 0;
>       }
>   }
>   
> diff --git a/libavformat/internal.h b/libavformat/internal.h
> index c6181683ef..0a5d512697 100644
> --- a/libavformat/internal.h
> +++ b/libavformat/internal.h
> @@ -212,7 +212,7 @@ typedef struct FFStream {
>       /**
>        * bitstream filter to run on stream
>        * - encoding: Set by muxer using ff_stream_add_bitstream_filter
> -     * - decoding: unused
> +     * - decoding: Set by demuxer using ff_stream_add_bitstream_filter
>        */
>       struct AVBSFContext *bsfc;
>   
> @@ -757,4 +757,15 @@ int ff_match_url_ext(const char *url, const char *extensions);
>   struct FFOutputFormat;
>   void avpriv_register_devices(const struct FFOutputFormat * const o[], const AVInputFormat * const i[]);
>   
> +/**
> + * Add a bitstream filter to a stream.
> + *
> + * @param st output stream to add a filter to
> + * @param name the name of the filter to add
> + * @param args filter-specific argument string
> + * @return  >0 on success;
> + *          AVERROR code on failure
> + */
> +int ff_stream_add_bitstream_filter(AVStream *st, const char *name, const char *args);
> +
>   #endif /* AVFORMAT_INTERNAL_H */
> diff --git a/libavformat/mux.c b/libavformat/mux.c
> index de10d2c008..4bc8627617 100644
> --- a/libavformat/mux.c
> +++ b/libavformat/mux.c
> @@ -1344,49 +1344,6 @@ int av_get_output_timestamp(struct AVFormatContext *s, int stream,
>       return 0;
>   }
>   
> -int ff_stream_add_bitstream_filter(AVStream *st, const char *name, const char *args)
> -{
> -    int ret;
> -    const AVBitStreamFilter *bsf;
> -    FFStream *const sti = ffstream(st);
> -    AVBSFContext *bsfc;
> -
> -    av_assert0(!sti->bsfc);
> -
> -    if (!(bsf = av_bsf_get_by_name(name))) {
> -        av_log(NULL, AV_LOG_ERROR, "Unknown bitstream filter '%s'\n", name);
> -        return AVERROR_BSF_NOT_FOUND;
> -    }
> -
> -    if ((ret = av_bsf_alloc(bsf, &bsfc)) < 0)
> -        return ret;
> -
> -    bsfc->time_base_in = st->time_base;
> -    if ((ret = avcodec_parameters_copy(bsfc->par_in, st->codecpar)) < 0) {
> -        av_bsf_free(&bsfc);
> -        return ret;
> -    }
> -
> -    if (args && bsfc->filter->priv_class) {
> -        if ((ret = av_set_options_string(bsfc->priv_data, args, "=", ":")) < 0) {
> -            av_bsf_free(&bsfc);
> -            return ret;
> -        }
> -    }
> -
> -    if ((ret = av_bsf_init(bsfc)) < 0) {
> -        av_bsf_free(&bsfc);
> -        return ret;
> -    }
> -
> -    sti->bsfc = bsfc;
> -
> -    av_log(NULL, AV_LOG_VERBOSE,
> -           "Automatically inserted bitstream filter '%s'; args='%s'\n",
> -           name, args ? args : "");
> -    return 1;
> -}
> -
>   int ff_write_chained(AVFormatContext *dst, int dst_stream, AVPacket *pkt,
>                        AVFormatContext *src, int interleave)
>   {
> diff --git a/libavformat/mux.h b/libavformat/mux.h
> index b9ec75641d..ab3e8edd60 100644
> --- a/libavformat/mux.h
> +++ b/libavformat/mux.h
> @@ -171,17 +171,6 @@ const AVPacket *ff_interleaved_peek(AVFormatContext *s, int stream);
>   
>   int ff_get_muxer_ts_offset(AVFormatContext *s, int stream_index, int64_t *offset);
>   
> -/**
> - * Add a bitstream filter to a stream.
> - *
> - * @param st output stream to add a filter to
> - * @param name the name of the filter to add
> - * @param args filter-specific argument string
> - * @return  >0 on success;
> - *          AVERROR code on failure
> - */
> -int ff_stream_add_bitstream_filter(AVStream *st, const char *name, const char *args);
> -
>   /**
>    * Write a packet to another muxer than the one the user originally
>    * intended. Useful when chaining muxers, where one muxer internally
> diff --git a/libavformat/rawenc.c b/libavformat/rawenc.c
> index f916db13a2..ec31d76d88 100644
> --- a/libavformat/rawenc.c
> +++ b/libavformat/rawenc.c
> @@ -25,6 +25,7 @@
>   #include "libavutil/intreadwrite.h"
>   
>   #include "avformat.h"
> +#include "internal.h"
>   #include "rawenc.h"
>   #include "mux.h"
>

On 11/28/2023 1:00 AM, Gyan Doshi wrote:
> 
> 
> On 2023-11-28 12:13 am, James Almer wrote:
>> Packets will be passed to the bsf immediately after being generated by a
>> demuxer, and no further data will be read from the input until all 
>> packets
>> have been returned by the bsf.
> 
> Do you plan to add a lib/cli option for user-specified insertions?

No. This is internal to lavf, same as the muxing implementation.

> Will be useful for something like dts2pts in some cases.

Is the existing -bsfs option not enough?

On 2023-11-30 03:25 am, James Almer wrote:
> On 11/28/2023 1:00 AM, Gyan Doshi wrote:
>>
>>
>> On 2023-11-28 12:13 am, James Almer wrote:
>>> Packets will be passed to the bsf immediately after being generated 
>>> by a
>>> demuxer, and no further data will be read from the input until all 
>>> packets
>>> have been returned by the bsf.
>>
>> Do you plan to add a lib/cli option for user-specified insertions?
>
> No. This is internal to lavf, same as the muxing implementation.
>
>> Will be useful for something like dts2pts in some cases.
>
> Is the existing -bsfs option not enough?

It's not flagged with OPT_INPUT  so can't be set for inputs.

Regards,
Gyan