diff mbox series

[FFmpeg-devel,v2,7/8] avfilter/sleet: Add sleet filter

Message ID MN2PR04MB5981B9E6A547F6A4FFF379B7BACB9@MN2PR04MB5981.namprd04.prod.outlook.com
State New
Headers show
Series [FFmpeg-devel,v2,1/8] lavu/frame: avframe add type property
Related show

Checks

Context Check Description
andriy/configurex86 warning Failed to apply patch
andriy/configureppc warning Failed to apply patch

Commit Message

Soft Works Aug. 30, 2021, 8:17 a.m. UTC
Signed-off-by: softworkz <softworkz@hotmail.com>
---
v2 Update:

- Implemented Andreas' suggestions
- overlay_subs filter:
  - removed duplicated code
  - implemented direct (no pre-conversion) blending of graphical
    subtitle rects
  - Supported input formats:
    - all packed RGB formats (with and without alpha)
	- yuv420p, yuv422p, yuv444p

 libavfilter/Makefile     |   3 +
 libavfilter/allfilters.c |   1 +
 libavfilter/sf_sleet.c   | 209 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 213 insertions(+)
 create mode 100644 libavfilter/sf_sleet.c

Comments

Mapul Bhola Aug. 31, 2021, 9:39 p.m. UTC | #1
August 30, 2021 4:17 AM, "Soft Works" <softworkz@hotmail.com> wrote:

> Signed-off-by: softworkz <softworkz@hotmail.com>
> ---
> v2 Update:
> 
> - Implemented Andreas' suggestions
> - overlay_subs filter:
> - removed duplicated code
> - implemented direct (no pre-conversion) blending of graphical
> subtitle rects
> - Supported input formats:
> - all packed RGB formats (with and without alpha)
> - yuv420p, yuv422p, yuv444p
> 
> libavfilter/Makefile | 3 +
> libavfilter/allfilters.c | 1 +
> libavfilter/sf_sleet.c | 209 +++++++++++++++++++++++++++++++++++++++
> 3 files changed, 213 insertions(+)
> create mode 100644 libavfilter/sf_sleet.c
> 
> diff --git a/libavfilter/Makefile b/libavfilter/Makefile
> index e38c6b6f6d..25dd1276de 100644
> --- a/libavfilter/Makefile
> +++ b/libavfilter/Makefile
> @@ -526,6 +526,9 @@ OBJS-$(CONFIG_YUVTESTSRC_FILTER) += vsrc_testsrc.o
> 
> OBJS-$(CONFIG_NULLSINK_FILTER) += vsink_nullsink.o
> 
> +# subtitle filters
> +OBJS-$(CONFIG_SLEET_FILTER) += sf_sleet.o
> +
> # multimedia filters
> OBJS-$(CONFIG_ABITSCOPE_FILTER) += avf_abitscope.o
> OBJS-$(CONFIG_ADRAWGRAPH_FILTER) += f_drawgraph.o
> diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
> index 5bd54db2c8..efe16b8e1b 100644
> --- a/libavfilter/allfilters.c
> +++ b/libavfilter/allfilters.c
> @@ -519,6 +519,7 @@ extern const AVFilter ff_avf_showwaves;
> extern const AVFilter ff_avf_showwavespic;
> extern const AVFilter ff_vaf_spectrumsynth;
> extern const AVFilter ff_svf_sub2video;
> +extern const AVFilter ff_sf_sleet;
> 
> /* multimedia sources */
> extern const AVFilter ff_avsrc_amovie;
> diff --git a/libavfilter/sf_sleet.c b/libavfilter/sf_sleet.c
> new file mode 100644
> index 0000000000..cf7701c01f
> --- /dev/null
> +++ b/libavfilter/sf_sleet.c
> @@ -0,0 +1,209 @@
> +/*
> + * Copyright (c) 2021 softworkz
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +/**
> + * @file
> + * text subtitle filter which translates to 'leet speak'
> + */
> +
> +#include "libavutil/avassert.h"
> +#include "libavutil/avstring.h"
> +#include "libavutil/opt.h"
> +#include "avfilter.h"
> +#include "internal.h"
> +#include "libavcodec/avcodec.h"
> +
> +static const char* alphabet_src = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
> +static const char* alphabet_dst = "abcd3f6#1jklmn0pq257uvwxyzAB(D3F6#1JKLMN0PQ257UVWXYZ";
> +
> +
> +typedef struct LeetContext {
> + const AVClass *class;
> + enum AVSubtitleType format;
> +} LeetContext;
> +
> +static const AVOption sleet_options[] = {
> + { NULL }
> +};
> +
> +AVFILTER_DEFINE_CLASS(sleet);
> +
> +static int query_formats(AVFilterContext *ctx)
> +{
> + AVFilterFormats *formats = NULL;
> + AVFilterLink *inlink = ctx->inputs[0];
> + AVFilterLink *outlink = ctx->outputs[0];
> + static const enum AVSubtitleType subtitle_fmts[] = { SUBTITLE_ASS, SUBTITLE_NONE };
> + static const enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_RGB32, AV_PIX_FMT_NONE };
> + int ret;
> +
> + /* set input subtitle format */
> + formats = ff_make_format_list(subtitle_fmts);
> + if ((ret = ff_formats_ref(formats, &inlink->outcfg.formats)) < 0)
> + return ret;
> +
> + /* set output video format */
> + formats = ff_make_format_list(pix_fmts);
> + if ((ret = ff_formats_ref(formats, &outlink->incfg.formats)) < 0)
> + return ret;
> +
> + return 0;
> +}
> +
> +static int config_input(AVFilterLink *inlink)
> +{
> + AVFilterContext *ctx = inlink->dst;
> + LeetContext *s = ctx->priv;
> +
> + s->format = inlink->format;
> + return 0;
> +}
> +
> +static int config_output(AVFilterLink *outlink)
> +{
> + LeetContext *s = outlink->src->priv;
> +
> + outlink->format = s->format;
> +
> + return 0;
> +}
> +
> +static void avsubtitle_free_ref(void *opaque, uint8_t *data)
> +{
> + avsubtitle_free((AVSubtitle *)data);
> +}
> +
> +static int filter_frame(AVFilterLink *inlink, AVFrame *src_frame)
> +{
> + LeetContext *s = inlink->dst->priv;
> + AVFilterLink *outlink = inlink->dst->outputs[0];
> + AVSubtitle *sub;
> + int ret;
> + AVFrame *out;
> + unsigned int num_rects;
> + uint8_t *dst;
> +
> + outlink->format = inlink->format;
> +
> + out = av_frame_alloc();
> + if (!out) {
> + av_frame_free(&src_frame);
> + return AVERROR(ENOMEM);
> + }
> +
> + out->format = outlink->format;
> +
> + if ((ret = av_frame_get_buffer2(out, AVMEDIA_TYPE_SUBTITLE, 0)) < 0)
> + return ret;
> +
> + out->pts = src_frame->pts;
> + out->repeat_pict = src_frame->repeat_pict;
> + out->pkt_dts = src_frame->pkt_dts;
> + out->pkt_pos = src_frame->pkt_pos;
> + out->pkt_size = src_frame->pkt_size;
> + out->pkt_duration = src_frame->pkt_duration;
> + out->reordered_opaque = src_frame->reordered_opaque;
> + out->best_effort_timestamp = src_frame->best_effort_timestamp;
> + out->flags = src_frame->flags;
> +
> + sub = (AVSubtitle *)src_frame->data[0];
> +
> + if (sub) {
> + AVSubtitle *out_sub = av_memdup(sub, sizeof(*out_sub));
> + if (!out_sub)
> + return AVERROR(ENOMEM);
> +
> + out->buf[0] = av_buffer_create((uint8_t*)out_sub, sizeof(*out_sub), avsubtitle_free_ref, NULL,
> AV_BUFFER_FLAG_READONLY);
> + out->data[0] = (uint8_t*)out_sub;
> +
> + if (sub->num_rects) {
> + out_sub->rects = av_malloc_array(sub->num_rects, sizeof(AVSubtitleRect *));
> + }
> +
> + for (unsigned i = 0; i < sub->num_rects; i++) {
> +
> + AVSubtitleRect *src_rect = sub->rects[i];
> + AVSubtitleRect *dst_rect = av_memdup(src_rect, sizeof(*dst_rect));
> + out_sub->rects[i] = dst_rect;
> +
> + if (src_rect->text) {
> + dst_rect->text = av_strdup(src_rect->text);
> + if (!dst_rect->text)
> + return AVERROR(ENOMEM);
> +
> + for (size_t n = 0; n < strlen(dst_rect->text); n++) {
> + for (size_t t = 0; t < FF_ARRAY_ELEMS(alphabet_src); t++) {
> + if (dst_rect->text[n] == alphabet_src[t]) {
> + dst_rect->text[n] = alphabet_dst[t];
> + break;
> + }
> + }
> + }
> + }
> +
> + if (src_rect->ass) {
> + dst_rect->ass = av_strdup(src_rect->ass);
> + if (!dst_rect->ass)
> + return AVERROR(ENOMEM);
> +
> + for (size_t n = 0; n < strlen(dst_rect->ass); n++) {
> + for (size_t t = 0; t < FF_ARRAY_ELEMS(alphabet_src); t++) {
> + if (dst_rect->ass[n] == alphabet_src[t]) {
> + dst_rect->ass[n] = alphabet_dst[t];
> + break;
> + }
> + }
> + }
> + }
> + }
> + }
> +
> + av_frame_free(&src_frame);
> + return ff_filter_frame(outlink, out);
> +}
> +
> +static const AVFilterPad sleet_inputs[] = {
> + {
> + .name = "default",
> + .type = AVMEDIA_TYPE_SUBTITLE,
> + .filter_frame = filter_frame,
> + .config_props = config_input,
> + },
> + { NULL }
> +};
> +
> +static const AVFilterPad sleet_outputs[] = {
> + {
> + .name = "default",
> + .type = AVMEDIA_TYPE_SUBTITLE,
> + .config_props = config_output,
> + },
> + { NULL }
> +};
> +
> +const AVFilter ff_sf_sleet = {
> + .name = "sleet",
> + .description = NULL_IF_CONFIG_SMALL("Translate text subtitles to 'leet speak'"),
> + .query_formats = query_formats,
> + .priv_size = sizeof(LeetContext),
> + .priv_class = &sleet_class,
> + .inputs = sleet_inputs,
> + .outputs = sleet_outputs,
> +};
> -- 
> 2.30.2.windows.1
> 
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> 
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Softworkz, I like ya, I like ya patches, and the effort your putting into the FFMpeg project, but I just don't think this is necessary. Contributes to overhead/maintence for what is essentially a gimmick feature.

Code itself loks good to me
Soft Works Aug. 31, 2021, 10:49 p.m. UTC | #2
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
> Mapul Bhola
> Sent: Tuesday, 31 August 2021 23:39
> To: FFmpeg development discussions and patches <ffmpeg-
> devel@ffmpeg.org>
> Subject: Re: [FFmpeg-devel] [PATCH v2 7/8] avfilter/sleet: Add sleet
> filter
> 
> August 30, 2021 4:17 AM, "Soft Works" <softworkz@hotmail.com> wrote:
> 
> > Signed-off-by: softworkz <softworkz@hotmail.com>
> > ---
> > v2 Update:
> >
> > - Implemented Andreas' suggestions
> > - overlay_subs filter:
> > - removed duplicated code
> > - implemented direct (no pre-conversion) blending of graphical
> > subtitle rects
> > - Supported input formats:
> > - all packed RGB formats (with and without alpha)
> > - yuv420p, yuv422p, yuv444p
> >
> > libavfilter/Makefile | 3 +
> > libavfilter/allfilters.c | 1 +
> > libavfilter/sf_sleet.c | 209
> +++++++++++++++++++++++++++++++++++++++

[..]

> Softworkz, I like ya, I like ya patches, and the effort your putting
> into the FFMpeg project, but I just don't think this is necessary.
> Contributes to overhead/maintence for what is essentially a gimmick
> feature.
> 
> Code itself loks good to me

Thanks for the kind words, which are rare here in this little conclave
of (partially) freaky developers who are considering themselves as the 
top of excellence, but haven't been able for years to proceed on some 
rather trivial yet highly important and useful issues and shortcomings 
in the current set of ffmpeg functionality.

As for the "sleet" filter: Yes - it is totally useless for practical
use. I had explained why I added it in the 0/8 message:

> Why leet? Real world use is surely questionable, but it has two advantages 
> that make it a great choice for testing: 
> You can see from almost every single line whether it has been filtered, 
> and the text is still readable which allows to verify that timings are 
> still valid.

Much more useful examples for textsub>>textsub filters would be like:

- Censoring filter
- Translation filter (lingual)
- Include Actor Name filter

Though, I wanted to have something super-simple for demonstration of the
new ability to have textsub>>textsub filters.

Maybe an "sallcaps" filter makes slightly more sense than "sleet"?

softworkz
Paul B Mahol Sept. 1, 2021, 6:20 a.m. UTC | #3
On Wed, Sep 1, 2021 at 12:49 AM Soft Works <softworkz@hotmail.com> wrote:

>
>
> > -----Original Message-----
> > From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
> > Mapul Bhola
> > Sent: Tuesday, 31 August 2021 23:39
> > To: FFmpeg development discussions and patches <ffmpeg-
> > devel@ffmpeg.org>
> > Subject: Re: [FFmpeg-devel] [PATCH v2 7/8] avfilter/sleet: Add sleet
> > filter
> >
> > August 30, 2021 4:17 AM, "Soft Works" <softworkz@hotmail.com> wrote:
> >
> > > Signed-off-by: softworkz <softworkz@hotmail.com>
> > > ---
> > > v2 Update:
> > >
> > > - Implemented Andreas' suggestions
> > > - overlay_subs filter:
> > > - removed duplicated code
> > > - implemented direct (no pre-conversion) blending of graphical
> > > subtitle rects
> > > - Supported input formats:
> > > - all packed RGB formats (with and without alpha)
> > > - yuv420p, yuv422p, yuv444p
> > >
> > > libavfilter/Makefile | 3 +
> > > libavfilter/allfilters.c | 1 +
> > > libavfilter/sf_sleet.c | 209
> > +++++++++++++++++++++++++++++++++++++++
>
> [..]
>
> > Softworkz, I like ya, I like ya patches, and the effort your putting
> > into the FFMpeg project, but I just don't think this is necessary.
> > Contributes to overhead/maintence for what is essentially a gimmick
> > feature.
> >
> > Code itself loks good to me
>
> Thanks for the kind words, which are rare here in this little conclave
> of (partially) freaky developers who are considering themselves as the
> top of excellence, but haven't been able for years to proceed on some
> rather trivial yet highly important and useful issues and shortcomings
> in the current set of ffmpeg functionality.
>

That approach is getting you nowhere.


>
> As for the "sleet" filter: Yes - it is totally useless for practical
> use. I had explained why I added it in the 0/8 message:
>
> > Why leet? Real world use is surely questionable, but it has two
> advantages
> > that make it a great choice for testing:
> > You can see from almost every single line whether it has been filtered,
> > and the text is still readable which allows to verify that timings are
> > still valid.
>
> Much more useful examples for textsub>>textsub filters would be like:
>
> - Censoring filter
> - Translation filter (lingual)
> - Include Actor Name filter
>
> Though, I wanted to have something super-simple for demonstration of the
> new ability to have textsub>>textsub filters.
>
> Maybe an "sallcaps" filter makes slightly more sense than "sleet"?
>
> softworkz
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
>
Moritz Barsnick Sept. 3, 2021, 8:23 a.m. UTC | #4
On Tue, Aug 31, 2021 at 22:49:12 +0000, Soft Works wrote:
> As for the "sleet" filter: Yes - it is totally useless for practical
> use. I had explained why I added it in the 0/8 message:
>
> > Why leet? Real world use is surely questionable, but it has two advantages
> > that make it a great choice for testing:
> > You can see from almost every single line whether it has been filtered,
> > and the text is still readable which allows to verify that timings are
> > still valid.

Okay, I get that case, though it seems a bit special. Isn't this just a
debug feature for developers then?

> Much more useful examples for textsub>>textsub filters would be like:
>
> - Censoring filter
> - Translation filter (lingual)
> - Include Actor Name filter
>
> Though, I wanted to have something super-simple for demonstration of the
> new ability to have textsub>>textsub filters.

Well, at least it demonstrates the boilerplate required for a simple
S->S filter, which touches only text (no timings or positioning or so).
A review of this boilerplate would be great.

For the sake of review:
- You need to add documentation to doc/filters.texi.
- You need to bump MINOR in libavfilter/version.h.

> Maybe an "sallcaps" filter makes slightly more sense than "sleet"?

For the sake of usability, I would rather have suggested something like
"tr" (https://linux.die.net/man/1/tr,
https://www.geeksforgeeks.org/perl-tr-operator/), where you can specify
the replacement list. That said, tr has some funky shortcuts ("[a-z]
[A-Z]" for uppercase, which may be harder to implement). Or actual
regular expression support.

What I suggest is still not really useful for a user, though. Or is it?

Cheers,
Moritz
Nicolas George Sept. 3, 2021, 8:30 a.m. UTC | #5
Moritz Barsnick (12021-09-03):
> Well, at least it demonstrates the boilerplate required for a simple
> S->S filter, which touches only text (no timings or positioning or so).
> A review of this boilerplate would be great.

That would be a waste of time since the grounds of this patch series are
unacceptable and have been rejected :

- There was no advanced reflection on how to overhaul the subtitles API
  and data structures.

- No new media type can be accepted into libavfilter before the
  negotiation process have been cleaned up and extended to allow
  negotiating the media type.

Regards,
Soft Works Sept. 3, 2021, 4:37 p.m. UTC | #6
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
> Nicolas George
> Sent: Friday, 3 September 2021 10:31
> To: FFmpeg development discussions and patches <ffmpeg-
> devel@ffmpeg.org>
> Subject: Re: [FFmpeg-devel] [PATCH v2 7/8] avfilter/sleet: Add sleet
> filter
> 
> Moritz Barsnick (12021-09-03):
> > Well, at least it demonstrates the boilerplate required for a
> simple
> > S->S filter, which touches only text (no timings or positioning or
> so).
> > A review of this boilerplate would be great.
> 
> That would be a waste of time since the grounds of this patch series
> are
> unacceptable and have been rejected :


By whom? By you, responding "REJECTED" instead of providing any useful
suggestions?

 
> - There was no advanced reflection on how to overhaul the subtitles
> API
>   and data structures.

I think everybody got it already that you are considering yourself
as a member of an elected circle with the exclusive ability of doing
"advanced reflection". 

> 
> - No new media type can be accepted into libavfilter before the
>   negotiation process have been cleaned up and extended to allow
>   negotiating the media type.

You are trying to play me and all others here:
(not in a really sophisticated way, though)

- I tried to understand your proposal from the pointers you gave
  and your linked presentation
- I responded with some questions I had
- You responded to my questions about it by insulting me personally
  instead, clearly without any intention of discussing details with
  me
- You are trying to build up a blocking wall that is supposed to be
  impossible to surpass


Now, I see the situation as follows:

Whatever changes you are proposing exactly - you didn't manage to 
get going with these changes for years and I see no reason why
I should wait for your ideas to get implemented first.

I'm submitting a patchset that is highly useful right now and
based on the CURRENT code base. 
I can't submit a patchset based on some proposed architectural 
changes that solely exist in some other developers mind.

Whatever your proposed changes are in detail. If you are a 
developer as great as you trying to let all of us know from
every second sentence you are typing, I'm sure it will be totally
easy for you to implement your ideas later on top of my patchset
whenever you're ready to do so. I hope it won't take another
few years like the project is already stuck on that subject for..

softworkz
Soft Works Sept. 3, 2021, 5:10 p.m. UTC | #7
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
> Moritz Barsnick
> Sent: Friday, 3 September 2021 10:24
> To: FFmpeg development discussions and patches <ffmpeg-
> devel@ffmpeg.org>
> Subject: Re: [FFmpeg-devel] [PATCH v2 7/8] avfilter/sleet: Add sleet
> filter
> 
> On Tue, Aug 31, 2021 at 22:49:12 +0000, Soft Works wrote:
> > As for the "sleet" filter: Yes - it is totally useless for
> practical
> > use. I had explained why I added it in the 0/8 message:
> >
> > > Why leet? Real world use is surely questionable, but it has two
> advantages
> > > that make it a great choice for testing:
> > > You can see from almost every single line whether it has been
> filtered,
> > > and the text is still readable which allows to verify that
> timings are
> > > still valid.


Hi Moritz,

thanks for looking into my patchset!

> 
> Okay, I get that case, though it seems a bit special. Isn't this just
> a
> debug feature for developers then?

Yes, correct. Or probably rather a demo and also (like you mentioned)
boilerplate code.

Finally, I'll also need to add Fate tests for this new capability and
that requires to have a simple filter as well.

> A review of this boilerplate would be great.
> 
> For the sake of review:
> - You need to add documentation to doc/filters.texi.

Yes, I'm already working on it.

> - You need to bump MINOR in libavfilter/version.h.

I think it even needs a MAJOR bump due to the change in 1/8 to 
AVFrame.

> 
> > Maybe an "sallcaps" filter makes slightly more sense than "sleet"?
> 
> For the sake of usability, I would rather have suggested something
> like
> "tr" (https://linux.die.net/man/1/tr,
> https://www.geeksforgeeks.org/perl-tr-operator/), where you can
> specify
> the replacement list. That said, tr has some funky shortcuts ("[a-z]
> [A-Z]" for uppercase, which may be harder to implement). Or actual
> regular expression support.
> 
> What I suggest is still not really useful for a user, though. Or is
> it?

I'm not sure - probably this would be ranking similarly to sleet and 
sallcaps ;-)

One kind of text filter that would be both, simple and useful would 
be something like "saddname".

With ASS subtitles, the name of the speaking person is often included
in the event data and sometimes subtitles can be confusing as to who
is actually speaking or it would be helpful to see character names
to better understand and follow.
So, what "saddname" would do is simply to change an event like:

338,0,Default,John,0000,0000,0000,,Hey, welcome!

to

338,0,Default,John,0000,0000,0000,,JOHN: Hey, welcome!


The reason why I did sleet instead is that it will always have a visible
effect while saddname would appear non-functional in case when 
character names are missing in the subtitle events.

Though, I don't have any feelings regarding sleet and I'd be totally
fine to drop it in the final version of the patchset.

Thanks,
softworkz
Soft Works Sept. 7, 2021, 7:31 a.m. UTC | #8
> -----Original Message-----
> From: ffmpeg-devel <ffmpeg-devel-bounces@ffmpeg.org> On Behalf Of
> Moritz Barsnick
> Sent: Friday, 3 September 2021 10:24
> To: FFmpeg development discussions and patches <ffmpeg-
> devel@ffmpeg.org>
> Subject: Re: [FFmpeg-devel] [PATCH v2 7/8] avfilter/sleet: Add sleet
> filter
> 
> On Tue, Aug 31, 2021 at 22:49:12 +0000, Soft Works wrote:

[..]

> 
> Well, at least it demonstrates the boilerplate required for a simple
> S->S filter, which touches only text (no timings or positioning or
> so).
> A review of this boilerplate would be great.
> 
> For the sake of review:
> - You need to add documentation to doc/filters.texi.
> - You need to bump MINOR in libavfilter/version.h.
> 
> > Maybe an "sallcaps" filter makes slightly more sense than "sleet"?
> 
> For the sake of usability, I would rather have suggested something
> like
> "tr" (https://linux.die.net/man/1/tr,
> https://www.geeksforgeeks.org/perl-tr-operator/), where you can
> specify
> the replacement list. That said, tr has some funky shortcuts ("[a-z]
> [A-Z]" for uppercase, which may be harder to implement). Or actual
> regular expression support.
> 
> What I suggest is still not really useful for a user, though. Or is
> it?

Hi Moritz,

I have changed the filter to a multi-purpose text modification filter
and renamed it to 'textmod'.

It has the following options:

"mode",             "set operation mode",            
  "leet",           "convert text to 'leet speak'",  
  "to_upper",       "change to upper case",          
  "to_lower",       "change to lower case",          
  "replace_chars",  "replace characters",            
  "remove_chars",   "remove characters",             
  "replace_words",  "replace words",                 
  "remove_words",   "remove words",                  
"find",             "chars/words to find or remove", 
"replace",          "chars/words to replace",        
"separator",        "word separator (default: ',')",

I think that makes it sufficiently useful.

Some plausible use case examples:

replace_words

Some movies (like Disney's Moana) are using different character 
names depending on country (Moana aka Vaiana aka Oceania, ..).
The character naming in subtitles might not always match what you're
expecting - you can use replace_words to fix it

remove_chars

Certain characters might be visually annoying for some (like the 
flipped exclamation and question marks in Spanish). With remove_chars,
these can be removed

to_upper

For improved readability

leet

Still highly useless (except for testing) ;-)


softworkz
diff mbox series

Patch

diff --git a/libavfilter/Makefile b/libavfilter/Makefile
index e38c6b6f6d..25dd1276de 100644
--- a/libavfilter/Makefile
+++ b/libavfilter/Makefile
@@ -526,6 +526,9 @@  OBJS-$(CONFIG_YUVTESTSRC_FILTER)             += vsrc_testsrc.o
 
 OBJS-$(CONFIG_NULLSINK_FILTER)               += vsink_nullsink.o
 
+# subtitle filters
+OBJS-$(CONFIG_SLEET_FILTER)                  += sf_sleet.o
+
 # multimedia filters
 OBJS-$(CONFIG_ABITSCOPE_FILTER)              += avf_abitscope.o
 OBJS-$(CONFIG_ADRAWGRAPH_FILTER)             += f_drawgraph.o
diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
index 5bd54db2c8..efe16b8e1b 100644
--- a/libavfilter/allfilters.c
+++ b/libavfilter/allfilters.c
@@ -519,6 +519,7 @@  extern const AVFilter ff_avf_showwaves;
 extern const AVFilter ff_avf_showwavespic;
 extern const AVFilter ff_vaf_spectrumsynth;
 extern const AVFilter ff_svf_sub2video;
+extern const AVFilter ff_sf_sleet;
 
 /* multimedia sources */
 extern const AVFilter ff_avsrc_amovie;
diff --git a/libavfilter/sf_sleet.c b/libavfilter/sf_sleet.c
new file mode 100644
index 0000000000..cf7701c01f
--- /dev/null
+++ b/libavfilter/sf_sleet.c
@@ -0,0 +1,209 @@ 
+/*
+ * Copyright (c) 2021 softworkz
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file
+ * text subtitle filter which translates to 'leet speak'
+ */
+
+#include "libavutil/avassert.h"
+#include "libavutil/avstring.h"
+#include "libavutil/opt.h"
+#include "avfilter.h"
+#include "internal.h"
+#include "libavcodec/avcodec.h"
+
+static const char* alphabet_src = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
+static const char* alphabet_dst = "abcd3f6#1jklmn0pq257uvwxyzAB(D3F6#1JKLMN0PQ257UVWXYZ";
+
+
+typedef struct LeetContext {
+    const AVClass *class;
+    enum AVSubtitleType format;
+} LeetContext;
+
+static const AVOption sleet_options[] = {
+    { NULL }
+};
+
+AVFILTER_DEFINE_CLASS(sleet);
+
+static int query_formats(AVFilterContext *ctx)
+{
+    AVFilterFormats *formats = NULL;
+    AVFilterLink *inlink = ctx->inputs[0];
+    AVFilterLink *outlink = ctx->outputs[0];
+    static const enum AVSubtitleType subtitle_fmts[] = { SUBTITLE_ASS, SUBTITLE_NONE };
+    static const enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_RGB32, AV_PIX_FMT_NONE };
+    int ret;
+
+    /* set input subtitle format */
+    formats = ff_make_format_list(subtitle_fmts);
+    if ((ret = ff_formats_ref(formats, &inlink->outcfg.formats)) < 0)
+        return ret;
+
+    /* set output video format */
+    formats = ff_make_format_list(pix_fmts);
+    if ((ret = ff_formats_ref(formats, &outlink->incfg.formats)) < 0)
+        return ret;
+
+    return 0;
+}
+
+static int config_input(AVFilterLink *inlink)
+{
+    AVFilterContext *ctx = inlink->dst;
+    LeetContext *s = ctx->priv;
+
+    s->format = inlink->format;
+    return 0;
+}
+
+static int config_output(AVFilterLink *outlink)
+{
+    LeetContext *s = outlink->src->priv;
+
+    outlink->format = s->format;
+
+    return 0;
+}
+
+static void avsubtitle_free_ref(void *opaque, uint8_t *data)
+{
+    avsubtitle_free((AVSubtitle *)data);
+}
+
+static int filter_frame(AVFilterLink *inlink, AVFrame *src_frame)
+{
+    LeetContext *s = inlink->dst->priv;
+    AVFilterLink *outlink = inlink->dst->outputs[0];
+    AVSubtitle *sub;
+    int ret;
+    AVFrame *out;
+    unsigned int num_rects;
+    uint8_t *dst;
+
+    outlink->format = inlink->format;
+
+    out = av_frame_alloc();
+    if (!out) {
+        av_frame_free(&src_frame);
+        return AVERROR(ENOMEM);
+    }
+
+    out->format = outlink->format;
+
+    if ((ret = av_frame_get_buffer2(out, AVMEDIA_TYPE_SUBTITLE, 0)) < 0)
+        return ret;
+
+    out->pts                    = src_frame->pts;
+    out->repeat_pict            = src_frame->repeat_pict;
+    out->pkt_dts                = src_frame->pkt_dts;
+    out->pkt_pos                = src_frame->pkt_pos;
+    out->pkt_size               = src_frame->pkt_size;
+    out->pkt_duration           = src_frame->pkt_duration;
+    out->reordered_opaque       = src_frame->reordered_opaque;
+    out->best_effort_timestamp  = src_frame->best_effort_timestamp;
+    out->flags                  = src_frame->flags;
+
+    sub = (AVSubtitle *)src_frame->data[0];
+
+    if (sub) {
+        AVSubtitle *out_sub = av_memdup(sub, sizeof(*out_sub));
+        if (!out_sub)
+            return AVERROR(ENOMEM);
+
+        out->buf[0] = av_buffer_create((uint8_t*)out_sub, sizeof(*out_sub), avsubtitle_free_ref, NULL, AV_BUFFER_FLAG_READONLY);
+        out->data[0] = (uint8_t*)out_sub;
+
+        if (sub->num_rects) {
+            out_sub->rects = av_malloc_array(sub->num_rects, sizeof(AVSubtitleRect *));
+        }
+
+        for (unsigned i = 0; i < sub->num_rects; i++) {
+
+            AVSubtitleRect *src_rect = sub->rects[i];
+            AVSubtitleRect *dst_rect = av_memdup(src_rect, sizeof(*dst_rect));
+            out_sub->rects[i] = dst_rect;
+
+            if (src_rect->text) {
+                dst_rect->text = av_strdup(src_rect->text);
+                if (!dst_rect->text)
+                    return AVERROR(ENOMEM);
+
+                for (size_t n = 0; n < strlen(dst_rect->text); n++) {
+                    for (size_t t = 0; t < FF_ARRAY_ELEMS(alphabet_src); t++) {
+                        if (dst_rect->text[n] == alphabet_src[t]) {
+                            dst_rect->text[n] = alphabet_dst[t];
+                            break;
+                        }
+                    }
+                }
+            }
+
+            if (src_rect->ass) {
+                dst_rect->ass = av_strdup(src_rect->ass);
+                if (!dst_rect->ass)
+                    return AVERROR(ENOMEM);
+
+                for (size_t n = 0; n < strlen(dst_rect->ass); n++) {
+                    for (size_t t = 0; t < FF_ARRAY_ELEMS(alphabet_src); t++) {
+                        if (dst_rect->ass[n] == alphabet_src[t]) {
+                            dst_rect->ass[n] = alphabet_dst[t];
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    av_frame_free(&src_frame);
+    return ff_filter_frame(outlink, out);
+}
+
+static const AVFilterPad sleet_inputs[] = {
+    {
+        .name         = "default",
+        .type         = AVMEDIA_TYPE_SUBTITLE,
+        .filter_frame = filter_frame,
+        .config_props = config_input,
+    },
+    { NULL }
+};
+
+static const AVFilterPad sleet_outputs[] = {
+    {
+        .name          = "default",
+        .type          = AVMEDIA_TYPE_SUBTITLE,
+        .config_props  = config_output,
+    },
+    { NULL }
+};
+
+const AVFilter ff_sf_sleet = {
+    .name          = "sleet",
+    .description   = NULL_IF_CONFIG_SMALL("Translate text subtitles to 'leet speak'"),
+    .query_formats = query_formats,
+    .priv_size     = sizeof(LeetContext),
+    .priv_class    = &sleet_class,
+    .inputs        = sleet_inputs,
+    .outputs       = sleet_outputs,
+};