| Message ID | 20190712092634.22755-1-onemda@gmail.com |
|---|---|
| State | New |
| Headers | show |
On Fri, 12 Jul 2019, Paul B Mahol wrote: > From: Vladimir Panteleev <git@thecybershadow.net> I guess you should mention ticket #2104. Regards, Marton > > Signed-off-by: Paul B Mahol <onemda@gmail.com> > --- > doc/filters.texi | 20 ++ > libavfilter/Makefile | 1 + > libavfilter/allfilters.c | 1 + > libavfilter/vf_photosensitivity.c | 336 ++++++++++++++++++++++++++++++ > 4 files changed, 358 insertions(+) > create mode 100644 libavfilter/vf_photosensitivity.c > > diff --git a/doc/filters.texi b/doc/filters.texi > index c92ed8de07..39ee2cb001 100644 > --- a/doc/filters.texi > +++ b/doc/filters.texi > @@ -13938,6 +13938,26 @@ Filter selects among @samp{t}, @samp{b} and @samp{p} using image analysis only. > @end table > @end table > > +@section photosensitivity > +Reduce various flashes in video, so to help users with epilepsy. > + > +It accepts the following options: > +@table @option > +@item frames, f > +Set how many frames to use when filtering. Default is 30. > + > +@item threshold, t > +Set detection threshold factor. Default is 1. > +Lower is stricter. > + > +@item skip > +Set how many pixels to skip when sampling frames. Defalt is 1. > +Allowed range is from 1 to 1024. > + > +@item bypass > +Leave frames unchanged. Default is disabled. > +@end table > + > @section pixdesctest > > Pixel format descriptor test filter, mainly useful for internal > diff --git a/libavfilter/Makefile b/libavfilter/Makefile > index 455c809b15..6a3521e813 100644 > --- a/libavfilter/Makefile > +++ b/libavfilter/Makefile > @@ -319,6 +319,7 @@ OBJS-$(CONFIG_PALETTEUSE_FILTER) += vf_paletteuse.o framesync.o > OBJS-$(CONFIG_PERMS_FILTER) += f_perms.o > OBJS-$(CONFIG_PERSPECTIVE_FILTER) += vf_perspective.o > OBJS-$(CONFIG_PHASE_FILTER) += vf_phase.o > +OBJS-$(CONFIG_PHOTOSENSITIVITY_FILTER) += vf_photosensitivity.o > OBJS-$(CONFIG_PIXDESCTEST_FILTER) += vf_pixdesctest.o > OBJS-$(CONFIG_PIXSCOPE_FILTER) += vf_datascope.o > OBJS-$(CONFIG_PP_FILTER) += vf_pp.o > diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c > index 04a3df7d56..0a05030a01 100644 > --- a/libavfilter/allfilters.c > +++ b/libavfilter/allfilters.c > @@ -303,6 +303,7 @@ extern AVFilter ff_vf_paletteuse; > extern AVFilter ff_vf_perms; > extern AVFilter ff_vf_perspective; > extern AVFilter ff_vf_phase; > +extern AVFilter ff_vf_photosensitivity; > extern AVFilter ff_vf_pixdesctest; > extern AVFilter ff_vf_pixscope; > extern AVFilter ff_vf_pp; > diff --git a/libavfilter/vf_photosensitivity.c b/libavfilter/vf_photosensitivity.c > new file mode 100644 > index 0000000000..6805e92712 > --- /dev/null > +++ b/libavfilter/vf_photosensitivity.c > @@ -0,0 +1,336 @@ > +/* > + * Copyright (c) 2019 Vladimir Panteleev > + * > + * This file is part of FFmpeg. > + * > + * FFmpeg is free software; you can redistribute it and/or > + * modify it under the terms of the GNU Lesser General Public > + * License as published by the Free Software Foundation; either > + * version 2.1 of the License, or (at your option) any later version. > + * > + * FFmpeg is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + * Lesser General Public License for more details. > + * > + * You should have received a copy of the GNU Lesser General Public > + * License along with FFmpeg; if not, write to the Free Software > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > + */ > + > +#include <float.h> > + > +#include "libavutil/imgutils.h" > +#include "libavutil/opt.h" > +#include "libavutil/pixdesc.h" > +#include "avfilter.h" > + > +#include "formats.h" > +#include "internal.h" > +#include "video.h" > + > +#define MAX_FRAMES 240 > +#define GRID_SIZE 8 > +#define NUM_CHANNELS 3 > + > +typedef struct PhotosensitivityFrame { > + uint8_t grid[GRID_SIZE][GRID_SIZE][4]; > +} PhotosensitivityFrame; > + > +typedef struct PhotosensitivityContext { > + const AVClass *class; > + > + int nb_frames; > + int skip; > + float threshold_multiplier; > + int bypass; > + > + int badness_threshold; > + > + /* Circular buffer */ > + int history[MAX_FRAMES]; > + int history_pos; > + > + PhotosensitivityFrame last_frame_e; > + AVFrame *last_frame_av; > +} PhotosensitivityContext; > + > +#define OFFSET(x) offsetof(PhotosensitivityContext, x) > +#define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM > + > +static const AVOption photosensitivity_options[] = { > + { "frames", "set how many frames to use" , OFFSET(nb_frames ), AV_OPT_TYPE_INT , {.i64=30}, 2, MAX_FRAMES, FLAGS }, > + { "f", "set how many frames to use" , OFFSET(nb_frames ), AV_OPT_TYPE_INT , {.i64=30}, 2, MAX_FRAMES, FLAGS }, > + { "threshold", "set detection threshold factor (lower is stricter)", OFFSET(threshold_multiplier), AV_OPT_TYPE_FLOAT, {.dbl= 1}, 0.1, FLT_MAX, FLAGS }, > + { "t" , "set detection threshold factor (lower is stricter)", OFFSET(threshold_multiplier), AV_OPT_TYPE_FLOAT, {.dbl= 1}, 0.1, FLT_MAX, FLAGS }, > + { "skip" , "set pixels to skip when sampling frames" , OFFSET(skip ), AV_OPT_TYPE_INT , {.i64= 1}, 1, 1024 , FLAGS }, > + { "bypass" , "leave frames unchanged" , OFFSET(bypass ), AV_OPT_TYPE_BOOL , {.i64= 0}, 0, 1 , FLAGS }, > + { NULL } > +}; > + > +AVFILTER_DEFINE_CLASS(photosensitivity); > + > +static int query_formats(AVFilterContext *ctx) > +{ > + static const enum AVPixelFormat pixel_fmts[] = { > + AV_PIX_FMT_RGB24, > + AV_PIX_FMT_BGR24, > + AV_PIX_FMT_NONE > + }; > + AVFilterFormats *formats = ff_make_format_list(pixel_fmts); > + if (!formats) > + return AVERROR(ENOMEM); > + return ff_set_common_formats(ctx, formats); > +} > + > +typedef struct ThreadData_convert_frame > +{ > + AVFrame *in; > + PhotosensitivityFrame* out; > + int skip; > +} ThreadData_convert_frame; > + > +#define NUM_CELLS (GRID_SIZE * GRID_SIZE) > + > +static int convert_frame_partial(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs) > +{ > + int cell, gx, gy, x0, x1, y0, y1, x, y, c, area; > + int sum[NUM_CHANNELS]; > + const uint8_t *p; > + > + ThreadData_convert_frame *td = arg; > + > + const int slice_start = (NUM_CELLS * jobnr) / nb_jobs; > + const int slice_end = (NUM_CELLS * (jobnr+1)) / nb_jobs; > + > + int width = td->in->width, height = td->in->height, linesize = td->in->linesize[0], skip = td->skip; > + const uint8_t *data = td->in->data[0]; > + > + for (cell = slice_start; cell < slice_end; cell++) { > + gx = cell % GRID_SIZE; > + gy = cell / GRID_SIZE; > + > + x0 = width * gx / GRID_SIZE; > + x1 = width * (gx+1) / GRID_SIZE; > + y0 = height * gy / GRID_SIZE; > + y1 = height * (gy+1) / GRID_SIZE; > + > + for (c = 0; c < NUM_CHANNELS; c++) { > + sum[c] = 0; > + } > + for (y = y0; y < y1; y += skip) { > + p = data + y * linesize + x0 * NUM_CHANNELS; > + for (x = x0; x < x1; x += skip) { > + //av_log(NULL, AV_LOG_VERBOSE, "%d %d %d : (%d,%d) (%d,%d) -> %d,%d | *%d\n", c, gx, gy, x0, y0, x1, y1, x, y, (int)row); > + sum[0] += p[0]; > + sum[1] += p[1]; > + sum[2] += p[2]; > + p += NUM_CHANNELS * skip; > + // TODO: variable size > + } > + } > + > + area = ((x1 - x0 + skip - 1) / skip) * ((y1 - y0 + skip - 1) / skip); > + for (c = 0; c < NUM_CHANNELS; c++) { > + if (area) > + sum[c] /= area; > + td->out->grid[gy][gx][c] = sum[c]; > + } > + } > + return 0; > +} > + > +static void convert_frame(AVFilterContext *ctx, AVFrame *in, PhotosensitivityFrame* out, int skip) > +{ > + ThreadData_convert_frame td; > + td.in = in; > + td.out = out; > + td.skip = skip; > + ctx->internal->execute(ctx, convert_frame_partial, &td, NULL, FFMIN(NUM_CELLS, ff_filter_get_nb_threads(ctx))); > +} > + > +typedef struct ThreadData_blend_frame > +{ > + AVFrame *target; > + AVFrame *source; > + uint16_t s_mul; > +} ThreadData_blend_frame; > + > +static int blend_frame_partial(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs) > +{ > + int x, y; > + uint8_t *t, *s; > + > + ThreadData_blend_frame *td = arg; > + const uint16_t s_mul = td->s_mul; > + const uint16_t t_mul = 0x100 - s_mul; > + const int slice_start = (td->target->height * jobnr) / nb_jobs; > + const int slice_end = (td->target->height * (jobnr+1)) / nb_jobs; > + const int linesize = td->target->linesize[0]; > + > + for (y = slice_start; y < slice_end; y++) { > + t = td->target->data[0] + y * td->target->linesize[0]; > + s = td->source->data[0] + y * td->source->linesize[0]; > + for (x = 0; x < linesize; x++) { > + *t = (*t * t_mul + *s * s_mul) >> 8; > + t++; s++; > + } > + } > + return 0; > +} > + > +static void blend_frame(AVFilterContext *ctx, AVFrame *target, AVFrame *source, float factor) > +{ > + ThreadData_blend_frame td; > + td.target = target; > + td.source = source; > + td.s_mul = (uint16_t)(factor * 0x100); > + ctx->internal->execute(ctx, blend_frame_partial, &td, NULL, FFMIN(ctx->outputs[0]->h, ff_filter_get_nb_threads(ctx))); > +} > + > +static int get_badness(PhotosensitivityFrame* a, PhotosensitivityFrame* b) > +{ > + int badness, x, y, c; > + badness = 0; > + for (c = 0; c < NUM_CHANNELS; c++) { > + for (y = 0; y < GRID_SIZE; y++) { > + for (x = 0; x < GRID_SIZE; x++) { > + badness += abs((int)a->grid[y][x][c] - (int)b->grid[y][x][c]); > + //av_log(NULL, AV_LOG_VERBOSE, "%d - %d -> %d \n", a->grid[y][x], b->grid[y][x], badness); > + //av_log(NULL, AV_LOG_VERBOSE, "%d -> %d \n", abs((int)a->grid[y][x] - (int)b->grid[y][x]), badness); > + } > + } > + } > + return badness; > +} > + > +static int config_input(AVFilterLink *inlink) > +{ > + /* const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format); */ > + AVFilterContext *ctx = inlink->dst; > + PhotosensitivityContext *s = ctx->priv; > + > + s->badness_threshold = (int)(GRID_SIZE * GRID_SIZE * 4 * 256 * s->nb_frames * s->threshold_multiplier / 128); > + > + return 0; > +} > + > +static int filter_frame(AVFilterLink *inlink, AVFrame *in) > +{ > + int this_badness, current_badness, fixed_badness, new_badness, i, res; > + PhotosensitivityFrame ef; > + AVFrame *src, *out; > + float factor; > + AVDictionary **metadata; > + > + AVFilterContext *ctx = inlink->dst; > + AVFilterLink *outlink = ctx->outputs[0]; > + PhotosensitivityContext *s = ctx->priv; > + > + /* weighted moving average */ > + current_badness = 0; > + for (i = 1; i < s->nb_frames; i++) > + current_badness += i * s->history[(s->history_pos + i) % s->nb_frames]; > + current_badness /= s->nb_frames; > + > + convert_frame(ctx, in, &ef, s->skip); > + this_badness = get_badness(&ef, &s->last_frame_e); > + new_badness = current_badness + this_badness; > + av_log(s, AV_LOG_VERBOSE, "badness: %6d -> %6d / %6d (%3d%% - %s) \n", > + current_badness, new_badness, s->badness_threshold, > + 100 * new_badness / s->badness_threshold, new_badness < s->badness_threshold ? "OK" : "EXCEEDED"); > + > + fixed_badness = new_badness; > + if (new_badness < s->badness_threshold || !s->last_frame_av || s->bypass) { > + factor = 1; /* for metadata */ > + av_frame_free(&s->last_frame_av); > + s->last_frame_av = src = in; > + s->last_frame_e = ef; > + s->history[s->history_pos] = this_badness; > + } else { > + factor = (float)(s->badness_threshold - current_badness) / (new_badness - current_badness); > + if (factor <= 0) { > + /* just duplicate the frame */ > + s->history[s->history_pos] = 0; /* frame was duplicated, thus, delta is zero */ > + } else { > + res = av_frame_make_writable(s->last_frame_av); > + if (res) { > + av_frame_free(&in); > + return res; > + } > + blend_frame(ctx, s->last_frame_av, in, factor); > + > + convert_frame(ctx, s->last_frame_av, &ef, s->skip); > + this_badness = get_badness(&ef, &s->last_frame_e); > + fixed_badness = current_badness + this_badness; > + av_log(s, AV_LOG_VERBOSE, " fixed: %6d -> %6d / %6d (%3d%%) factor=%5.3f \n", > + current_badness, fixed_badness, s->badness_threshold, > + 100 * new_badness / s->badness_threshold, factor); > + s->last_frame_e = ef; > + s->history[s->history_pos] = this_badness; > + } > + src = s->last_frame_av; > + } > + s->history_pos = (s->history_pos + 1) % s->nb_frames; > + > + out = ff_get_video_buffer(outlink, in->width, in->height); > + if (!out) { > + av_frame_free(&in); > + return AVERROR(ENOMEM); > + } > + av_frame_copy_props(out, in); > + metadata = &out->metadata; > + if (metadata) { > + char value[128]; > + > + snprintf(value, sizeof(value), "%f", (float)new_badness / s->badness_threshold); > + av_dict_set(metadata, "lavfi.photosensitivity.badness", value, 0); > + > + snprintf(value, sizeof(value), "%f", (float)fixed_badness / s->badness_threshold); > + av_dict_set(metadata, "lavfi.photosensitivity.fixed-badness", value, 0); > + > + snprintf(value, sizeof(value), "%f", (float)this_badness / s->badness_threshold); > + av_dict_set(metadata, "lavfi.photosensitivity.frame-badness", value, 0); > + > + snprintf(value, sizeof(value), "%f", factor); > + av_dict_set(metadata, "lavfi.photosensitivity.factor", value, 0); > + } > + av_frame_copy(out, src); > + return ff_filter_frame(outlink, out); > +} > + > +static av_cold void uninit(AVFilterContext *ctx) > +{ > + PhotosensitivityContext *s = ctx->priv; > + > + av_frame_free(&s->last_frame_av); > +} > + > +static const AVFilterPad inputs[] = { > + { > + .name = "default", > + .type = AVMEDIA_TYPE_VIDEO, > + .filter_frame = filter_frame, > + .config_props = config_input, > + }, > + { NULL } > +}; > + > +static const AVFilterPad outputs[] = { > + { > + .name = "default", > + .type = AVMEDIA_TYPE_VIDEO, > + }, > + { NULL } > +}; > + > +AVFilter ff_vf_photosensitivity = { > + .name = "photosensitivity", > + .description = NULL_IF_CONFIG_SMALL("Attempt to filter out photosensitive epilepsy seizure-inducing flashes."), > + .priv_size = sizeof(PhotosensitivityContext), > + .priv_class = &photosensitivity_class, > + .uninit = uninit, > + .query_formats = query_formats, > + .inputs = inputs, > + .outputs = outputs, > +}; > -- > 2.17.1 > > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
On Fri, Jul 12, 2019 at 11:26:34AM +0200, Paul B Mahol wrote: > From: Vladimir Panteleev <git@thecybershadow.net> > > Signed-off-by: Paul B Mahol <onemda@gmail.com> > --- > doc/filters.texi | 20 ++ > libavfilter/Makefile | 1 + > libavfilter/allfilters.c | 1 + > libavfilter/vf_photosensitivity.c | 336 ++++++++++++++++++++++++++++++ > 4 files changed, 358 insertions(+) > create mode 100644 libavfilter/vf_photosensitivity.c > > diff --git a/doc/filters.texi b/doc/filters.texi > index c92ed8de07..39ee2cb001 100644 > --- a/doc/filters.texi > +++ b/doc/filters.texi > @@ -13938,6 +13938,26 @@ Filter selects among @samp{t}, @samp{b} and @samp{p} using image analysis only. > @end table > @end table > > +@section photosensitivity > +Reduce various flashes in video, so to help users with epilepsy. This should clearly specify if this filter is known to work reliably, untested, experimental or what exactly. Consider someone might end up injuring themselfs if this doesnt work for them. > + > +It accepts the following options: > +@table @option > +@item frames, f > +Set how many frames to use when filtering. Default is 30. > + > +@item threshold, t > +Set detection threshold factor. Default is 1. > +Lower is stricter. > + > +@item skip > +Set how many pixels to skip when sampling frames. Defalt is 1. > +Allowed range is from 1 to 1024. > + > +@item bypass > +Leave frames unchanged. Default is disabled. > +@end table > + > @section pixdesctest > > Pixel format descriptor test filter, mainly useful for internal > diff --git a/libavfilter/Makefile b/libavfilter/Makefile > index 455c809b15..6a3521e813 100644 > --- a/libavfilter/Makefile > +++ b/libavfilter/Makefile > @@ -319,6 +319,7 @@ OBJS-$(CONFIG_PALETTEUSE_FILTER) += vf_paletteuse.o framesync.o > OBJS-$(CONFIG_PERMS_FILTER) += f_perms.o > OBJS-$(CONFIG_PERSPECTIVE_FILTER) += vf_perspective.o > OBJS-$(CONFIG_PHASE_FILTER) += vf_phase.o > +OBJS-$(CONFIG_PHOTOSENSITIVITY_FILTER) += vf_photosensitivity.o > OBJS-$(CONFIG_PIXDESCTEST_FILTER) += vf_pixdesctest.o > OBJS-$(CONFIG_PIXSCOPE_FILTER) += vf_datascope.o > OBJS-$(CONFIG_PP_FILTER) += vf_pp.o > diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c > index 04a3df7d56..0a05030a01 100644 > --- a/libavfilter/allfilters.c > +++ b/libavfilter/allfilters.c > @@ -303,6 +303,7 @@ extern AVFilter ff_vf_paletteuse; > extern AVFilter ff_vf_perms; > extern AVFilter ff_vf_perspective; > extern AVFilter ff_vf_phase; > +extern AVFilter ff_vf_photosensitivity; > extern AVFilter ff_vf_pixdesctest; > extern AVFilter ff_vf_pixscope; > extern AVFilter ff_vf_pp; > diff --git a/libavfilter/vf_photosensitivity.c b/libavfilter/vf_photosensitivity.c > new file mode 100644 > index 0000000000..6805e92712 > --- /dev/null > +++ b/libavfilter/vf_photosensitivity.c > @@ -0,0 +1,336 @@ > +/* > + * Copyright (c) 2019 Vladimir Panteleev > + * > + * This file is part of FFmpeg. > + * > + * FFmpeg is free software; you can redistribute it and/or > + * modify it under the terms of the GNU Lesser General Public > + * License as published by the Free Software Foundation; either > + * version 2.1 of the License, or (at your option) any later version. > + * > + * FFmpeg is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + * Lesser General Public License for more details. > + * > + * You should have received a copy of the GNU Lesser General Public > + * License along with FFmpeg; if not, write to the Free Software > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > + */ > + > +#include <float.h> > + > +#include "libavutil/imgutils.h" > +#include "libavutil/opt.h" > +#include "libavutil/pixdesc.h" > +#include "avfilter.h" > + > +#include "formats.h" > +#include "internal.h" > +#include "video.h" > + > +#define MAX_FRAMES 240 > +#define GRID_SIZE 8 > +#define NUM_CHANNELS 3 > + > +typedef struct PhotosensitivityFrame { > + uint8_t grid[GRID_SIZE][GRID_SIZE][4]; > +} PhotosensitivityFrame; > + > +typedef struct PhotosensitivityContext { > + const AVClass *class; > + > + int nb_frames; > + int skip; > + float threshold_multiplier; > + int bypass; > + > + int badness_threshold; > + > + /* Circular buffer */ > + int history[MAX_FRAMES]; > + int history_pos; > + > + PhotosensitivityFrame last_frame_e; > + AVFrame *last_frame_av; > +} PhotosensitivityContext; > + > +#define OFFSET(x) offsetof(PhotosensitivityContext, x) > +#define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM > + > +static const AVOption photosensitivity_options[] = { > + { "frames", "set how many frames to use" , OFFSET(nb_frames ), AV_OPT_TYPE_INT , {.i64=30}, 2, MAX_FRAMES, FLAGS }, > + { "f", "set how many frames to use" , OFFSET(nb_frames ), AV_OPT_TYPE_INT , {.i64=30}, 2, MAX_FRAMES, FLAGS }, > + { "threshold", "set detection threshold factor (lower is stricter)", OFFSET(threshold_multiplier), AV_OPT_TYPE_FLOAT, {.dbl= 1}, 0.1, FLT_MAX, FLAGS }, > + { "t" , "set detection threshold factor (lower is stricter)", OFFSET(threshold_multiplier), AV_OPT_TYPE_FLOAT, {.dbl= 1}, 0.1, FLT_MAX, FLAGS }, > + { "skip" , "set pixels to skip when sampling frames" , OFFSET(skip ), AV_OPT_TYPE_INT , {.i64= 1}, 1, 1024 , FLAGS }, > + { "bypass" , "leave frames unchanged" , OFFSET(bypass ), AV_OPT_TYPE_BOOL , {.i64= 0}, 0, 1 , FLAGS }, > + { NULL } > +}; > + > +AVFILTER_DEFINE_CLASS(photosensitivity); > + > +static int query_formats(AVFilterContext *ctx) > +{ > + static const enum AVPixelFormat pixel_fmts[] = { > + AV_PIX_FMT_RGB24, > + AV_PIX_FMT_BGR24, > + AV_PIX_FMT_NONE > + }; as most videos are in YUV formats and as this filter only analyses a subset of pixels it would be good not to have to convert all to RGB but only the pixels needed. Ill refrain from suggesting changes to what this filter does exactly as i do not know enough about the subject and have no reasonable way to test this. Thanks [...]
On Fri, Jul 12, 2019 at 11:26:34AM +0200, Paul B Mahol wrote: > From: Vladimir Panteleev <git@thecybershadow.net> > > Signed-off-by: Paul B Mahol <onemda@gmail.com> > --- > doc/filters.texi | 20 ++ > libavfilter/Makefile | 1 + > libavfilter/allfilters.c | 1 + > libavfilter/vf_photosensitivity.c | 336 ++++++++++++++++++++++++++++++ > 4 files changed, 358 insertions(+) > create mode 100644 libavfilter/vf_photosensitivity.c I support this filter or some variant of it to be applied some area of work is that it seems to work less well at higher framerates for example: -vf fps=120,photosensitivity Denno.ogg or 240fps Thanks [...]
On 7/13/19, Michael Niedermayer <michael@niedermayer.cc> wrote: > On Fri, Jul 12, 2019 at 11:26:34AM +0200, Paul B Mahol wrote: >> From: Vladimir Panteleev <git@thecybershadow.net> >> >> Signed-off-by: Paul B Mahol <onemda@gmail.com> >> --- >> doc/filters.texi | 20 ++ >> libavfilter/Makefile | 1 + >> libavfilter/allfilters.c | 1 + >> libavfilter/vf_photosensitivity.c | 336 ++++++++++++++++++++++++++++++ >> 4 files changed, 358 insertions(+) >> create mode 100644 libavfilter/vf_photosensitivity.c > > I support this filter or some variant of it to be applied > > some area of work is that it seems to work less well at higher > framerates > for example: > -vf fps=120,photosensitivity Denno.ogg > or 240fps > Please refrain from commenting for stuff you know very little or nothing. Thanks. Your comments are not valuable at all.
On Sat, Jul 13, 2019 at 09:32:38AM +0200, Paul B Mahol wrote: > On 7/13/19, Michael Niedermayer <michael@niedermayer.cc> wrote: > > On Fri, Jul 12, 2019 at 11:26:34AM +0200, Paul B Mahol wrote: > >> From: Vladimir Panteleev <git@thecybershadow.net> > >> > >> Signed-off-by: Paul B Mahol <onemda@gmail.com> > >> --- > >> doc/filters.texi | 20 ++ > >> libavfilter/Makefile | 1 + > >> libavfilter/allfilters.c | 1 + > >> libavfilter/vf_photosensitivity.c | 336 ++++++++++++++++++++++++++++++ > >> 4 files changed, 358 insertions(+) > >> create mode 100644 libavfilter/vf_photosensitivity.c > > > > I support this filter or some variant of it to be applied > > > > some area of work is that it seems to work less well at higher > > framerates > > for example: > > -vf fps=120,photosensitivity Denno.ogg > > or 240fps > > > > Please refrain from commenting for stuff you know very little or nothing. > Thanks. > > Your comments are not valuable at all. i normally just ignore your insults but here i think i should not if the same video at 120fps fails to be filtered as well as it is at 30fps there is a problem. Because again its the same data, its not played faster it just internally has more frames. A viewer seeing a frame for 33ms cannot know if thats 1 frame at 30fps or 4 times the same frame at 120fps. So if one is filtered stronger than the other they cannot both be optimal About understanding, i looked at the paper referenced by compn, and the code of the filter. That is not much indeed, but i doubt the other people here have better knowledge about the subject. If some expert is around here who knows this stuff. His/her oppinion and input would no doubt be very valuable Thanks [...]
On 7/13/19, Michael Niedermayer <michael@niedermayer.cc> wrote: > On Sat, Jul 13, 2019 at 09:32:38AM +0200, Paul B Mahol wrote: >> On 7/13/19, Michael Niedermayer <michael@niedermayer.cc> wrote: >> > On Fri, Jul 12, 2019 at 11:26:34AM +0200, Paul B Mahol wrote: >> >> From: Vladimir Panteleev <git@thecybershadow.net> >> >> >> >> Signed-off-by: Paul B Mahol <onemda@gmail.com> >> >> --- >> >> doc/filters.texi | 20 ++ >> >> libavfilter/Makefile | 1 + >> >> libavfilter/allfilters.c | 1 + >> >> libavfilter/vf_photosensitivity.c | 336 >> >> ++++++++++++++++++++++++++++++ >> >> 4 files changed, 358 insertions(+) >> >> create mode 100644 libavfilter/vf_photosensitivity.c >> > >> > I support this filter or some variant of it to be applied >> > >> > some area of work is that it seems to work less well at higher >> > framerates >> > for example: >> > -vf fps=120,photosensitivity Denno.ogg >> > or 240fps >> > >> >> Please refrain from commenting for stuff you know very little or nothing. >> Thanks. >> >> Your comments are not valuable at all. > > i normally just ignore your insults but here i think i should not > > if the same video at 120fps fails to be filtered as well as it is at 30fps > there is a problem. Because again its the same data, its not played faster > it just internally has more frames. > A viewer seeing a frame for 33ms cannot know if thats 1 frame at 30fps > or 4 times the same frame at 120fps. > So if one is filtered stronger than the other they cannot both be optimal > > About understanding, i looked at the paper referenced by compn, and the > code > of the filter. That is not much indeed, but i doubt the other people here > have > better knowledge about the subject. If some expert is around here who knows > this stuff. His/her oppinion and input would no doubt be very valuable I'm expert to this very subject. I'm stating that filter is good at reducing/removing abrupt changes in color/brightness for typical videos. On another hand, you are pointing to irrelevant corner cases. Besides user can increase number of frames used in calculation of changes. What I will not ever tolerate is your belittling other peoples work.
diff --git a/doc/filters.texi b/doc/filters.texi index c92ed8de07..39ee2cb001 100644 --- a/doc/filters.texi +++ b/doc/filters.texi @@ -13938,6 +13938,26 @@ Filter selects among @samp{t}, @samp{b} and @samp{p} using image analysis only. @end table @end table +@section photosensitivity +Reduce various flashes in video, so to help users with epilepsy. + +It accepts the following options: +@table @option +@item frames, f +Set how many frames to use when filtering. Default is 30. + +@item threshold, t +Set detection threshold factor. Default is 1. +Lower is stricter. + +@item skip +Set how many pixels to skip when sampling frames. Defalt is 1. +Allowed range is from 1 to 1024. + +@item bypass +Leave frames unchanged. Default is disabled. +@end table + @section pixdesctest Pixel format descriptor test filter, mainly useful for internal diff --git a/libavfilter/Makefile b/libavfilter/Makefile index 455c809b15..6a3521e813 100644 --- a/libavfilter/Makefile +++ b/libavfilter/Makefile @@ -319,6 +319,7 @@ OBJS-$(CONFIG_PALETTEUSE_FILTER) += vf_paletteuse.o framesync.o OBJS-$(CONFIG_PERMS_FILTER) += f_perms.o OBJS-$(CONFIG_PERSPECTIVE_FILTER) += vf_perspective.o OBJS-$(CONFIG_PHASE_FILTER) += vf_phase.o +OBJS-$(CONFIG_PHOTOSENSITIVITY_FILTER) += vf_photosensitivity.o OBJS-$(CONFIG_PIXDESCTEST_FILTER) += vf_pixdesctest.o OBJS-$(CONFIG_PIXSCOPE_FILTER) += vf_datascope.o OBJS-$(CONFIG_PP_FILTER) += vf_pp.o diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c index 04a3df7d56..0a05030a01 100644 --- a/libavfilter/allfilters.c +++ b/libavfilter/allfilters.c @@ -303,6 +303,7 @@ extern AVFilter ff_vf_paletteuse; extern AVFilter ff_vf_perms; extern AVFilter ff_vf_perspective; extern AVFilter ff_vf_phase; +extern AVFilter ff_vf_photosensitivity; extern AVFilter ff_vf_pixdesctest; extern AVFilter ff_vf_pixscope; extern AVFilter ff_vf_pp; diff --git a/libavfilter/vf_photosensitivity.c b/libavfilter/vf_photosensitivity.c new file mode 100644 index 0000000000..6805e92712 --- /dev/null +++ b/libavfilter/vf_photosensitivity.c @@ -0,0 +1,336 @@ +/* + * Copyright (c) 2019 Vladimir Panteleev + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <float.h> + +#include "libavutil/imgutils.h" +#include "libavutil/opt.h" +#include "libavutil/pixdesc.h" +#include "avfilter.h" + +#include "formats.h" +#include "internal.h" +#include "video.h" + +#define MAX_FRAMES 240 +#define GRID_SIZE 8 +#define NUM_CHANNELS 3 + +typedef struct PhotosensitivityFrame { + uint8_t grid[GRID_SIZE][GRID_SIZE][4]; +} PhotosensitivityFrame; + +typedef struct PhotosensitivityContext { + const AVClass *class; + + int nb_frames; + int skip; + float threshold_multiplier; + int bypass; + + int badness_threshold; + + /* Circular buffer */ + int history[MAX_FRAMES]; + int history_pos; + + PhotosensitivityFrame last_frame_e; + AVFrame *last_frame_av; +} PhotosensitivityContext; + +#define OFFSET(x) offsetof(PhotosensitivityContext, x) +#define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM + +static const AVOption photosensitivity_options[] = { + { "frames", "set how many frames to use" , OFFSET(nb_frames ), AV_OPT_TYPE_INT , {.i64=30}, 2, MAX_FRAMES, FLAGS }, + { "f", "set how many frames to use" , OFFSET(nb_frames ), AV_OPT_TYPE_INT , {.i64=30}, 2, MAX_FRAMES, FLAGS }, + { "threshold", "set detection threshold factor (lower is stricter)", OFFSET(threshold_multiplier), AV_OPT_TYPE_FLOAT, {.dbl= 1}, 0.1, FLT_MAX, FLAGS }, + { "t" , "set detection threshold factor (lower is stricter)", OFFSET(threshold_multiplier), AV_OPT_TYPE_FLOAT, {.dbl= 1}, 0.1, FLT_MAX, FLAGS }, + { "skip" , "set pixels to skip when sampling frames" , OFFSET(skip ), AV_OPT_TYPE_INT , {.i64= 1}, 1, 1024 , FLAGS }, + { "bypass" , "leave frames unchanged" , OFFSET(bypass ), AV_OPT_TYPE_BOOL , {.i64= 0}, 0, 1 , FLAGS }, + { NULL } +}; + +AVFILTER_DEFINE_CLASS(photosensitivity); + +static int query_formats(AVFilterContext *ctx) +{ + static const enum AVPixelFormat pixel_fmts[] = { + AV_PIX_FMT_RGB24, + AV_PIX_FMT_BGR24, + AV_PIX_FMT_NONE + }; + AVFilterFormats *formats = ff_make_format_list(pixel_fmts); + if (!formats) + return AVERROR(ENOMEM); + return ff_set_common_formats(ctx, formats); +} + +typedef struct ThreadData_convert_frame +{ + AVFrame *in; + PhotosensitivityFrame* out; + int skip; +} ThreadData_convert_frame; + +#define NUM_CELLS (GRID_SIZE * GRID_SIZE) + +static int convert_frame_partial(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs) +{ + int cell, gx, gy, x0, x1, y0, y1, x, y, c, area; + int sum[NUM_CHANNELS]; + const uint8_t *p; + + ThreadData_convert_frame *td = arg; + + const int slice_start = (NUM_CELLS * jobnr) / nb_jobs; + const int slice_end = (NUM_CELLS * (jobnr+1)) / nb_jobs; + + int width = td->in->width, height = td->in->height, linesize = td->in->linesize[0], skip = td->skip; + const uint8_t *data = td->in->data[0]; + + for (cell = slice_start; cell < slice_end; cell++) { + gx = cell % GRID_SIZE; + gy = cell / GRID_SIZE; + + x0 = width * gx / GRID_SIZE; + x1 = width * (gx+1) / GRID_SIZE; + y0 = height * gy / GRID_SIZE; + y1 = height * (gy+1) / GRID_SIZE; + + for (c = 0; c < NUM_CHANNELS; c++) { + sum[c] = 0; + } + for (y = y0; y < y1; y += skip) { + p = data + y * linesize + x0 * NUM_CHANNELS; + for (x = x0; x < x1; x += skip) { + //av_log(NULL, AV_LOG_VERBOSE, "%d %d %d : (%d,%d) (%d,%d) -> %d,%d | *%d\n", c, gx, gy, x0, y0, x1, y1, x, y, (int)row); + sum[0] += p[0]; + sum[1] += p[1]; + sum[2] += p[2]; + p += NUM_CHANNELS * skip; + // TODO: variable size + } + } + + area = ((x1 - x0 + skip - 1) / skip) * ((y1 - y0 + skip - 1) / skip); + for (c = 0; c < NUM_CHANNELS; c++) { + if (area) + sum[c] /= area; + td->out->grid[gy][gx][c] = sum[c]; + } + } + return 0; +} + +static void convert_frame(AVFilterContext *ctx, AVFrame *in, PhotosensitivityFrame* out, int skip) +{ + ThreadData_convert_frame td; + td.in = in; + td.out = out; + td.skip = skip; + ctx->internal->execute(ctx, convert_frame_partial, &td, NULL, FFMIN(NUM_CELLS, ff_filter_get_nb_threads(ctx))); +} + +typedef struct ThreadData_blend_frame +{ + AVFrame *target; + AVFrame *source; + uint16_t s_mul; +} ThreadData_blend_frame; + +static int blend_frame_partial(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs) +{ + int x, y; + uint8_t *t, *s; + + ThreadData_blend_frame *td = arg; + const uint16_t s_mul = td->s_mul; + const uint16_t t_mul = 0x100 - s_mul; + const int slice_start = (td->target->height * jobnr) / nb_jobs; + const int slice_end = (td->target->height * (jobnr+1)) / nb_jobs; + const int linesize = td->target->linesize[0]; + + for (y = slice_start; y < slice_end; y++) { + t = td->target->data[0] + y * td->target->linesize[0]; + s = td->source->data[0] + y * td->source->linesize[0]; + for (x = 0; x < linesize; x++) { + *t = (*t * t_mul + *s * s_mul) >> 8; + t++; s++; + } + } + return 0; +} + +static void blend_frame(AVFilterContext *ctx, AVFrame *target, AVFrame *source, float factor) +{ + ThreadData_blend_frame td; + td.target = target; + td.source = source; + td.s_mul = (uint16_t)(factor * 0x100); + ctx->internal->execute(ctx, blend_frame_partial, &td, NULL, FFMIN(ctx->outputs[0]->h, ff_filter_get_nb_threads(ctx))); +} + +static int get_badness(PhotosensitivityFrame* a, PhotosensitivityFrame* b) +{ + int badness, x, y, c; + badness = 0; + for (c = 0; c < NUM_CHANNELS; c++) { + for (y = 0; y < GRID_SIZE; y++) { + for (x = 0; x < GRID_SIZE; x++) { + badness += abs((int)a->grid[y][x][c] - (int)b->grid[y][x][c]); + //av_log(NULL, AV_LOG_VERBOSE, "%d - %d -> %d \n", a->grid[y][x], b->grid[y][x], badness); + //av_log(NULL, AV_LOG_VERBOSE, "%d -> %d \n", abs((int)a->grid[y][x] - (int)b->grid[y][x]), badness); + } + } + } + return badness; +} + +static int config_input(AVFilterLink *inlink) +{ + /* const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format); */ + AVFilterContext *ctx = inlink->dst; + PhotosensitivityContext *s = ctx->priv; + + s->badness_threshold = (int)(GRID_SIZE * GRID_SIZE * 4 * 256 * s->nb_frames * s->threshold_multiplier / 128); + + return 0; +} + +static int filter_frame(AVFilterLink *inlink, AVFrame *in) +{ + int this_badness, current_badness, fixed_badness, new_badness, i, res; + PhotosensitivityFrame ef; + AVFrame *src, *out; + float factor; + AVDictionary **metadata; + + AVFilterContext *ctx = inlink->dst; + AVFilterLink *outlink = ctx->outputs[0]; + PhotosensitivityContext *s = ctx->priv; + + /* weighted moving average */ + current_badness = 0; + for (i = 1; i < s->nb_frames; i++) + current_badness += i * s->history[(s->history_pos + i) % s->nb_frames]; + current_badness /= s->nb_frames; + + convert_frame(ctx, in, &ef, s->skip); + this_badness = get_badness(&ef, &s->last_frame_e); + new_badness = current_badness + this_badness; + av_log(s, AV_LOG_VERBOSE, "badness: %6d -> %6d / %6d (%3d%% - %s) \n", + current_badness, new_badness, s->badness_threshold, + 100 * new_badness / s->badness_threshold, new_badness < s->badness_threshold ? "OK" : "EXCEEDED"); + + fixed_badness = new_badness; + if (new_badness < s->badness_threshold || !s->last_frame_av || s->bypass) { + factor = 1; /* for metadata */ + av_frame_free(&s->last_frame_av); + s->last_frame_av = src = in; + s->last_frame_e = ef; + s->history[s->history_pos] = this_badness; + } else { + factor = (float)(s->badness_threshold - current_badness) / (new_badness - current_badness); + if (factor <= 0) { + /* just duplicate the frame */ + s->history[s->history_pos] = 0; /* frame was duplicated, thus, delta is zero */ + } else { + res = av_frame_make_writable(s->last_frame_av); + if (res) { + av_frame_free(&in); + return res; + } + blend_frame(ctx, s->last_frame_av, in, factor); + + convert_frame(ctx, s->last_frame_av, &ef, s->skip); + this_badness = get_badness(&ef, &s->last_frame_e); + fixed_badness = current_badness + this_badness; + av_log(s, AV_LOG_VERBOSE, " fixed: %6d -> %6d / %6d (%3d%%) factor=%5.3f \n", + current_badness, fixed_badness, s->badness_threshold, + 100 * new_badness / s->badness_threshold, factor); + s->last_frame_e = ef; + s->history[s->history_pos] = this_badness; + } + src = s->last_frame_av; + } + s->history_pos = (s->history_pos + 1) % s->nb_frames; + + out = ff_get_video_buffer(outlink, in->width, in->height); + if (!out) { + av_frame_free(&in); + return AVERROR(ENOMEM); + } + av_frame_copy_props(out, in); + metadata = &out->metadata; + if (metadata) { + char value[128]; + + snprintf(value, sizeof(value), "%f", (float)new_badness / s->badness_threshold); + av_dict_set(metadata, "lavfi.photosensitivity.badness", value, 0); + + snprintf(value, sizeof(value), "%f", (float)fixed_badness / s->badness_threshold); + av_dict_set(metadata, "lavfi.photosensitivity.fixed-badness", value, 0); + + snprintf(value, sizeof(value), "%f", (float)this_badness / s->badness_threshold); + av_dict_set(metadata, "lavfi.photosensitivity.frame-badness", value, 0); + + snprintf(value, sizeof(value), "%f", factor); + av_dict_set(metadata, "lavfi.photosensitivity.factor", value, 0); + } + av_frame_copy(out, src); + return ff_filter_frame(outlink, out); +} + +static av_cold void uninit(AVFilterContext *ctx) +{ + PhotosensitivityContext *s = ctx->priv; + + av_frame_free(&s->last_frame_av); +} + +static const AVFilterPad inputs[] = { + { + .name = "default", + .type = AVMEDIA_TYPE_VIDEO, + .filter_frame = filter_frame, + .config_props = config_input, + }, + { NULL } +}; + +static const AVFilterPad outputs[] = { + { + .name = "default", + .type = AVMEDIA_TYPE_VIDEO, + }, + { NULL } +}; + +AVFilter ff_vf_photosensitivity = { + .name = "photosensitivity", + .description = NULL_IF_CONFIG_SMALL("Attempt to filter out photosensitive epilepsy seizure-inducing flashes."), + .priv_size = sizeof(PhotosensitivityContext), + .priv_class = &photosensitivity_class, + .uninit = uninit, + .query_formats = query_formats, + .inputs = inputs, + .outputs = outputs, +};