[FFmpeg-devel,v1] avfilter/vf_scenechangedetect: add filter to detect scene change

Submitted by lance.lmwang@gmail.com on Sept. 24, 2019, 11:32 a.m.

Details

Message ID 20190924113200.14115-1-lance.lmwang@gmail.com
State New
Headers show

Commit Message

lance.lmwang@gmail.com Sept. 24, 2019, 11:32 a.m.
From: Limin Wang <lance.lmwang@gmail.com>

Signed-off-by: Limin Wang <lance.lmwang@gmail.com>
---
I am not sure if a new filter should be added to detect scene changes. Now some filters f_select, vf_fieldmatch, vf_framerate, vf_minterpotate implement their own version of the scene change detection, I think this is not easy to maintain and it's better to have a pure scene change detection filter rather than using each version of your filter. Let keep it simple and let it output the mafd metadata to avoid repeat calculate by later filters which need mafd in the filterchain.

In some condition, if the input video resolution is 4K and output is same, we can resize it low resolution first, then use the resized frame to get the scene score and mafd results and pass them to later filters without change the original frame. It'll greatly improve the processing performance in the case. I wish to add this if the patch is acceptable.
 
 configure                          |   1 +
 doc/filters.texi                   |  35 +++++
 libavfilter/Makefile               |   1 +
 libavfilter/allfilters.c           |   1 +
 libavfilter/version.h              |   2 +-
 libavfilter/vf_scenechangedetect.c | 221 +++++++++++++++++++++++++++++
 6 files changed, 260 insertions(+), 1 deletion(-)
 create mode 100644 libavfilter/vf_scenechangedetect.c

Patch hide | download patch | download mbox

diff --git a/configure b/configure
index 8413826f9e..909a77b9cf 100755
--- a/configure
+++ b/configure
@@ -3520,6 +3520,7 @@  sab_filter_deps="gpl swscale"
 scale2ref_filter_deps="swscale"
 scale_filter_deps="swscale"
 scale_qsv_filter_deps="libmfx"
+scenechangedetect_filter_select="scene_sad"
 select_filter_select="scene_sad"
 sharpness_vaapi_filter_deps="vaapi"
 showcqt_filter_deps="avcodec avformat swscale"
diff --git a/doc/filters.texi b/doc/filters.texi
index e41384aed8..d0fdf957af 100644
--- a/doc/filters.texi
+++ b/doc/filters.texi
@@ -15642,6 +15642,41 @@  Scale a logo to 1/10th the height of a video, while preserving its display aspec
 @end example
 @end itemize
 
+@anchor{scenechangedetect}
+@section scenechangedetect
+
+Detect video scene change.
+
+This filter sets frame metadata with mafd between frame, the scene score, and
+forward the frame to the next filter, so they can use these metadata to detect
+scene change or others.
+
+In addition, this filter logs a message and sets frame metadata when it detects
+a scene change by @option{threshold}.
+
+@code{lavfi.scd.mafd} metadata keys are set with mafd for every frame.
+
+@code{lavfi.scd.score} metadata keys are set with scene change score for every frame
+to detect scene change.
+
+@code{lavfi.scd.time} metadata keys are set with current filtered frame time which
+detect scene change with @option{threshold}.
+
+The filter accepts the following options:
+
+@table @option
+@item threshold, t
+Set the scene change detection threshold as a percentage of maximum change.  Good
+values are in the @code{[8.0, 14.0]} range. The range for @option{threshold} is
+@code{[0., 100.]}.
+
+Default value is @code{10.}.
+
+@item sc_pass, s
+Set the scene change pass the frame to the next filter or not. Default value is @code{0}
+You can enable it if you want to get snapshot of scene change frames.
+@end table
+
 @anchor{selectivecolor}
 @section selectivecolor
 
diff --git a/libavfilter/Makefile b/libavfilter/Makefile
index 3ef4191d9a..fe564de535 100644
--- a/libavfilter/Makefile
+++ b/libavfilter/Makefile
@@ -356,6 +356,7 @@  OBJS-$(CONFIG_SCALE_NPP_FILTER)              += vf_scale_npp.o scale.o
 OBJS-$(CONFIG_SCALE_QSV_FILTER)              += vf_scale_qsv.o
 OBJS-$(CONFIG_SCALE_VAAPI_FILTER)            += vf_scale_vaapi.o scale.o vaapi_vpp.o
 OBJS-$(CONFIG_SCALE2REF_FILTER)              += vf_scale.o scale.o
+OBJS-$(CONFIG_SCENECHANGEDETECT_FILTER)      += vf_scenechangedetect.o
 OBJS-$(CONFIG_SELECT_FILTER)                 += f_select.o
 OBJS-$(CONFIG_SELECTIVECOLOR_FILTER)         += vf_selectivecolor.o
 OBJS-$(CONFIG_SENDCMD_FILTER)                += f_sendcmd.o
diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
index b675c688ee..297fc432a4 100644
--- a/libavfilter/allfilters.c
+++ b/libavfilter/allfilters.c
@@ -338,6 +338,7 @@  extern AVFilter ff_vf_scale_npp;
 extern AVFilter ff_vf_scale_qsv;
 extern AVFilter ff_vf_scale_vaapi;
 extern AVFilter ff_vf_scale2ref;
+extern AVFilter ff_vf_scenechangedetect;
 extern AVFilter ff_vf_select;
 extern AVFilter ff_vf_selectivecolor;
 extern AVFilter ff_vf_sendcmd;
diff --git a/libavfilter/version.h b/libavfilter/version.h
index 0261f34772..8d0f543e31 100644
--- a/libavfilter/version.h
+++ b/libavfilter/version.h
@@ -30,7 +30,7 @@ 
 #include "libavutil/version.h"
 
 #define LIBAVFILTER_VERSION_MAJOR   7
-#define LIBAVFILTER_VERSION_MINOR  58
+#define LIBAVFILTER_VERSION_MINOR  59
 #define LIBAVFILTER_VERSION_MICRO 102
 
 
diff --git a/libavfilter/vf_scenechangedetect.c b/libavfilter/vf_scenechangedetect.c
new file mode 100644
index 0000000000..64d163e480
--- /dev/null
+++ b/libavfilter/vf_scenechangedetect.c
@@ -0,0 +1,221 @@ 
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file
+ * video scene change detection filter
+ */
+
+#include "libavutil/avassert.h"
+#include "libavutil/imgutils.h"
+#include "libavutil/opt.h"
+#include "libavutil/pixdesc.h"
+#include "libavutil/timestamp.h"
+
+#include "avfilter.h"
+#include "filters.h"
+#include "scene_sad.h"
+
+typedef struct SceneChangeDetectContext {
+    const AVClass *class;
+
+    ptrdiff_t width[4];
+    ptrdiff_t height[4];
+    int nb_planes;
+    int bitdepth;
+    ff_scene_sad_fn sad;
+    double prev_mafd;
+    double scene_score;
+    AVFrame *prev_picref;
+    double threshold;
+    int sc_pass;
+} SceneChangeDetectContext;
+
+#define OFFSET(x) offsetof(SceneChangeDetectContext, x)
+#define V AV_OPT_FLAG_VIDEO_PARAM
+#define F AV_OPT_FLAG_FILTERING_PARAM
+
+static const AVOption scenechangedetect_options[] = {
+    { "threshold",   "set scene change detect level",   OFFSET(threshold),  AV_OPT_TYPE_DOUBLE,   {.dbl = 10.},     0,  100., V|F },
+    { "t",           "set scene change detect level",   OFFSET(threshold),  AV_OPT_TYPE_DOUBLE,   {.dbl = 10.},     0,  100., V|F },
+    { "sc_pass",     "set scene change frame pass",     OFFSET(sc_pass),  AV_OPT_TYPE_BOOL,       {.dbl = 0  },     0,    1., V|F },
+    { "s",           "set scene change frame pass",     OFFSET(sc_pass),  AV_OPT_TYPE_BOOL,       {.dbl = 0  },     0,    1., V|F },
+    {NULL}
+};
+
+AVFILTER_DEFINE_CLASS(scenechangedetect);
+
+static int query_formats(AVFilterContext *ctx)
+{
+    static const enum AVPixelFormat pix_fmts[] = {
+            AV_PIX_FMT_RGB24, AV_PIX_FMT_BGR24, AV_PIX_FMT_RGBA,
+            AV_PIX_FMT_ABGR, AV_PIX_FMT_BGRA, AV_PIX_FMT_GRAY8,
+            AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUVJ420P,
+            AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUVJ422P,
+            AV_PIX_FMT_YUV420P10,
+            AV_PIX_FMT_NONE
+    };
+
+    AVFilterFormats *fmts_list = ff_make_format_list(pix_fmts);
+    if (!fmts_list)
+        return AVERROR(ENOMEM);
+    return ff_set_common_formats(ctx, fmts_list);
+}
+
+static int config_input(AVFilterLink *inlink)
+{
+    AVFilterContext *ctx = inlink->dst;
+    SceneChangeDetectContext *s = ctx->priv;
+    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
+    int is_yuv = !(desc->flags & AV_PIX_FMT_FLAG_RGB) &&
+        (desc->flags & AV_PIX_FMT_FLAG_PLANAR) &&
+        desc->nb_components >= 3;
+
+    s->bitdepth = desc->comp[0].depth;
+    s->nb_planes = is_yuv ? 1 : av_pix_fmt_count_planes(inlink->format);
+
+    for (int plane = 0; plane < 4; plane++) {
+        ptrdiff_t line_size = av_image_get_linesize(inlink->format, inlink->w, plane);
+        s->width[plane] = line_size >> (s->bitdepth > 8);
+        s->height[plane] = inlink->h >> ((plane == 1 || plane == 2) ? desc->log2_chroma_h : 0);
+    }
+
+    s->sad = ff_scene_sad_get_fn(s->bitdepth == 8 ? 8 : 16);
+    if (!s->sad)
+        return AVERROR(EINVAL);
+
+    return 0;
+}
+
+static av_cold void uninit(AVFilterContext *ctx)
+{
+    SceneChangeDetectContext *s = ctx->priv;
+
+    av_frame_free(&s->prev_picref);
+}
+
+static double get_scene_score(AVFilterContext *ctx, AVFrame *frame)
+{
+    double ret = 0;
+    SceneChangeDetectContext *s = ctx->priv;
+    AVFrame *prev_picref = s->prev_picref;
+
+    if (prev_picref && frame->height == prev_picref->height
+                    && frame->width  == prev_picref->width) {
+        uint64_t sad = 0;
+        double mafd, diff;
+        uint64_t count = 0;
+
+        for (int plane = 0; plane < s->nb_planes; plane++) {
+            uint64_t plane_sad;
+            s->sad(prev_picref->data[plane], prev_picref->linesize[plane],
+                    frame->data[plane], frame->linesize[plane],
+                    s->width[plane], s->height[plane], &plane_sad);
+            sad += plane_sad;
+            count += s->width[plane] * s->height[plane];
+        }
+
+        emms_c();
+        mafd = (double)sad * 100. / count / (1ULL << s->bitdepth);
+        diff = fabs(mafd - s->prev_mafd);
+        ret  = av_clipf(FFMIN(mafd, diff), 0, 100.);
+        s->prev_mafd = mafd;
+        av_frame_free(&prev_picref);
+    }
+    s->prev_picref = av_frame_clone(frame);
+    return ret;
+}
+
+static int set_meta(SceneChangeDetectContext *s, AVFrame *frame, const char *key, const char *value)
+{
+    return av_dict_set(&frame->metadata, key, value, 0);
+}
+
+static int activate(AVFilterContext *ctx)
+{
+    int ret;
+    AVFilterLink *inlink = ctx->inputs[0];
+    AVFilterLink *outlink = ctx->outputs[0];
+    SceneChangeDetectContext *s = ctx->priv;
+    AVFrame *frame;
+
+    FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink);
+
+    ret = ff_inlink_consume_frame(inlink, &frame);
+    if (ret < 0)
+        return ret;
+
+    if (frame) {
+        char buf[64];
+        s->scene_score = get_scene_score(ctx, frame);
+        snprintf(buf, sizeof(buf), "%0.3f", s->prev_mafd);
+        set_meta(s, frame, "lavfi.scd.mafd", buf);
+        snprintf(buf, sizeof(buf), "%0.3f", s->scene_score);
+        set_meta(s, frame, "lavfi.scd.score", buf);
+
+        if (s->scene_score > s->threshold) {
+            av_log(s, AV_LOG_INFO, "lavfi.scd.score: %.3f, lavfi.scd.time: %s\n",
+                    s->scene_score, av_ts2timestr(frame->pts, &inlink->time_base));
+            set_meta(s, frame, "lavfi.scd.time",
+                    av_ts2timestr(frame->pts, &inlink->time_base));
+        }
+        if (s->sc_pass) {
+            if (s->scene_score > s->threshold)
+                return ff_filter_frame(outlink, frame);
+            else {
+                av_frame_free(&frame);
+            }
+
+        } else
+            return ff_filter_frame(outlink, frame);
+    }
+
+    FF_FILTER_FORWARD_STATUS(inlink, outlink);
+    FF_FILTER_FORWARD_WANTED(outlink, inlink);
+
+    return FFERROR_NOT_READY;
+}
+
+static const AVFilterPad scenechangedetect_inputs[] = {
+    {
+        .name         = "default",
+        .type         = AVMEDIA_TYPE_VIDEO,
+        .config_props = config_input,
+    },
+    { NULL }
+};
+
+static const AVFilterPad scenechangedetect_outputs[] = {
+    {
+        .name          = "default",
+        .type          = AVMEDIA_TYPE_VIDEO,
+    },
+    { NULL }
+};
+
+AVFilter ff_vf_scenechangedetect = {
+    .name          = "scenechangedetect",
+    .description   = NULL_IF_CONFIG_SMALL("Detect video scene change"),
+    .priv_size     = sizeof(SceneChangeDetectContext),
+    .priv_class    = &scenechangedetect_class,
+    .uninit        = uninit,
+    .query_formats = query_formats,
+    .inputs        = scenechangedetect_inputs,
+    .outputs       = scenechangedetect_outputs,
+    .activate      = activate,
+};