| Message ID | 20191026120025.24822-1-ckennedy@ellation.com |
|---|---|
| State | New |
| Headers | show |
Why is this not generic filter like already existing opencv filter? On 10/26/19, ckennedy@ellation.com <ckennedy@ellation.com> wrote: > From: Christopher Kennedy <ckennedy@ellation.com> > > this adds a phqm filter and OpenCV img_hash based resource usable > by the phqm and future filters using image hash functionality > from OpenCV. > > C++ to C handling so that full OpenCV functionality and API can > be used instead of the C versions (which are incomplete and > don't always exist). > > Example command line: > > ffmpeg -i encode.mp4 -i reference.mp4 \ > -filter_complex "[0:v][1:v]phqm=stats_file=out.log" \ > -y -f null /dev/null > > Signed-off-by: Christopher Kennedy <ckennedy@ellation.com> > --- > Changelog | 1 + > configure | 2 + > libavfilter/Makefile | 2 + > libavfilter/allfilters.c | 1 + > libavfilter/img_hash.cpp | 98 ++++++++++++ > libavfilter/img_hash.h | 46 ++++++ > libavfilter/vf_phqm.c | 334 +++++++++++++++++++++++++++++++++++++++ > 7 files changed, 484 insertions(+) > create mode 100644 libavfilter/img_hash.cpp > create mode 100644 libavfilter/img_hash.h > create mode 100644 libavfilter/vf_phqm.c > > diff --git a/Changelog b/Changelog > index 316589e336..4a22f77d37 100644 > --- a/Changelog > +++ b/Changelog > @@ -17,6 +17,7 @@ version <next>: > - anlms filter > - arnndn filter > - bilateral filter > +- phqm perceptual hash filter using OpenCV img_lib > > > version 4.2: > diff --git a/configure b/configure > index 8413826f9e..e231d359bb 100755 > --- a/configure > +++ b/configure > @@ -3497,6 +3497,8 @@ nlmeans_opencl_filter_deps="opencl" > nnedi_filter_deps="gpl" > ocr_filter_deps="libtesseract" > ocv_filter_deps="libopencv" > +phqm_filter_deps="libopencv" > +phqm_filter_extralibs="-lstdc++ -lopencv_img_hash" > openclsrc_filter_deps="opencl" > overlay_opencl_filter_deps="opencl" > overlay_qsv_filter_deps="libmfx" > diff --git a/libavfilter/Makefile b/libavfilter/Makefile > index 63d2fba861..645e232b3e 100644 > --- a/libavfilter/Makefile > +++ b/libavfilter/Makefile > @@ -325,6 +325,7 @@ OBJS-$(CONFIG_PERMS_FILTER) += > f_perms.o > OBJS-$(CONFIG_PERSPECTIVE_FILTER) += vf_perspective.o > OBJS-$(CONFIG_PHASE_FILTER) += vf_phase.o > OBJS-$(CONFIG_PHOTOSENSITIVITY_FILTER) += vf_photosensitivity.o > +OBJS-$(CONFIG_PHQM_FILTER) += vf_phqm.o img_hash.o > OBJS-$(CONFIG_PIXDESCTEST_FILTER) += vf_pixdesctest.o > OBJS-$(CONFIG_PIXSCOPE_FILTER) += vf_datascope.o > OBJS-$(CONFIG_PP_FILTER) += vf_pp.o > @@ -498,6 +499,7 @@ OBJS-$(CONFIG_SHARED) += > log2_tab.o > SKIPHEADERS-$(CONFIG_QSVVPP) += qsvvpp.h > SKIPHEADERS-$(CONFIG_OPENCL) += opencl.h > SKIPHEADERS-$(CONFIG_VAAPI) += vaapi_vpp.h > +SKIPHEADERS-$(CONFIG_LIBOPENCV) += img_hash.h > > TOOLS = graph2dot > TESTPROGS = drawutils filtfmts formats integral > diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c > index e4186f93db..f0fcaad235 100644 > --- a/libavfilter/allfilters.c > +++ b/libavfilter/allfilters.c > @@ -309,6 +309,7 @@ extern AVFilter ff_vf_perms; > extern AVFilter ff_vf_perspective; > extern AVFilter ff_vf_phase; > extern AVFilter ff_vf_photosensitivity; > +extern AVFilter ff_vf_phqm; > extern AVFilter ff_vf_pixdesctest; > extern AVFilter ff_vf_pixscope; > extern AVFilter ff_vf_pp; > diff --git a/libavfilter/img_hash.cpp b/libavfilter/img_hash.cpp > new file mode 100644 > index 0000000000..4d5843da22 > --- /dev/null > +++ b/libavfilter/img_hash.cpp > @@ -0,0 +1,98 @@ > +/* > + * Copyright (c) 2019 Christopher Kennedy > + * > + * OpenCV img_hash > + * > + * This file is part of FFmpeg. > + * > + * FFmpeg is free software; you can redistribute it and/or > + * modify it under the terms of the GNU Lesser General Public > + * License as published by the Free Software Foundation; either > + * version 2.1 of the License, or (at your option) any later version. > + * > + * FFmpeg is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + * Lesser General Public License for more details. > + * > + * You should have received a copy of the GNU Lesser General Public > + * License along with FFmpeg; if not, write to the Free Software > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 > USA > + */ > + > +#include <opencv2/core.hpp> > +#include <opencv2/core/ocl.hpp> > +#include <opencv2/highgui.hpp> > +#include <opencv2/img_hash.hpp> > +#include <opencv2/imgproc.hpp> > + > +#include <iostream> > + > +#include "img_hash.h" > +#include "libavutil/pixdesc.h" > +extern "C" { > +#include "avfilter.h" > +} > + > +// convert from avframe to iplimage format > +static int fill_iplimage_from_frame(IplImage *img, const AVFrame *frame, > enum AVPixelFormat pixfmt) > +{ > + IplImage *tmpimg; > + int depth = IPL_DEPTH_8U, channels_nb; > + > + switch (pixfmt) { > + case AV_PIX_FMT_GRAY8: channels_nb = 1; break; > + case AV_PIX_FMT_BGRA: channels_nb = 4; break; > + case AV_PIX_FMT_BGR24: channels_nb = 3; break; > + default: return -1; > + } > + > + tmpimg = cvCreateImageHeader((CvSize){frame->width, frame->height}, > depth, channels_nb); > + *img = *tmpimg; > + img->imageData = img->imageDataOrigin = (char *) frame->data[0]; > + img->dataOrder = IPL_DATA_ORDER_PIXEL; > + img->origin = IPL_ORIGIN_TL; > + img->widthStep = frame->linesize[0]; > + > + return 0; > +} > + > +// Get the score of two Video Frames by comparing the perceptual hashes and > deriving a hamming distance > +// showing how similar they are or different. lower score is better for > most algorithms > +extern "C" double getScore(const AVFrame *frame1, const AVFrame *frame2, > enum AVPixelFormat pixfmt, int hash_type) { > + cv::Ptr<cv::img_hash::ImgHashBase> algo; > + IplImage ipl1, ipl2; > + cv::Mat h1; > + cv::Mat h2; > + cv::Mat m1; > + cv::Mat m2; > + > + // Take FFmpeg video frame and convert into an IplImage for OpenCV > + if (fill_iplimage_from_frame(&ipl1, frame1, pixfmt) != 0 || > + fill_iplimage_from_frame(&ipl2, frame2, pixfmt) != 0) > + return DBL_MAX; // Return an invalid value if either fails > + > + // Convert an IplImage to an Mat Image for OpenCV (newer format) > + m1 = cv::cvarrToMat(&ipl1); > + m2 = cv::cvarrToMat(&ipl2); > + > + // substantiate the hash type algorithm > + switch (hash_type) { > + case PHASH: algo = cv::img_hash::PHash::create(); > break; > + case AVERAGE: algo = cv::img_hash::AverageHash::create(); > break; > + case MARRHILDRETH: algo = > cv::img_hash::MarrHildrethHash::create(); break; > + case RADIALVARIANCE: algo = > cv::img_hash::RadialVarianceHash::create(); break; > + // BlockMeanHash support mode 0 and mode 1, they associate to > + // mode 1 and mode 2 of PHash library > + case BLOCKMEAN1: algo = > cv::img_hash::BlockMeanHash::create(0); break; > + case BLOCKMEAN2: algo = > cv::img_hash::BlockMeanHash::create(1); break; > + case COLORMOMENT: algo = > cv::img_hash::ColorMomentHash::create(); break; > + } > + > + // Compute the hash > + algo->compute(m1, h1); > + algo->compute(m2, h2); > + > + // Compare the hashes and return the hamming distance > + return algo->compare(h1, h2); > +} > diff --git a/libavfilter/img_hash.h b/libavfilter/img_hash.h > new file mode 100644 > index 0000000000..76f55c3013 > --- /dev/null > +++ b/libavfilter/img_hash.h > @@ -0,0 +1,46 @@ > +/* > + * Copyright (c) 2019 Christopher Kennedy > + * > + * PHQM Perceptual Hash Quality Metric > + * > + * This file is part of FFmpeg. > + * > + * FFmpeg is free software; you can redistribute it and/or > + * modify it under the terms of the GNU Lesser General Public > + * License as published by the Free Software Foundation; either > + * version 2.1 of the License, or (at your option) any later version. > + * > + * FFmpeg is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + * Lesser General Public License for more details. > + * > + * You should have received a copy of the GNU Lesser General Public > + * License along with FFmpeg; if not, write to the Free Software > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 > USA > + */ > + > +#ifndef AVFILTER_IMG_HASH_H > +#define AVFILTER_IMG_HASH_H > + > +#include "avfilter.h" > + > +#if defined(__cplusplus) > +extern "C" > +{ > +#endif > + > +#define AVERAGE 0 > +#define BLOCKMEAN1 1 > +#define BLOCKMEAN2 2 > +#define COLORMOMENT 3 > +#define MARRHILDRETH 4 > +#define PHASH 5 > +#define RADIALVARIANCE 6 > + > +double getScore(const AVFrame *frame1, const AVFrame *frame2, enum > AVPixelFormat pixfmt, int hash_type); > +#if defined(__cplusplus) > +} > +#endif > + > +#endif > diff --git a/libavfilter/vf_phqm.c b/libavfilter/vf_phqm.c > new file mode 100644 > index 0000000000..0930386b10 > --- /dev/null > +++ b/libavfilter/vf_phqm.c > @@ -0,0 +1,334 @@ > +/* > + * Copyright (c) 2019 Christopher Kennedy > + * > + * PHQM Perceptual Hash Quality Metric > + * > + * This file is part of FFmpeg. > + * > + * FFmpeg is free software; you can redistribute it and/or > + * modify it under the terms of the GNU Lesser General Public > + * License as published by the Free Software Foundation; either > + * version 2.1 of the License, or (at your option) any later version. > + * > + * FFmpeg is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + * Lesser General Public License for more details. > + * > + * You should have received a copy of the GNU Lesser General Public > + * License along with FFmpeg; if not, write to the Free Software > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 > USA > + */ > + > +/** > + * @file > + * PHQM: Calculate the Image Hash Hamming Difference between two input > videos. > + */ > + > +#include <float.h> > +#include "libavutil/avstring.h" > +#include "libavutil/opt.h" > +#include "libavutil/pixdesc.h" > +#include "avfilter.h" > +#include "drawutils.h" > +#include "formats.h" > +#include "framesync.h" > +#include "internal.h" > +#include "video.h" > + > +#include "img_hash.h" > +#include "scene_sad.h" > + > +typedef struct PHQMContext { > + const AVClass *class; > + FFFrameSync fs; > + double shd, hd, min_hd, max_hd, smin_hd, smax_hd; > + uint64_t nb_shd; > + uint64_t nb_frames; > + FILE *stats_file; > + char *stats_file_str; > + int hash_type; > + ff_scene_sad_fn sad; ///< Sum of the absolute difference > function (scene detect only) > + double prev_mafd; ///< previous MAFD > (scene detect only) > + AVFrame *prev_picref; ///< previous frame > (scene detect only) > + double scd_thresh; > + double scene_score; > +} PHQMContext; > + > +#define OFFSET(x) offsetof(PHQMContext, x) > +#define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM > + > +static const AVOption phqm_options[] = { > + { "stats_file", "Set file where to store per-frame difference > information.", OFFSET(stats_file_str), AV_OPT_TYPE_STRING, {.str=NULL}, 0, > 0, FLAGS }, > + { "f", "Set file where to store per-frame difference > information.", OFFSET(stats_file_str), AV_OPT_TYPE_STRING, {.str=NULL}, 0, > 0, FLAGS }, > + { "scd_thresh", "Scene Change Detection Threshold.", > OFFSET(scd_thresh), AV_OPT_TYPE_DOUBLE, {.dbl=0.5}, 0, 1, FLAGS }, > + { "hash_type", "Type of Image Hash to use from OpenCV.", > OFFSET(hash_type), AV_OPT_TYPE_INT, {.i64 = PHASH}, 0, 6, > FLAGS, "hash_type" }, > + { "average", "Average Hash", 0, > AV_OPT_TYPE_CONST, {.i64 = AVERAGE}, 0, 0, FLAGS, "hash_type" }, > + { "blockmean1", "Block Mean Hash 1", 0, > AV_OPT_TYPE_CONST, {.i64 = BLOCKMEAN1}, 0, 0, FLAGS, "hash_type" }, > + { "blockmean2", "Block Mean Hash 2", 0, > AV_OPT_TYPE_CONST, {.i64 = BLOCKMEAN2}, 0, 0, FLAGS, "hash_type" }, > + { "colormoment", "Color Moment Hash", 0, > AV_OPT_TYPE_CONST, {.i64 = COLORMOMENT}, 0, 0, FLAGS, "hash_type" }, > + { "marrhildreth", "Marr Hildreth Hash", 0, > AV_OPT_TYPE_CONST, {.i64 = MARRHILDRETH}, 0, 0, FLAGS, "hash_type" }, > + { "phash", "Perceptual Hash (PHash)", 0, > AV_OPT_TYPE_CONST, {.i64 = PHASH}, 0, 0, FLAGS, "hash_type" }, > + { "radialvariance", "Radial Variance Hash", 0, > AV_OPT_TYPE_CONST, {.i64 = RADIALVARIANCE}, 0, 0, FLAGS, "hash_type" }, > + { NULL } > +}; > + > +FRAMESYNC_DEFINE_CLASS(phqm, PHQMContext, fs); > + > +static void set_meta(AVDictionary **metadata, const char *key, char comp, > float d) > +{ > + char value[128]; > + snprintf(value, sizeof(value), "%0.2f", d); > + if (comp) { > + char key2[128]; > + snprintf(key2, sizeof(key2), "%s%c", key, comp); > + av_dict_set(metadata, key2, value, 0); > + } else { > + av_dict_set(metadata, key, value, 0); > + } > +} > + > +static double get_scene_score(AVFilterContext *ctx, AVFrame *frame) > +{ > + double ret = 0.; > + PHQMContext *s = ctx->priv; > + AVFrame *prev_picref = s->prev_picref; > + > + if (prev_picref && > + frame->height == prev_picref->height && > + frame->width == prev_picref->width) { > + uint64_t sad; > + double mafd, diff; > + > + s->sad(prev_picref->data[0], prev_picref->linesize[0], > frame->data[0], frame->linesize[0], frame->width * 3, frame->height, &sad); > + emms_c(); > + mafd = (double)sad / (frame->width * 3 * frame->height); > + diff = fabs(mafd - s->prev_mafd); > + ret = av_clipf(FFMIN(mafd, diff) / 100., 0, 1); > + s->prev_mafd = mafd; > + av_frame_free(&prev_picref); > + } > + s->prev_picref = av_frame_clone(frame); > + return ret; > +} > + > +static int do_phqm(FFFrameSync *fs) > +{ > + AVFilterContext *ctx = fs->parent; > + PHQMContext *s = ctx->priv; > + AVFrame *master, *ref; > + double hd = 0.; > + int ret; > + double hd_limit = 1000000.; > + AVDictionary **metadata; > + > + ret = ff_framesync_dualinput_get(fs, &master, &ref); > + if (ret < 0) > + return ret; > + if (!ref) > + return ff_filter_frame(ctx->outputs[0], master); > + metadata = &master->metadata; > + > + > + s->nb_frames++; > + > + /* scene change detection score */ > + s->scene_score = get_scene_score(ctx, ref); > + if (s->scene_score >= s->scd_thresh && s->nb_shd >= 48) { > + av_log(s, AV_LOG_INFO, "ImgHashScene: n:%"PRId64"-%"PRId64" > hd_avg:%0.3lf hd_min:%0.3lf hd_max:%0.3lf scd:%0.2lf\n", > + (s->nb_frames - s->nb_shd), s->nb_frames - 1, (s->shd / > s->nb_shd), s->smin_hd, s->smax_hd, s->scene_score); > + s->shd = 0.; > + s->nb_shd = 0; > + s->smin_hd = 0.; > + s->smax_hd = 0.; > + } > + > + /* limit the highest value so we cut off at perceptual difference match > */ > + switch (s->hash_type) { > + case PHASH: > + case AVERAGE: hd_limit = 5; break; > + case MARRHILDRETH: hd_limit = 30; break; > + case RADIALVARIANCE: hd_limit = 0.9; break; > + case BLOCKMEAN1: hd_limit = 12; break; > + case BLOCKMEAN2: hd_limit = 48; break; > + case COLORMOMENT: hd_limit = 8; break; > + } > + > + /* get ref / enc perceptual hashes and calc hamming distance difference > value */ > + hd = getScore(ref, master, ref->format, s->hash_type); > + if (hd == DBL_MAX) { > + av_log(s, AV_LOG_ERROR, "Failure with handling pix_fmt of AVFrame > for conversion to IPLimage.\n"); > + return AVERROR(EINVAL); > + } > + s->hd += FFMIN(hd, hd_limit); > + set_meta(metadata, "lavfi.phqm.phqm", 0, hd); > + > + /* scene hamming distance avg */ > + s->shd += FFMIN(hd, hd_limit); > + s->nb_shd++; > + av_log(s, AV_LOG_DEBUG, "ImgHashFrame: hd:%0.3lf scd:%0.2lf\n", hd, > s->scene_score); > + > + s->min_hd = FFMIN(s->min_hd, hd); > + s->max_hd = FFMAX(s->max_hd, hd); > + s->smin_hd = FFMIN(s->smin_hd, hd); > + s->smax_hd = FFMAX(s->smax_hd, hd); > + > + if (s->stats_file) { > + fprintf(s->stats_file, > + "n:%"PRId64" phqm:%0.3f phqm_min:%0.3f phqm_max:%0.3f > sad:%0.2f", > + s->nb_frames, hd, s->min_hd, s->max_hd, s->scene_score); > + fprintf(s->stats_file, "\n"); > + } > + > + return ff_filter_frame(ctx->outputs[0], master); > +} > + > +static av_cold int init(AVFilterContext *ctx) > +{ > + PHQMContext *s = ctx->priv; > + > + if (s->stats_file_str) { > + if (!strcmp(s->stats_file_str, "-")) { > + s->stats_file = stdout; > + } else { > + s->stats_file = fopen(s->stats_file_str, "w"); > + if (!s->stats_file) { > + int err = AVERROR(errno); > + char buf[128]; > + av_strerror(err, buf, sizeof(buf)); > + av_log(ctx, AV_LOG_ERROR, "Could not open stats file %s: > %s\n", > + s->stats_file_str, buf); > + return err; > + } > + } > + } > + > + s->sad = ff_scene_sad_get_fn(8); > + if (!s->sad) > + return AVERROR(EINVAL); > + > + s->fs.on_event = do_phqm; > + return 0; > +} > + > +static int query_formats(AVFilterContext *ctx) > +{ > + PHQMContext *s = ctx->priv; > + AVFilterFormats *fmts_list = NULL; > + static const enum AVPixelFormat gray8_pix_fmts[] = { > + AV_PIX_FMT_GRAY8, > + AV_PIX_FMT_NONE > + }; > + static const enum AVPixelFormat bgr24_pix_fmts[] = { > + AV_PIX_FMT_BGR24, > + AV_PIX_FMT_NONE > + }; > + static const enum AVPixelFormat bgra_pix_fmts[] = { > + AV_PIX_FMT_BGRA, > + AV_PIX_FMT_NONE > + }; > + > + switch (s->hash_type) { > + case COLORMOMENT: fmts_list = ff_make_format_list(bgr24_pix_fmts); > break; > + case MARRHILDRETH: fmts_list = ff_make_format_list(bgra_pix_fmts); > break; > + /* all other hashes take the gray8 format */ > + default: fmts_list = ff_make_format_list(gray8_pix_fmts); break; > + } > + if (!fmts_list) > + return AVERROR(ENOMEM); > + return ff_set_common_formats(ctx, fmts_list); > +} > + > +static int config_input_ref(AVFilterLink *inlink) > +{ > + AVFilterContext *ctx = inlink->dst; > + > + if (ctx->inputs[0]->w != ctx->inputs[1]->w || > + ctx->inputs[0]->h != ctx->inputs[1]->h) { > + av_log(ctx, AV_LOG_ERROR, "Width and height of input videos must be > same.\n"); > + return AVERROR(EINVAL); > + } > + if (ctx->inputs[0]->format != ctx->inputs[1]->format) { > + av_log(ctx, AV_LOG_ERROR, "Inputs must be of same pixel > format.\n"); > + return AVERROR(EINVAL); > + } > + > + return 0; > +} > + > +static int config_output(AVFilterLink *outlink) > +{ > + AVFilterContext *ctx = outlink->src; > + PHQMContext *s = ctx->priv; > + AVFilterLink *mainlink = ctx->inputs[0]; > + int ret; > + > + ret = ff_framesync_init_dualinput(&s->fs, ctx); > + if (ret < 0) > + return ret; > + outlink->w = mainlink->w; > + outlink->h = mainlink->h; > + outlink->time_base = mainlink->time_base; > + outlink->sample_aspect_ratio = mainlink->sample_aspect_ratio; > + outlink->frame_rate = mainlink->frame_rate; > + if ((ret = ff_framesync_configure(&s->fs)) < 0) > + return ret; > + > + return 0; > +} > + > +static int activate(AVFilterContext *ctx) > +{ > + PHQMContext *s = ctx->priv; > + return ff_framesync_activate(&s->fs); > +} > + > +static av_cold void uninit(AVFilterContext *ctx) > +{ > + PHQMContext *s = ctx->priv; > + > + if (s->nb_frames > 0) > + av_log(ctx, AV_LOG_WARNING, "PHQM average:%f min:%f max:%f\n", > + s->hd / s->nb_frames, s->min_hd, s->max_hd); > + > + ff_framesync_uninit(&s->fs); > + > + if (s->stats_file && s->stats_file != stdout) > + fclose(s->stats_file); > + av_frame_free(&s->prev_picref); > +} > + > +static const AVFilterPad phqm_inputs[] = { > + { > + .name = "main", > + .type = AVMEDIA_TYPE_VIDEO, > + },{ > + .name = "reference", > + .type = AVMEDIA_TYPE_VIDEO, > + .config_props = config_input_ref, > + }, > + { NULL } > +}; > + > +static const AVFilterPad phqm_outputs[] = { > + { > + .name = "default", > + .type = AVMEDIA_TYPE_VIDEO, > + .config_props = config_output, > + }, > + { NULL } > +}; > + > +AVFilter ff_vf_phqm= { > + .name = "phqm", > + .description = NULL_IF_CONFIG_SMALL("PHQM: Calculate the Perceptual > Hash Hamming Difference between two video streams."), > + .preinit = phqm_framesync_preinit, > + .init = init, > + .uninit = uninit, > + .query_formats = query_formats, > + .activate = activate, > + .priv_size = sizeof(PHQMContext), > + .priv_class = &phqm_class, > + .inputs = phqm_inputs, > + .outputs = phqm_outputs, > +}; > -- > 2.20.1 > > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
Am Sa., 26. Okt. 2019 um 14:00 Uhr schrieb <ckennedy@ellation.com>: > > From: Christopher Kennedy <ckennedy@ellation.com> > > this adds a phqm filter and OpenCV img_hash based resource usable > by the phqm and future filters using image hash functionality > from OpenCV. > C++ to C handling so that full OpenCV functionality and API can > be used instead of the C versions (which are incomplete and > don't always exist). This may not be acceptable, I just wanted to point out an obvious issue. Carl Eugen
This is a reference/encode comparison filter with two files input like the psnr or vmaf filter. So it is completely different and uses the C++ OpenCV API since this img_hash library is not in the C API. It's unique to what the OCV filter does, and has more research implications from my talk at Demuxed 2019. Christopher On Sat, Oct 26, 2019 at 7:38 AM Paul B Mahol <onemda@gmail.com> wrote: > > Why is this not generic filter like already existing opencv filter? > > On 10/26/19, ckennedy@ellation.com <ckennedy@ellation.com> wrote: > > From: Christopher Kennedy <ckennedy@ellation.com> > > > > this adds a phqm filter and OpenCV img_hash based resource usable > > by the phqm and future filters using image hash functionality > > from OpenCV. > > > > C++ to C handling so that full OpenCV functionality and API can > > be used instead of the C versions (which are incomplete and > > don't always exist). > > > > Example command line: > > > > ffmpeg -i encode.mp4 -i reference.mp4 \ > > -filter_complex "[0:v][1:v]phqm=stats_file=out.log" \ > > -y -f null /dev/null > > > > Signed-off-by: Christopher Kennedy <ckennedy@ellation.com> > > --- > > Changelog | 1 + > > configure | 2 + > > libavfilter/Makefile | 2 + > > libavfilter/allfilters.c | 1 + > > libavfilter/img_hash.cpp | 98 ++++++++++++ > > libavfilter/img_hash.h | 46 ++++++ > > libavfilter/vf_phqm.c | 334 +++++++++++++++++++++++++++++++++++++++ > > 7 files changed, 484 insertions(+) > > create mode 100644 libavfilter/img_hash.cpp > > create mode 100644 libavfilter/img_hash.h > > create mode 100644 libavfilter/vf_phqm.c > > > > diff --git a/Changelog b/Changelog > > index 316589e336..4a22f77d37 100644 > > --- a/Changelog > > +++ b/Changelog > > @@ -17,6 +17,7 @@ version <next>: > > - anlms filter > > - arnndn filter > > - bilateral filter > > +- phqm perceptual hash filter using OpenCV img_lib > > > > > > version 4.2: > > diff --git a/configure b/configure > > index 8413826f9e..e231d359bb 100755 > > --- a/configure > > +++ b/configure > > @@ -3497,6 +3497,8 @@ nlmeans_opencl_filter_deps="opencl" > > nnedi_filter_deps="gpl" > > ocr_filter_deps="libtesseract" > > ocv_filter_deps="libopencv" > > +phqm_filter_deps="libopencv" > > +phqm_filter_extralibs="-lstdc++ -lopencv_img_hash" > > openclsrc_filter_deps="opencl" > > overlay_opencl_filter_deps="opencl" > > overlay_qsv_filter_deps="libmfx" > > diff --git a/libavfilter/Makefile b/libavfilter/Makefile > > index 63d2fba861..645e232b3e 100644 > > --- a/libavfilter/Makefile > > +++ b/libavfilter/Makefile > > @@ -325,6 +325,7 @@ OBJS-$(CONFIG_PERMS_FILTER) += > > f_perms.o > > OBJS-$(CONFIG_PERSPECTIVE_FILTER) += vf_perspective.o > > OBJS-$(CONFIG_PHASE_FILTER) += vf_phase.o > > OBJS-$(CONFIG_PHOTOSENSITIVITY_FILTER) += vf_photosensitivity.o > > +OBJS-$(CONFIG_PHQM_FILTER) += vf_phqm.o img_hash.o > > OBJS-$(CONFIG_PIXDESCTEST_FILTER) += vf_pixdesctest.o > > OBJS-$(CONFIG_PIXSCOPE_FILTER) += vf_datascope.o > > OBJS-$(CONFIG_PP_FILTER) += vf_pp.o > > @@ -498,6 +499,7 @@ OBJS-$(CONFIG_SHARED) += > > log2_tab.o > > SKIPHEADERS-$(CONFIG_QSVVPP) += qsvvpp.h > > SKIPHEADERS-$(CONFIG_OPENCL) += opencl.h > > SKIPHEADERS-$(CONFIG_VAAPI) += vaapi_vpp.h > > +SKIPHEADERS-$(CONFIG_LIBOPENCV) += img_hash.h > > > > TOOLS = graph2dot > > TESTPROGS = drawutils filtfmts formats integral > > diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c > > index e4186f93db..f0fcaad235 100644 > > --- a/libavfilter/allfilters.c > > +++ b/libavfilter/allfilters.c > > @@ -309,6 +309,7 @@ extern AVFilter ff_vf_perms; > > extern AVFilter ff_vf_perspective; > > extern AVFilter ff_vf_phase; > > extern AVFilter ff_vf_photosensitivity; > > +extern AVFilter ff_vf_phqm; > > extern AVFilter ff_vf_pixdesctest; > > extern AVFilter ff_vf_pixscope; > > extern AVFilter ff_vf_pp; > > diff --git a/libavfilter/img_hash.cpp b/libavfilter/img_hash.cpp > > new file mode 100644 > > index 0000000000..4d5843da22 > > --- /dev/null > > +++ b/libavfilter/img_hash.cpp > > @@ -0,0 +1,98 @@ > > +/* > > + * Copyright (c) 2019 Christopher Kennedy > > + * > > + * OpenCV img_hash > > + * > > + * This file is part of FFmpeg. > > + * > > + * FFmpeg is free software; you can redistribute it and/or > > + * modify it under the terms of the GNU Lesser General Public > > + * License as published by the Free Software Foundation; either > > + * version 2.1 of the License, or (at your option) any later version. > > + * > > + * FFmpeg is distributed in the hope that it will be useful, > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > + * Lesser General Public License for more details. > > + * > > + * You should have received a copy of the GNU Lesser General Public > > + * License along with FFmpeg; if not, write to the Free Software > > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 > > USA > > + */ > > + > > +#include <opencv2/core.hpp> > > +#include <opencv2/core/ocl.hpp> > > +#include <opencv2/highgui.hpp> > > +#include <opencv2/img_hash.hpp> > > +#include <opencv2/imgproc.hpp> > > + > > +#include <iostream> > > + > > +#include "img_hash.h" > > +#include "libavutil/pixdesc.h" > > +extern "C" { > > +#include "avfilter.h" > > +} > > + > > +// convert from avframe to iplimage format > > +static int fill_iplimage_from_frame(IplImage *img, const AVFrame *frame, > > enum AVPixelFormat pixfmt) > > +{ > > + IplImage *tmpimg; > > + int depth = IPL_DEPTH_8U, channels_nb; > > + > > + switch (pixfmt) { > > + case AV_PIX_FMT_GRAY8: channels_nb = 1; break; > > + case AV_PIX_FMT_BGRA: channels_nb = 4; break; > > + case AV_PIX_FMT_BGR24: channels_nb = 3; break; > > + default: return -1; > > + } > > + > > + tmpimg = cvCreateImageHeader((CvSize){frame->width, frame->height}, > > depth, channels_nb); > > + *img = *tmpimg; > > + img->imageData = img->imageDataOrigin = (char *) frame->data[0]; > > + img->dataOrder = IPL_DATA_ORDER_PIXEL; > > + img->origin = IPL_ORIGIN_TL; > > + img->widthStep = frame->linesize[0]; > > + > > + return 0; > > +} > > + > > +// Get the score of two Video Frames by comparing the perceptual hashes and > > deriving a hamming distance > > +// showing how similar they are or different. lower score is better for > > most algorithms > > +extern "C" double getScore(const AVFrame *frame1, const AVFrame *frame2, > > enum AVPixelFormat pixfmt, int hash_type) { > > + cv::Ptr<cv::img_hash::ImgHashBase> algo; > > + IplImage ipl1, ipl2; > > + cv::Mat h1; > > + cv::Mat h2; > > + cv::Mat m1; > > + cv::Mat m2; > > + > > + // Take FFmpeg video frame and convert into an IplImage for OpenCV > > + if (fill_iplimage_from_frame(&ipl1, frame1, pixfmt) != 0 || > > + fill_iplimage_from_frame(&ipl2, frame2, pixfmt) != 0) > > + return DBL_MAX; // Return an invalid value if either fails > > + > > + // Convert an IplImage to an Mat Image for OpenCV (newer format) > > + m1 = cv::cvarrToMat(&ipl1); > > + m2 = cv::cvarrToMat(&ipl2); > > + > > + // substantiate the hash type algorithm > > + switch (hash_type) { > > + case PHASH: algo = cv::img_hash::PHash::create(); > > break; > > + case AVERAGE: algo = cv::img_hash::AverageHash::create(); > > break; > > + case MARRHILDRETH: algo = > > cv::img_hash::MarrHildrethHash::create(); break; > > + case RADIALVARIANCE: algo = > > cv::img_hash::RadialVarianceHash::create(); break; > > + // BlockMeanHash support mode 0 and mode 1, they associate to > > + // mode 1 and mode 2 of PHash library > > + case BLOCKMEAN1: algo = > > cv::img_hash::BlockMeanHash::create(0); break; > > + case BLOCKMEAN2: algo = > > cv::img_hash::BlockMeanHash::create(1); break; > > + case COLORMOMENT: algo = > > cv::img_hash::ColorMomentHash::create(); break; > > + } > > + > > + // Compute the hash > > + algo->compute(m1, h1); > > + algo->compute(m2, h2); > > + > > + // Compare the hashes and return the hamming distance > > + return algo->compare(h1, h2); > > +} > > diff --git a/libavfilter/img_hash.h b/libavfilter/img_hash.h > > new file mode 100644 > > index 0000000000..76f55c3013 > > --- /dev/null > > +++ b/libavfilter/img_hash.h > > @@ -0,0 +1,46 @@ > > +/* > > + * Copyright (c) 2019 Christopher Kennedy > > + * > > + * PHQM Perceptual Hash Quality Metric > > + * > > + * This file is part of FFmpeg. > > + * > > + * FFmpeg is free software; you can redistribute it and/or > > + * modify it under the terms of the GNU Lesser General Public > > + * License as published by the Free Software Foundation; either > > + * version 2.1 of the License, or (at your option) any later version. > > + * > > + * FFmpeg is distributed in the hope that it will be useful, > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > + * Lesser General Public License for more details. > > + * > > + * You should have received a copy of the GNU Lesser General Public > > + * License along with FFmpeg; if not, write to the Free Software > > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 > > USA > > + */ > > + > > +#ifndef AVFILTER_IMG_HASH_H > > +#define AVFILTER_IMG_HASH_H > > + > > +#include "avfilter.h" > > + > > +#if defined(__cplusplus) > > +extern "C" > > +{ > > +#endif > > + > > +#define AVERAGE 0 > > +#define BLOCKMEAN1 1 > > +#define BLOCKMEAN2 2 > > +#define COLORMOMENT 3 > > +#define MARRHILDRETH 4 > > +#define PHASH 5 > > +#define RADIALVARIANCE 6 > > + > > +double getScore(const AVFrame *frame1, const AVFrame *frame2, enum > > AVPixelFormat pixfmt, int hash_type); > > +#if defined(__cplusplus) > > +} > > +#endif > > + > > +#endif > > diff --git a/libavfilter/vf_phqm.c b/libavfilter/vf_phqm.c > > new file mode 100644 > > index 0000000000..0930386b10 > > --- /dev/null > > +++ b/libavfilter/vf_phqm.c > > @@ -0,0 +1,334 @@ > > +/* > > + * Copyright (c) 2019 Christopher Kennedy > > + * > > + * PHQM Perceptual Hash Quality Metric > > + * > > + * This file is part of FFmpeg. > > + * > > + * FFmpeg is free software; you can redistribute it and/or > > + * modify it under the terms of the GNU Lesser General Public > > + * License as published by the Free Software Foundation; either > > + * version 2.1 of the License, or (at your option) any later version. > > + * > > + * FFmpeg is distributed in the hope that it will be useful, > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > + * Lesser General Public License for more details. > > + * > > + * You should have received a copy of the GNU Lesser General Public > > + * License along with FFmpeg; if not, write to the Free Software > > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 > > USA > > + */ > > + > > +/** > > + * @file > > + * PHQM: Calculate the Image Hash Hamming Difference between two input > > videos. > > + */ > > + > > +#include <float.h> > > +#include "libavutil/avstring.h" > > +#include "libavutil/opt.h" > > +#include "libavutil/pixdesc.h" > > +#include "avfilter.h" > > +#include "drawutils.h" > > +#include "formats.h" > > +#include "framesync.h" > > +#include "internal.h" > > +#include "video.h" > > + > > +#include "img_hash.h" > > +#include "scene_sad.h" > > + > > +typedef struct PHQMContext { > > + const AVClass *class; > > + FFFrameSync fs; > > + double shd, hd, min_hd, max_hd, smin_hd, smax_hd; > > + uint64_t nb_shd; > > + uint64_t nb_frames; > > + FILE *stats_file; > > + char *stats_file_str; > > + int hash_type; > > + ff_scene_sad_fn sad; ///< Sum of the absolute difference > > function (scene detect only) > > + double prev_mafd; ///< previous MAFD > > (scene detect only) > > + AVFrame *prev_picref; ///< previous frame > > (scene detect only) > > + double scd_thresh; > > + double scene_score; > > +} PHQMContext; > > + > > +#define OFFSET(x) offsetof(PHQMContext, x) > > +#define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM > > + > > +static const AVOption phqm_options[] = { > > + { "stats_file", "Set file where to store per-frame difference > > information.", OFFSET(stats_file_str), AV_OPT_TYPE_STRING, {.str=NULL}, 0, > > 0, FLAGS }, > > + { "f", "Set file where to store per-frame difference > > information.", OFFSET(stats_file_str), AV_OPT_TYPE_STRING, {.str=NULL}, 0, > > 0, FLAGS }, > > + { "scd_thresh", "Scene Change Detection Threshold.", > > OFFSET(scd_thresh), AV_OPT_TYPE_DOUBLE, {.dbl=0.5}, 0, 1, FLAGS }, > > + { "hash_type", "Type of Image Hash to use from OpenCV.", > > OFFSET(hash_type), AV_OPT_TYPE_INT, {.i64 = PHASH}, 0, 6, > > FLAGS, "hash_type" }, > > + { "average", "Average Hash", 0, > > AV_OPT_TYPE_CONST, {.i64 = AVERAGE}, 0, 0, FLAGS, "hash_type" }, > > + { "blockmean1", "Block Mean Hash 1", 0, > > AV_OPT_TYPE_CONST, {.i64 = BLOCKMEAN1}, 0, 0, FLAGS, "hash_type" }, > > + { "blockmean2", "Block Mean Hash 2", 0, > > AV_OPT_TYPE_CONST, {.i64 = BLOCKMEAN2}, 0, 0, FLAGS, "hash_type" }, > > + { "colormoment", "Color Moment Hash", 0, > > AV_OPT_TYPE_CONST, {.i64 = COLORMOMENT}, 0, 0, FLAGS, "hash_type" }, > > + { "marrhildreth", "Marr Hildreth Hash", 0, > > AV_OPT_TYPE_CONST, {.i64 = MARRHILDRETH}, 0, 0, FLAGS, "hash_type" }, > > + { "phash", "Perceptual Hash (PHash)", 0, > > AV_OPT_TYPE_CONST, {.i64 = PHASH}, 0, 0, FLAGS, "hash_type" }, > > + { "radialvariance", "Radial Variance Hash", 0, > > AV_OPT_TYPE_CONST, {.i64 = RADIALVARIANCE}, 0, 0, FLAGS, "hash_type" }, > > + { NULL } > > +}; > > + > > +FRAMESYNC_DEFINE_CLASS(phqm, PHQMContext, fs); > > + > > +static void set_meta(AVDictionary **metadata, const char *key, char comp, > > float d) > > +{ > > + char value[128]; > > + snprintf(value, sizeof(value), "%0.2f", d); > > + if (comp) { > > + char key2[128]; > > + snprintf(key2, sizeof(key2), "%s%c", key, comp); > > + av_dict_set(metadata, key2, value, 0); > > + } else { > > + av_dict_set(metadata, key, value, 0); > > + } > > +} > > + > > +static double get_scene_score(AVFilterContext *ctx, AVFrame *frame) > > +{ > > + double ret = 0.; > > + PHQMContext *s = ctx->priv; > > + AVFrame *prev_picref = s->prev_picref; > > + > > + if (prev_picref && > > + frame->height == prev_picref->height && > > + frame->width == prev_picref->width) { > > + uint64_t sad; > > + double mafd, diff; > > + > > + s->sad(prev_picref->data[0], prev_picref->linesize[0], > > frame->data[0], frame->linesize[0], frame->width * 3, frame->height, &sad); > > + emms_c(); > > + mafd = (double)sad / (frame->width * 3 * frame->height); > > + diff = fabs(mafd - s->prev_mafd); > > + ret = av_clipf(FFMIN(mafd, diff) / 100., 0, 1); > > + s->prev_mafd = mafd; > > + av_frame_free(&prev_picref); > > + } > > + s->prev_picref = av_frame_clone(frame); > > + return ret; > > +} > > + > > +static int do_phqm(FFFrameSync *fs) > > +{ > > + AVFilterContext *ctx = fs->parent; > > + PHQMContext *s = ctx->priv; > > + AVFrame *master, *ref; > > + double hd = 0.; > > + int ret; > > + double hd_limit = 1000000.; > > + AVDictionary **metadata; > > + > > + ret = ff_framesync_dualinput_get(fs, &master, &ref); > > + if (ret < 0) > > + return ret; > > + if (!ref) > > + return ff_filter_frame(ctx->outputs[0], master); > > + metadata = &master->metadata; > > + > > + > > + s->nb_frames++; > > + > > + /* scene change detection score */ > > + s->scene_score = get_scene_score(ctx, ref); > > + if (s->scene_score >= s->scd_thresh && s->nb_shd >= 48) { > > + av_log(s, AV_LOG_INFO, "ImgHashScene: n:%"PRId64"-%"PRId64" > > hd_avg:%0.3lf hd_min:%0.3lf hd_max:%0.3lf scd:%0.2lf\n", > > + (s->nb_frames - s->nb_shd), s->nb_frames - 1, (s->shd / > > s->nb_shd), s->smin_hd, s->smax_hd, s->scene_score); > > + s->shd = 0.; > > + s->nb_shd = 0; > > + s->smin_hd = 0.; > > + s->smax_hd = 0.; > > + } > > + > > + /* limit the highest value so we cut off at perceptual difference match > > */ > > + switch (s->hash_type) { > > + case PHASH: > > + case AVERAGE: hd_limit = 5; break; > > + case MARRHILDRETH: hd_limit = 30; break; > > + case RADIALVARIANCE: hd_limit = 0.9; break; > > + case BLOCKMEAN1: hd_limit = 12; break; > > + case BLOCKMEAN2: hd_limit = 48; break; > > + case COLORMOMENT: hd_limit = 8; break; > > + } > > + > > + /* get ref / enc perceptual hashes and calc hamming distance difference > > value */ > > + hd = getScore(ref, master, ref->format, s->hash_type); > > + if (hd == DBL_MAX) { > > + av_log(s, AV_LOG_ERROR, "Failure with handling pix_fmt of AVFrame > > for conversion to IPLimage.\n"); > > + return AVERROR(EINVAL); > > + } > > + s->hd += FFMIN(hd, hd_limit); > > + set_meta(metadata, "lavfi.phqm.phqm", 0, hd); > > + > > + /* scene hamming distance avg */ > > + s->shd += FFMIN(hd, hd_limit); > > + s->nb_shd++; > > + av_log(s, AV_LOG_DEBUG, "ImgHashFrame: hd:%0.3lf scd:%0.2lf\n", hd, > > s->scene_score); > > + > > + s->min_hd = FFMIN(s->min_hd, hd); > > + s->max_hd = FFMAX(s->max_hd, hd); > > + s->smin_hd = FFMIN(s->smin_hd, hd); > > + s->smax_hd = FFMAX(s->smax_hd, hd); > > + > > + if (s->stats_file) { > > + fprintf(s->stats_file, > > + "n:%"PRId64" phqm:%0.3f phqm_min:%0.3f phqm_max:%0.3f > > sad:%0.2f", > > + s->nb_frames, hd, s->min_hd, s->max_hd, s->scene_score); > > + fprintf(s->stats_file, "\n"); > > + } > > + > > + return ff_filter_frame(ctx->outputs[0], master); > > +} > > + > > +static av_cold int init(AVFilterContext *ctx) > > +{ > > + PHQMContext *s = ctx->priv; > > + > > + if (s->stats_file_str) { > > + if (!strcmp(s->stats_file_str, "-")) { > > + s->stats_file = stdout; > > + } else { > > + s->stats_file = fopen(s->stats_file_str, "w"); > > + if (!s->stats_file) { > > + int err = AVERROR(errno); > > + char buf[128]; > > + av_strerror(err, buf, sizeof(buf)); > > + av_log(ctx, AV_LOG_ERROR, "Could not open stats file %s: > > %s\n", > > + s->stats_file_str, buf); > > + return err; > > + } > > + } > > + } > > + > > + s->sad = ff_scene_sad_get_fn(8); > > + if (!s->sad) > > + return AVERROR(EINVAL); > > + > > + s->fs.on_event = do_phqm; > > + return 0; > > +} > > + > > +static int query_formats(AVFilterContext *ctx) > > +{ > > + PHQMContext *s = ctx->priv; > > + AVFilterFormats *fmts_list = NULL; > > + static const enum AVPixelFormat gray8_pix_fmts[] = { > > + AV_PIX_FMT_GRAY8, > > + AV_PIX_FMT_NONE > > + }; > > + static const enum AVPixelFormat bgr24_pix_fmts[] = { > > + AV_PIX_FMT_BGR24, > > + AV_PIX_FMT_NONE > > + }; > > + static const enum AVPixelFormat bgra_pix_fmts[] = { > > + AV_PIX_FMT_BGRA, > > + AV_PIX_FMT_NONE > > + }; > > + > > + switch (s->hash_type) { > > + case COLORMOMENT: fmts_list = ff_make_format_list(bgr24_pix_fmts); > > break; > > + case MARRHILDRETH: fmts_list = ff_make_format_list(bgra_pix_fmts); > > break; > > + /* all other hashes take the gray8 format */ > > + default: fmts_list = ff_make_format_list(gray8_pix_fmts); break; > > + } > > + if (!fmts_list) > > + return AVERROR(ENOMEM); > > + return ff_set_common_formats(ctx, fmts_list); > > +} > > + > > +static int config_input_ref(AVFilterLink *inlink) > > +{ > > + AVFilterContext *ctx = inlink->dst; > > + > > + if (ctx->inputs[0]->w != ctx->inputs[1]->w || > > + ctx->inputs[0]->h != ctx->inputs[1]->h) { > > + av_log(ctx, AV_LOG_ERROR, "Width and height of input videos must be > > same.\n"); > > + return AVERROR(EINVAL); > > + } > > + if (ctx->inputs[0]->format != ctx->inputs[1]->format) { > > + av_log(ctx, AV_LOG_ERROR, "Inputs must be of same pixel > > format.\n"); > > + return AVERROR(EINVAL); > > + } > > + > > + return 0; > > +} > > + > > +static int config_output(AVFilterLink *outlink) > > +{ > > + AVFilterContext *ctx = outlink->src; > > + PHQMContext *s = ctx->priv; > > + AVFilterLink *mainlink = ctx->inputs[0]; > > + int ret; > > + > > + ret = ff_framesync_init_dualinput(&s->fs, ctx); > > + if (ret < 0) > > + return ret; > > + outlink->w = mainlink->w; > > + outlink->h = mainlink->h; > > + outlink->time_base = mainlink->time_base; > > + outlink->sample_aspect_ratio = mainlink->sample_aspect_ratio; > > + outlink->frame_rate = mainlink->frame_rate; > > + if ((ret = ff_framesync_configure(&s->fs)) < 0) > > + return ret; > > + > > + return 0; > > +} > > + > > +static int activate(AVFilterContext *ctx) > > +{ > > + PHQMContext *s = ctx->priv; > > + return ff_framesync_activate(&s->fs); > > +} > > + > > +static av_cold void uninit(AVFilterContext *ctx) > > +{ > > + PHQMContext *s = ctx->priv; > > + > > + if (s->nb_frames > 0) > > + av_log(ctx, AV_LOG_WARNING, "PHQM average:%f min:%f max:%f\n", > > + s->hd / s->nb_frames, s->min_hd, s->max_hd); > > + > > + ff_framesync_uninit(&s->fs); > > + > > + if (s->stats_file && s->stats_file != stdout) > > + fclose(s->stats_file); > > + av_frame_free(&s->prev_picref); > > +} > > + > > +static const AVFilterPad phqm_inputs[] = { > > + { > > + .name = "main", > > + .type = AVMEDIA_TYPE_VIDEO, > > + },{ > > + .name = "reference", > > + .type = AVMEDIA_TYPE_VIDEO, > > + .config_props = config_input_ref, > > + }, > > + { NULL } > > +}; > > + > > +static const AVFilterPad phqm_outputs[] = { > > + { > > + .name = "default", > > + .type = AVMEDIA_TYPE_VIDEO, > > + .config_props = config_output, > > + }, > > + { NULL } > > +}; > > + > > +AVFilter ff_vf_phqm= { > > + .name = "phqm", > > + .description = NULL_IF_CONFIG_SMALL("PHQM: Calculate the Perceptual > > Hash Hamming Difference between two video streams."), > > + .preinit = phqm_framesync_preinit, > > + .init = init, > > + .uninit = uninit, > > + .query_formats = query_formats, > > + .activate = activate, > > + .priv_size = sizeof(PHQMContext), > > + .priv_class = &phqm_class, > > + .inputs = phqm_inputs, > > + .outputs = phqm_outputs, > > +}; > > -- > > 2.20.1 > > > > _______________________________________________ > > ffmpeg-devel mailing list > > ffmpeg-devel@ffmpeg.org > > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > > > To unsubscribe, visit link above, or email > > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
On Sat, Oct 26, 2019 at 7:39 AM Carl Eugen Hoyos <ceffmpeg@gmail.com> wrote: > > Am Sa., 26. Okt. 2019 um 14:00 Uhr schrieb <ckennedy@ellation.com>: > > > > From: Christopher Kennedy <ckennedy@ellation.com> > > > > this adds a phqm filter and OpenCV img_hash based resource usable > > by the phqm and future filters using image hash functionality > > from OpenCV. > > > C++ to C handling so that full OpenCV functionality and API can > > be used instead of the C versions (which are incomplete and > > don't always exist). > > This may not be acceptable, I just wanted to point out an obvious > issue. I will do whatever is necessary, unless completely impossible to accept the use of OpenCV directly and the img_hash C++ lib that isn't available any other way. It is better to be in FFmpeg main in mine and others' opinions. So any help in understanding how to get it right is greatly appreciated. Thanks, Christopher > > Carl Eugen > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
On 10/26/19, Christopher Kennedy <ckennedy@ellation.com> wrote: > This is a reference/encode comparison filter with two files input like > the psnr or vmaf filter. > So it is completely different and uses the C++ OpenCV API since this > img_hash library is not in the C API. > It's unique to what the OCV filter does, and has more research > implications from my talk at Demuxed 2019. I do not see how that is relevant. There should be generic opencv filter which could do this above in generic way, and not by adding yet another filter that uses only some part of opencv. > > Christopher > > On Sat, Oct 26, 2019 at 7:38 AM Paul B Mahol <onemda@gmail.com> wrote: >> >> Why is this not generic filter like already existing opencv filter? >> >> On 10/26/19, ckennedy@ellation.com <ckennedy@ellation.com> wrote: >> > From: Christopher Kennedy <ckennedy@ellation.com> >> > >> > this adds a phqm filter and OpenCV img_hash based resource usable >> > by the phqm and future filters using image hash functionality >> > from OpenCV. >> > >> > C++ to C handling so that full OpenCV functionality and API can >> > be used instead of the C versions (which are incomplete and >> > don't always exist). >> > >> > Example command line: >> > >> > ffmpeg -i encode.mp4 -i reference.mp4 \ >> > -filter_complex "[0:v][1:v]phqm=stats_file=out.log" \ >> > -y -f null /dev/null >> > >> > Signed-off-by: Christopher Kennedy <ckennedy@ellation.com> >> > --- >> > Changelog | 1 + >> > configure | 2 + >> > libavfilter/Makefile | 2 + >> > libavfilter/allfilters.c | 1 + >> > libavfilter/img_hash.cpp | 98 ++++++++++++ >> > libavfilter/img_hash.h | 46 ++++++ >> > libavfilter/vf_phqm.c | 334 +++++++++++++++++++++++++++++++++++++++ >> > 7 files changed, 484 insertions(+) >> > create mode 100644 libavfilter/img_hash.cpp >> > create mode 100644 libavfilter/img_hash.h >> > create mode 100644 libavfilter/vf_phqm.c >> > >> > diff --git a/Changelog b/Changelog >> > index 316589e336..4a22f77d37 100644 >> > --- a/Changelog >> > +++ b/Changelog >> > @@ -17,6 +17,7 @@ version <next>: >> > - anlms filter >> > - arnndn filter >> > - bilateral filter >> > +- phqm perceptual hash filter using OpenCV img_lib >> > >> > >> > version 4.2: >> > diff --git a/configure b/configure >> > index 8413826f9e..e231d359bb 100755 >> > --- a/configure >> > +++ b/configure >> > @@ -3497,6 +3497,8 @@ nlmeans_opencl_filter_deps="opencl" >> > nnedi_filter_deps="gpl" >> > ocr_filter_deps="libtesseract" >> > ocv_filter_deps="libopencv" >> > +phqm_filter_deps="libopencv" >> > +phqm_filter_extralibs="-lstdc++ -lopencv_img_hash" >> > openclsrc_filter_deps="opencl" >> > overlay_opencl_filter_deps="opencl" >> > overlay_qsv_filter_deps="libmfx" >> > diff --git a/libavfilter/Makefile b/libavfilter/Makefile >> > index 63d2fba861..645e232b3e 100644 >> > --- a/libavfilter/Makefile >> > +++ b/libavfilter/Makefile >> > @@ -325,6 +325,7 @@ OBJS-$(CONFIG_PERMS_FILTER) += >> > f_perms.o >> > OBJS-$(CONFIG_PERSPECTIVE_FILTER) += vf_perspective.o >> > OBJS-$(CONFIG_PHASE_FILTER) += vf_phase.o >> > OBJS-$(CONFIG_PHOTOSENSITIVITY_FILTER) += vf_photosensitivity.o >> > +OBJS-$(CONFIG_PHQM_FILTER) += vf_phqm.o img_hash.o >> > OBJS-$(CONFIG_PIXDESCTEST_FILTER) += vf_pixdesctest.o >> > OBJS-$(CONFIG_PIXSCOPE_FILTER) += vf_datascope.o >> > OBJS-$(CONFIG_PP_FILTER) += vf_pp.o >> > @@ -498,6 +499,7 @@ OBJS-$(CONFIG_SHARED) += >> > log2_tab.o >> > SKIPHEADERS-$(CONFIG_QSVVPP) += qsvvpp.h >> > SKIPHEADERS-$(CONFIG_OPENCL) += opencl.h >> > SKIPHEADERS-$(CONFIG_VAAPI) += vaapi_vpp.h >> > +SKIPHEADERS-$(CONFIG_LIBOPENCV) += img_hash.h >> > >> > TOOLS = graph2dot >> > TESTPROGS = drawutils filtfmts formats integral >> > diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c >> > index e4186f93db..f0fcaad235 100644 >> > --- a/libavfilter/allfilters.c >> > +++ b/libavfilter/allfilters.c >> > @@ -309,6 +309,7 @@ extern AVFilter ff_vf_perms; >> > extern AVFilter ff_vf_perspective; >> > extern AVFilter ff_vf_phase; >> > extern AVFilter ff_vf_photosensitivity; >> > +extern AVFilter ff_vf_phqm; >> > extern AVFilter ff_vf_pixdesctest; >> > extern AVFilter ff_vf_pixscope; >> > extern AVFilter ff_vf_pp; >> > diff --git a/libavfilter/img_hash.cpp b/libavfilter/img_hash.cpp >> > new file mode 100644 >> > index 0000000000..4d5843da22 >> > --- /dev/null >> > +++ b/libavfilter/img_hash.cpp >> > @@ -0,0 +1,98 @@ >> > +/* >> > + * Copyright (c) 2019 Christopher Kennedy >> > + * >> > + * OpenCV img_hash >> > + * >> > + * This file is part of FFmpeg. >> > + * >> > + * FFmpeg is free software; you can redistribute it and/or >> > + * modify it under the terms of the GNU Lesser General Public >> > + * License as published by the Free Software Foundation; either >> > + * version 2.1 of the License, or (at your option) any later version. >> > + * >> > + * FFmpeg is distributed in the hope that it will be useful, >> > + * but WITHOUT ANY WARRANTY; without even the implied warranty of >> > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU >> > + * Lesser General Public License for more details. >> > + * >> > + * You should have received a copy of the GNU Lesser General Public >> > + * License along with FFmpeg; if not, write to the Free Software >> > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA >> > 02110-1301 >> > USA >> > + */ >> > + >> > +#include <opencv2/core.hpp> >> > +#include <opencv2/core/ocl.hpp> >> > +#include <opencv2/highgui.hpp> >> > +#include <opencv2/img_hash.hpp> >> > +#include <opencv2/imgproc.hpp> >> > + >> > +#include <iostream> >> > + >> > +#include "img_hash.h" >> > +#include "libavutil/pixdesc.h" >> > +extern "C" { >> > +#include "avfilter.h" >> > +} >> > + >> > +// convert from avframe to iplimage format >> > +static int fill_iplimage_from_frame(IplImage *img, const AVFrame >> > *frame, >> > enum AVPixelFormat pixfmt) >> > +{ >> > + IplImage *tmpimg; >> > + int depth = IPL_DEPTH_8U, channels_nb; >> > + >> > + switch (pixfmt) { >> > + case AV_PIX_FMT_GRAY8: channels_nb = 1; break; >> > + case AV_PIX_FMT_BGRA: channels_nb = 4; break; >> > + case AV_PIX_FMT_BGR24: channels_nb = 3; break; >> > + default: return -1; >> > + } >> > + >> > + tmpimg = cvCreateImageHeader((CvSize){frame->width, >> > frame->height}, >> > depth, channels_nb); >> > + *img = *tmpimg; >> > + img->imageData = img->imageDataOrigin = (char *) frame->data[0]; >> > + img->dataOrder = IPL_DATA_ORDER_PIXEL; >> > + img->origin = IPL_ORIGIN_TL; >> > + img->widthStep = frame->linesize[0]; >> > + >> > + return 0; >> > +} >> > + >> > +// Get the score of two Video Frames by comparing the perceptual hashes >> > and >> > deriving a hamming distance >> > +// showing how similar they are or different. lower score is better >> > for >> > most algorithms >> > +extern "C" double getScore(const AVFrame *frame1, const AVFrame >> > *frame2, >> > enum AVPixelFormat pixfmt, int hash_type) { >> > + cv::Ptr<cv::img_hash::ImgHashBase> algo; >> > + IplImage ipl1, ipl2; >> > + cv::Mat h1; >> > + cv::Mat h2; >> > + cv::Mat m1; >> > + cv::Mat m2; >> > + >> > + // Take FFmpeg video frame and convert into an IplImage for OpenCV >> > + if (fill_iplimage_from_frame(&ipl1, frame1, pixfmt) != 0 || >> > + fill_iplimage_from_frame(&ipl2, frame2, pixfmt) != 0) >> > + return DBL_MAX; // Return an invalid value if either fails >> > + >> > + // Convert an IplImage to an Mat Image for OpenCV (newer format) >> > + m1 = cv::cvarrToMat(&ipl1); >> > + m2 = cv::cvarrToMat(&ipl2); >> > + >> > + // substantiate the hash type algorithm >> > + switch (hash_type) { >> > + case PHASH: algo = cv::img_hash::PHash::create(); >> > break; >> > + case AVERAGE: algo = >> > cv::img_hash::AverageHash::create(); >> > break; >> > + case MARRHILDRETH: algo = >> > cv::img_hash::MarrHildrethHash::create(); break; >> > + case RADIALVARIANCE: algo = >> > cv::img_hash::RadialVarianceHash::create(); break; >> > + // BlockMeanHash support mode 0 and mode 1, they associate to >> > + // mode 1 and mode 2 of PHash library >> > + case BLOCKMEAN1: algo = >> > cv::img_hash::BlockMeanHash::create(0); break; >> > + case BLOCKMEAN2: algo = >> > cv::img_hash::BlockMeanHash::create(1); break; >> > + case COLORMOMENT: algo = >> > cv::img_hash::ColorMomentHash::create(); break; >> > + } >> > + >> > + // Compute the hash >> > + algo->compute(m1, h1); >> > + algo->compute(m2, h2); >> > + >> > + // Compare the hashes and return the hamming distance >> > + return algo->compare(h1, h2); >> > +} >> > diff --git a/libavfilter/img_hash.h b/libavfilter/img_hash.h >> > new file mode 100644 >> > index 0000000000..76f55c3013 >> > --- /dev/null >> > +++ b/libavfilter/img_hash.h >> > @@ -0,0 +1,46 @@ >> > +/* >> > + * Copyright (c) 2019 Christopher Kennedy >> > + * >> > + * PHQM Perceptual Hash Quality Metric >> > + * >> > + * This file is part of FFmpeg. >> > + * >> > + * FFmpeg is free software; you can redistribute it and/or >> > + * modify it under the terms of the GNU Lesser General Public >> > + * License as published by the Free Software Foundation; either >> > + * version 2.1 of the License, or (at your option) any later version. >> > + * >> > + * FFmpeg is distributed in the hope that it will be useful, >> > + * but WITHOUT ANY WARRANTY; without even the implied warranty of >> > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU >> > + * Lesser General Public License for more details. >> > + * >> > + * You should have received a copy of the GNU Lesser General Public >> > + * License along with FFmpeg; if not, write to the Free Software >> > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA >> > 02110-1301 >> > USA >> > + */ >> > + >> > +#ifndef AVFILTER_IMG_HASH_H >> > +#define AVFILTER_IMG_HASH_H >> > + >> > +#include "avfilter.h" >> > + >> > +#if defined(__cplusplus) >> > +extern "C" >> > +{ >> > +#endif >> > + >> > +#define AVERAGE 0 >> > +#define BLOCKMEAN1 1 >> > +#define BLOCKMEAN2 2 >> > +#define COLORMOMENT 3 >> > +#define MARRHILDRETH 4 >> > +#define PHASH 5 >> > +#define RADIALVARIANCE 6 >> > + >> > +double getScore(const AVFrame *frame1, const AVFrame *frame2, enum >> > AVPixelFormat pixfmt, int hash_type); >> > +#if defined(__cplusplus) >> > +} >> > +#endif >> > + >> > +#endif >> > diff --git a/libavfilter/vf_phqm.c b/libavfilter/vf_phqm.c >> > new file mode 100644 >> > index 0000000000..0930386b10 >> > --- /dev/null >> > +++ b/libavfilter/vf_phqm.c >> > @@ -0,0 +1,334 @@ >> > +/* >> > + * Copyright (c) 2019 Christopher Kennedy >> > + * >> > + * PHQM Perceptual Hash Quality Metric >> > + * >> > + * This file is part of FFmpeg. >> > + * >> > + * FFmpeg is free software; you can redistribute it and/or >> > + * modify it under the terms of the GNU Lesser General Public >> > + * License as published by the Free Software Foundation; either >> > + * version 2.1 of the License, or (at your option) any later version. >> > + * >> > + * FFmpeg is distributed in the hope that it will be useful, >> > + * but WITHOUT ANY WARRANTY; without even the implied warranty of >> > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU >> > + * Lesser General Public License for more details. >> > + * >> > + * You should have received a copy of the GNU Lesser General Public >> > + * License along with FFmpeg; if not, write to the Free Software >> > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA >> > 02110-1301 >> > USA >> > + */ >> > + >> > +/** >> > + * @file >> > + * PHQM: Calculate the Image Hash Hamming Difference between two input >> > videos. >> > + */ >> > + >> > +#include <float.h> >> > +#include "libavutil/avstring.h" >> > +#include "libavutil/opt.h" >> > +#include "libavutil/pixdesc.h" >> > +#include "avfilter.h" >> > +#include "drawutils.h" >> > +#include "formats.h" >> > +#include "framesync.h" >> > +#include "internal.h" >> > +#include "video.h" >> > + >> > +#include "img_hash.h" >> > +#include "scene_sad.h" >> > + >> > +typedef struct PHQMContext { >> > + const AVClass *class; >> > + FFFrameSync fs; >> > + double shd, hd, min_hd, max_hd, smin_hd, smax_hd; >> > + uint64_t nb_shd; >> > + uint64_t nb_frames; >> > + FILE *stats_file; >> > + char *stats_file_str; >> > + int hash_type; >> > + ff_scene_sad_fn sad; ///< Sum of the absolute >> > difference >> > function (scene detect only) >> > + double prev_mafd; ///< previous MAFD >> > (scene detect only) >> > + AVFrame *prev_picref; ///< previous frame >> > (scene detect only) >> > + double scd_thresh; >> > + double scene_score; >> > +} PHQMContext; >> > + >> > +#define OFFSET(x) offsetof(PHQMContext, x) >> > +#define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM >> > + >> > +static const AVOption phqm_options[] = { >> > + { "stats_file", "Set file where to store per-frame difference >> > information.", OFFSET(stats_file_str), AV_OPT_TYPE_STRING, {.str=NULL}, >> > 0, >> > 0, FLAGS }, >> > + { "f", "Set file where to store per-frame difference >> > information.", OFFSET(stats_file_str), AV_OPT_TYPE_STRING, {.str=NULL}, >> > 0, >> > 0, FLAGS }, >> > + { "scd_thresh", "Scene Change Detection Threshold.", >> > OFFSET(scd_thresh), AV_OPT_TYPE_DOUBLE, {.dbl=0.5}, 0, 1, >> > FLAGS }, >> > + { "hash_type", "Type of Image Hash to use from OpenCV.", >> > OFFSET(hash_type), AV_OPT_TYPE_INT, {.i64 = PHASH}, 0, 6, >> > FLAGS, "hash_type" }, >> > + { "average", "Average Hash", 0, >> > AV_OPT_TYPE_CONST, {.i64 = AVERAGE}, 0, 0, FLAGS, "hash_type" }, >> > + { "blockmean1", "Block Mean Hash 1", 0, >> > AV_OPT_TYPE_CONST, {.i64 = BLOCKMEAN1}, 0, 0, FLAGS, "hash_type" }, >> > + { "blockmean2", "Block Mean Hash 2", 0, >> > AV_OPT_TYPE_CONST, {.i64 = BLOCKMEAN2}, 0, 0, FLAGS, "hash_type" }, >> > + { "colormoment", "Color Moment Hash", 0, >> > AV_OPT_TYPE_CONST, {.i64 = COLORMOMENT}, 0, 0, FLAGS, "hash_type" }, >> > + { "marrhildreth", "Marr Hildreth Hash", 0, >> > AV_OPT_TYPE_CONST, {.i64 = MARRHILDRETH}, 0, 0, FLAGS, "hash_type" }, >> > + { "phash", "Perceptual Hash (PHash)", 0, >> > AV_OPT_TYPE_CONST, {.i64 = PHASH}, 0, 0, FLAGS, "hash_type" }, >> > + { "radialvariance", "Radial Variance Hash", 0, >> > AV_OPT_TYPE_CONST, {.i64 = RADIALVARIANCE}, 0, 0, FLAGS, "hash_type" }, >> > + { NULL } >> > +}; >> > + >> > +FRAMESYNC_DEFINE_CLASS(phqm, PHQMContext, fs); >> > + >> > +static void set_meta(AVDictionary **metadata, const char *key, char >> > comp, >> > float d) >> > +{ >> > + char value[128]; >> > + snprintf(value, sizeof(value), "%0.2f", d); >> > + if (comp) { >> > + char key2[128]; >> > + snprintf(key2, sizeof(key2), "%s%c", key, comp); >> > + av_dict_set(metadata, key2, value, 0); >> > + } else { >> > + av_dict_set(metadata, key, value, 0); >> > + } >> > +} >> > + >> > +static double get_scene_score(AVFilterContext *ctx, AVFrame *frame) >> > +{ >> > + double ret = 0.; >> > + PHQMContext *s = ctx->priv; >> > + AVFrame *prev_picref = s->prev_picref; >> > + >> > + if (prev_picref && >> > + frame->height == prev_picref->height && >> > + frame->width == prev_picref->width) { >> > + uint64_t sad; >> > + double mafd, diff; >> > + >> > + s->sad(prev_picref->data[0], prev_picref->linesize[0], >> > frame->data[0], frame->linesize[0], frame->width * 3, frame->height, >> > &sad); >> > + emms_c(); >> > + mafd = (double)sad / (frame->width * 3 * frame->height); >> > + diff = fabs(mafd - s->prev_mafd); >> > + ret = av_clipf(FFMIN(mafd, diff) / 100., 0, 1); >> > + s->prev_mafd = mafd; >> > + av_frame_free(&prev_picref); >> > + } >> > + s->prev_picref = av_frame_clone(frame); >> > + return ret; >> > +} >> > + >> > +static int do_phqm(FFFrameSync *fs) >> > +{ >> > + AVFilterContext *ctx = fs->parent; >> > + PHQMContext *s = ctx->priv; >> > + AVFrame *master, *ref; >> > + double hd = 0.; >> > + int ret; >> > + double hd_limit = 1000000.; >> > + AVDictionary **metadata; >> > + >> > + ret = ff_framesync_dualinput_get(fs, &master, &ref); >> > + if (ret < 0) >> > + return ret; >> > + if (!ref) >> > + return ff_filter_frame(ctx->outputs[0], master); >> > + metadata = &master->metadata; >> > + >> > + >> > + s->nb_frames++; >> > + >> > + /* scene change detection score */ >> > + s->scene_score = get_scene_score(ctx, ref); >> > + if (s->scene_score >= s->scd_thresh && s->nb_shd >= 48) { >> > + av_log(s, AV_LOG_INFO, "ImgHashScene: n:%"PRId64"-%"PRId64" >> > hd_avg:%0.3lf hd_min:%0.3lf hd_max:%0.3lf scd:%0.2lf\n", >> > + (s->nb_frames - s->nb_shd), s->nb_frames - 1, (s->shd / >> > s->nb_shd), s->smin_hd, s->smax_hd, s->scene_score); >> > + s->shd = 0.; >> > + s->nb_shd = 0; >> > + s->smin_hd = 0.; >> > + s->smax_hd = 0.; >> > + } >> > + >> > + /* limit the highest value so we cut off at perceptual difference >> > match >> > */ >> > + switch (s->hash_type) { >> > + case PHASH: >> > + case AVERAGE: hd_limit = 5; break; >> > + case MARRHILDRETH: hd_limit = 30; break; >> > + case RADIALVARIANCE: hd_limit = 0.9; break; >> > + case BLOCKMEAN1: hd_limit = 12; break; >> > + case BLOCKMEAN2: hd_limit = 48; break; >> > + case COLORMOMENT: hd_limit = 8; break; >> > + } >> > + >> > + /* get ref / enc perceptual hashes and calc hamming distance >> > difference >> > value */ >> > + hd = getScore(ref, master, ref->format, s->hash_type); >> > + if (hd == DBL_MAX) { >> > + av_log(s, AV_LOG_ERROR, "Failure with handling pix_fmt of >> > AVFrame >> > for conversion to IPLimage.\n"); >> > + return AVERROR(EINVAL); >> > + } >> > + s->hd += FFMIN(hd, hd_limit); >> > + set_meta(metadata, "lavfi.phqm.phqm", 0, hd); >> > + >> > + /* scene hamming distance avg */ >> > + s->shd += FFMIN(hd, hd_limit); >> > + s->nb_shd++; >> > + av_log(s, AV_LOG_DEBUG, "ImgHashFrame: hd:%0.3lf scd:%0.2lf\n", >> > hd, >> > s->scene_score); >> > + >> > + s->min_hd = FFMIN(s->min_hd, hd); >> > + s->max_hd = FFMAX(s->max_hd, hd); >> > + s->smin_hd = FFMIN(s->smin_hd, hd); >> > + s->smax_hd = FFMAX(s->smax_hd, hd); >> > + >> > + if (s->stats_file) { >> > + fprintf(s->stats_file, >> > + "n:%"PRId64" phqm:%0.3f phqm_min:%0.3f phqm_max:%0.3f >> > sad:%0.2f", >> > + s->nb_frames, hd, s->min_hd, s->max_hd, >> > s->scene_score); >> > + fprintf(s->stats_file, "\n"); >> > + } >> > + >> > + return ff_filter_frame(ctx->outputs[0], master); >> > +} >> > + >> > +static av_cold int init(AVFilterContext *ctx) >> > +{ >> > + PHQMContext *s = ctx->priv; >> > + >> > + if (s->stats_file_str) { >> > + if (!strcmp(s->stats_file_str, "-")) { >> > + s->stats_file = stdout; >> > + } else { >> > + s->stats_file = fopen(s->stats_file_str, "w"); >> > + if (!s->stats_file) { >> > + int err = AVERROR(errno); >> > + char buf[128]; >> > + av_strerror(err, buf, sizeof(buf)); >> > + av_log(ctx, AV_LOG_ERROR, "Could not open stats file >> > %s: >> > %s\n", >> > + s->stats_file_str, buf); >> > + return err; >> > + } >> > + } >> > + } >> > + >> > + s->sad = ff_scene_sad_get_fn(8); >> > + if (!s->sad) >> > + return AVERROR(EINVAL); >> > + >> > + s->fs.on_event = do_phqm; >> > + return 0; >> > +} >> > + >> > +static int query_formats(AVFilterContext *ctx) >> > +{ >> > + PHQMContext *s = ctx->priv; >> > + AVFilterFormats *fmts_list = NULL; >> > + static const enum AVPixelFormat gray8_pix_fmts[] = { >> > + AV_PIX_FMT_GRAY8, >> > + AV_PIX_FMT_NONE >> > + }; >> > + static const enum AVPixelFormat bgr24_pix_fmts[] = { >> > + AV_PIX_FMT_BGR24, >> > + AV_PIX_FMT_NONE >> > + }; >> > + static const enum AVPixelFormat bgra_pix_fmts[] = { >> > + AV_PIX_FMT_BGRA, >> > + AV_PIX_FMT_NONE >> > + }; >> > + >> > + switch (s->hash_type) { >> > + case COLORMOMENT: fmts_list = >> > ff_make_format_list(bgr24_pix_fmts); >> > break; >> > + case MARRHILDRETH: fmts_list = >> > ff_make_format_list(bgra_pix_fmts); >> > break; >> > + /* all other hashes take the gray8 format */ >> > + default: fmts_list = ff_make_format_list(gray8_pix_fmts); >> > break; >> > + } >> > + if (!fmts_list) >> > + return AVERROR(ENOMEM); >> > + return ff_set_common_formats(ctx, fmts_list); >> > +} >> > + >> > +static int config_input_ref(AVFilterLink *inlink) >> > +{ >> > + AVFilterContext *ctx = inlink->dst; >> > + >> > + if (ctx->inputs[0]->w != ctx->inputs[1]->w || >> > + ctx->inputs[0]->h != ctx->inputs[1]->h) { >> > + av_log(ctx, AV_LOG_ERROR, "Width and height of input videos >> > must be >> > same.\n"); >> > + return AVERROR(EINVAL); >> > + } >> > + if (ctx->inputs[0]->format != ctx->inputs[1]->format) { >> > + av_log(ctx, AV_LOG_ERROR, "Inputs must be of same pixel >> > format.\n"); >> > + return AVERROR(EINVAL); >> > + } >> > + >> > + return 0; >> > +} >> > + >> > +static int config_output(AVFilterLink *outlink) >> > +{ >> > + AVFilterContext *ctx = outlink->src; >> > + PHQMContext *s = ctx->priv; >> > + AVFilterLink *mainlink = ctx->inputs[0]; >> > + int ret; >> > + >> > + ret = ff_framesync_init_dualinput(&s->fs, ctx); >> > + if (ret < 0) >> > + return ret; >> > + outlink->w = mainlink->w; >> > + outlink->h = mainlink->h; >> > + outlink->time_base = mainlink->time_base; >> > + outlink->sample_aspect_ratio = mainlink->sample_aspect_ratio; >> > + outlink->frame_rate = mainlink->frame_rate; >> > + if ((ret = ff_framesync_configure(&s->fs)) < 0) >> > + return ret; >> > + >> > + return 0; >> > +} >> > + >> > +static int activate(AVFilterContext *ctx) >> > +{ >> > + PHQMContext *s = ctx->priv; >> > + return ff_framesync_activate(&s->fs); >> > +} >> > + >> > +static av_cold void uninit(AVFilterContext *ctx) >> > +{ >> > + PHQMContext *s = ctx->priv; >> > + >> > + if (s->nb_frames > 0) >> > + av_log(ctx, AV_LOG_WARNING, "PHQM average:%f min:%f max:%f\n", >> > + s->hd / s->nb_frames, s->min_hd, s->max_hd); >> > + >> > + ff_framesync_uninit(&s->fs); >> > + >> > + if (s->stats_file && s->stats_file != stdout) >> > + fclose(s->stats_file); >> > + av_frame_free(&s->prev_picref); >> > +} >> > + >> > +static const AVFilterPad phqm_inputs[] = { >> > + { >> > + .name = "main", >> > + .type = AVMEDIA_TYPE_VIDEO, >> > + },{ >> > + .name = "reference", >> > + .type = AVMEDIA_TYPE_VIDEO, >> > + .config_props = config_input_ref, >> > + }, >> > + { NULL } >> > +}; >> > + >> > +static const AVFilterPad phqm_outputs[] = { >> > + { >> > + .name = "default", >> > + .type = AVMEDIA_TYPE_VIDEO, >> > + .config_props = config_output, >> > + }, >> > + { NULL } >> > +}; >> > + >> > +AVFilter ff_vf_phqm= { >> > + .name = "phqm", >> > + .description = NULL_IF_CONFIG_SMALL("PHQM: Calculate the >> > Perceptual >> > Hash Hamming Difference between two video streams."), >> > + .preinit = phqm_framesync_preinit, >> > + .init = init, >> > + .uninit = uninit, >> > + .query_formats = query_formats, >> > + .activate = activate, >> > + .priv_size = sizeof(PHQMContext), >> > + .priv_class = &phqm_class, >> > + .inputs = phqm_inputs, >> > + .outputs = phqm_outputs, >> > +}; >> > -- >> > 2.20.1 >> > >> > _______________________________________________ >> > ffmpeg-devel mailing list >> > ffmpeg-devel@ffmpeg.org >> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel >> > >> > To unsubscribe, visit link above, or email >> > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". >
On Sat, Oct 26, 2019 at 8:22 AM Paul B Mahol <onemda@gmail.com> wrote: > > On 10/26/19, Christopher Kennedy <ckennedy@ellation.com> wrote: > > This is a reference/encode comparison filter with two files input like > > the psnr or vmaf filter. > > So it is completely different and uses the C++ OpenCV API since this > > img_hash library is not in the C API. > > It's unique to what the OCV filter does, and has more research > > implications from my talk at Demuxed 2019. > > I do not see how that is relevant. > > There should be generic opencv filter which could do this above in > generic way, and not by adding yet another filter that uses only some > part of opencv. Is it really possible to do framesync() operations like dual input filters like psnr/vmaf and also handle input/output rendering of the frames too? This sounds odd to me but I would love to understand how this is possible. The C OpenCV API is not recommended so this does the OpenCV part in C++ which allows it to be fully utilized and supported. So that seems better to me than using the API OpenCV won't really support and doesn't allow usage of img_hash (it is NOT in the C API of OpenCV, impossible to use). The OpenCV C++ img_hash library is the fastest implementation and does work best in OpenCV. So implementing this in C directly isn't a task I believe is good to do. So should the current OpenCV stuff be merged into this filter, is that possible? If so, then your saying the PSNR filter could also alter frames and output them too from the reference/encode or one or the other? I need to understand this better, I want to make it right so am listening. Thanks Christopher > > > > > Christopher > > > > On Sat, Oct 26, 2019 at 7:38 AM Paul B Mahol <onemda@gmail.com> wrote: > >> > >> Why is this not generic filter like already existing opencv filter? > >> > >> On 10/26/19, ckennedy@ellation.com <ckennedy@ellation.com> wrote: > >> > From: Christopher Kennedy <ckennedy@ellation.com> > >> > > >> > this adds a phqm filter and OpenCV img_hash based resource usable > >> > by the phqm and future filters using image hash functionality > >> > from OpenCV. > >> > > >> > C++ to C handling so that full OpenCV functionality and API can > >> > be used instead of the C versions (which are incomplete and > >> > don't always exist). > >> > > >> > Example command line: > >> > > >> > ffmpeg -i encode.mp4 -i reference.mp4 \ > >> > -filter_complex "[0:v][1:v]phqm=stats_file=out.log" \ > >> > -y -f null /dev/null > >> > > >> > Signed-off-by: Christopher Kennedy <ckennedy@ellation.com> > >> > > >> > _______________________________________________ > >> > ffmpeg-devel mailing list > >> > ffmpeg-devel@ffmpeg.org > >> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > >> > > >> > To unsubscribe, visit link above, or email > >> > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". > >
Hi, On Sat, Oct 26, 2019 at 4:35 PM Christopher Kennedy <ckennedy@ellation.com> wrote: > The C OpenCV API is not recommended so this does the OpenCV part > in C++ which allows it to be fully utilized and supported. So that seems > better to me than using the API OpenCV won't really support and doesn't > allow usage of img_hash (it is NOT in the C API of OpenCV, impossible to use). Just noting my two cents about the OpenCV C API as I've noticed regarding people trying to utilize the opencv capabilities already in FFmpeg on the user IRC channel: It seems like upstream OpenCV actively has broken it quite a long time ago, and the earlier code we have utilizing OpenCV doesn't even compile with current OpenCV versions. https://github.com/opencv/opencv/issues/8438#issuecomment-288638915 Thus, unless the OpenCV project's view on C APIs changes, I don't see it being worthwhile adding any new code utilizing the C APIs that only work with old versions of OpenCV. One way of possibly changing this is to advocate for C interfaces at OpenCV, but for that the actual advocates/users of OpenCV usage should raise on the walls regarding that. So while we heavily dislike having C++ APIs as dependencies, I think if we want to have (current) OpenCV usage around, then modules utilizing C++ should be permitted (pretty sure we already have some of those in the code base for other libraries that have been deemed important enough by someone). Jan
On 10/26/19, Christopher Kennedy <ckennedy@ellation.com> wrote: > On Sat, Oct 26, 2019 at 8:22 AM Paul B Mahol <onemda@gmail.com> wrote: >> >> On 10/26/19, Christopher Kennedy <ckennedy@ellation.com> wrote: >> > This is a reference/encode comparison filter with two files input like >> > the psnr or vmaf filter. >> > So it is completely different and uses the C++ OpenCV API since this >> > img_hash library is not in the C API. >> > It's unique to what the OCV filter does, and has more research >> > implications from my talk at Demuxed 2019. >> >> I do not see how that is relevant. >> >> There should be generic opencv filter which could do this above in >> generic way, and not by adding yet another filter that uses only some >> part of opencv. > > Is it really possible to do framesync() operations like dual input filters > like psnr/vmaf and also handle input/output rendering of the frames too? > This sounds odd to me but I would love to understand how this is possible. framesync is nothing special, its just used a lot, there are video filters that do not use it and still operate on multiple inputs. > > The C OpenCV API is not recommended so this does the OpenCV part > in C++ which allows it to be fully utilized and supported. So that seems > better to me than using the API OpenCV won't really support and doesn't > allow usage of img_hash (it is NOT in the C API of OpenCV, impossible to > use). > > The OpenCV C++ img_hash library is the fastest implementation and does > work best in OpenCV. So implementing this in C directly isn't a task I > believe > is good to do. I do not care how generic opencv filter is done, it can be C++ just fine. > > So should the current OpenCV stuff be merged into this filter, is that > possible? Current opencv stuff in libavfilter is pretty dead and not maintained at all. So I'm not really fond of adding yet another opencv filter that not gonna be maintained. > If so, then your saying the PSNR filter could also alter frames and output > them > too from the reference/encode or one or the other? I need to understand > this > better, I want to make it right so am listening. I'm not sure what you really want. Native psnr filter does not alter video, it just takes two inputs and output first one unmodified and reports via log and metadata psnr values it measured. Also, I'm of opinion that hashing images is so trivial operation that could be done native to libavfilter. > > Thanks > Christopher > >> >> > >> > Christopher >> > >> > On Sat, Oct 26, 2019 at 7:38 AM Paul B Mahol <onemda@gmail.com> wrote: >> >> >> >> Why is this not generic filter like already existing opencv filter? >> >> >> >> On 10/26/19, ckennedy@ellation.com <ckennedy@ellation.com> wrote: >> >> > From: Christopher Kennedy <ckennedy@ellation.com> >> >> > >> >> > this adds a phqm filter and OpenCV img_hash based resource usable >> >> > by the phqm and future filters using image hash functionality >> >> > from OpenCV. >> >> > >> >> > C++ to C handling so that full OpenCV functionality and API can >> >> > be used instead of the C versions (which are incomplete and >> >> > don't always exist). >> >> > >> >> > Example command line: >> >> > >> >> > ffmpeg -i encode.mp4 -i reference.mp4 \ >> >> > -filter_complex "[0:v][1:v]phqm=stats_file=out.log" \ >> >> > -y -f null /dev/null >> >> > >> >> > Signed-off-by: Christopher Kennedy <ckennedy@ellation.com> >> >> > >> >> > _______________________________________________ >> >> > ffmpeg-devel mailing list >> >> > ffmpeg-devel@ffmpeg.org >> >> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel >> >> > >> >> > To unsubscribe, visit link above, or email >> >> > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". >> > >
On Sat, Oct 26, 2019 at 11:15 AM Paul B Mahol <onemda@gmail.com> wrote: > > On 10/26/19, Christopher Kennedy <ckennedy@ellation.com> wrote: > > On Sat, Oct 26, 2019 at 8:22 AM Paul B Mahol <onemda@gmail.com> wrote: > >> > >> On 10/26/19, Christopher Kennedy <ckennedy@ellation.com> wrote: > >> > This is a reference/encode comparison filter with two files input like > >> > the psnr or vmaf filter. > >> > So it is completely different and uses the C++ OpenCV API since this > >> > img_hash library is not in the C API. > >> > It's unique to what the OCV filter does, and has more research > >> > implications from my talk at Demuxed 2019. > >> > >> I do not see how that is relevant. > >> > >> There should be generic opencv filter which could do this above in > >> generic way, and not by adding yet another filter that uses only some > >> part of opencv. > > > > Is it really possible to do framesync() operations like dual input filters > > like psnr/vmaf and also handle input/output rendering of the frames too? > > This sounds odd to me but I would love to understand how this is possible. > > framesync is nothing special, its just used a lot, there are video > filters that do not use it and still operate on multiple inputs. > > > > > The C OpenCV API is not recommended so this does the OpenCV part > > in C++ which allows it to be fully utilized and supported. So that seems > > better to me than using the API OpenCV won't really support and doesn't > > allow usage of img_hash (it is NOT in the C API of OpenCV, impossible to > > use). > > > > The OpenCV C++ img_hash library is the fastest implementation and does > > work best in OpenCV. So implementing this in C directly isn't a task I > > believe > > is good to do. > > I do not care how generic opencv filter is done, it can be C++ just fine. > > > > > So should the current OpenCV stuff be merged into this filter, is that > > possible? > > Current opencv stuff in libavfilter is pretty dead and not maintained at all. > So I'm not really fond of adding yet another opencv filter that not > gonna be maintained. > > > If so, then your saying the PSNR filter could also alter frames and output > > them > > too from the reference/encode or one or the other? I need to understand > > this > > better, I want to make it right so am listening. > > I'm not sure what you really want. Native psnr filter does not alter video, > it just takes two inputs and output first one unmodified and reports > via log and metadata psnr values it measured. > > Also, I'm of opinion that hashing images is so trivial operation that > could be done native to libavfilter. Well one option is for me to port the other ocv filters few capabilities into this one that uses the recommended maintained API. Otherwise this feature isn't available in the other. Yeah I would love to have a C version of it to plop in. I don't think C++ linking like this can be kept out of individual parts of FFmpeg that aren't affecting the other parts forever. I don't like C++ but I do like all the other possible options in OpenCV this filter will open up for others to add into FFmpeg filters in the future. I do think there is great value and not enough people to rewrite OpenCV into C and maintain that. Thanks, Christopher > > > > > Thanks > > Christopher > > > >> > >> > > >> > Christopher > >> > > >> > On Sat, Oct 26, 2019 at 7:38 AM Paul B Mahol <onemda@gmail.com> wrote: > >> >> > >> >> Why is this not generic filter like already existing opencv filter? > >> >> > >> >> On 10/26/19, ckennedy@ellation.com <ckennedy@ellation.com> wrote: > >> >> > From: Christopher Kennedy <ckennedy@ellation.com> > >> >> > > >> >> > this adds a phqm filter and OpenCV img_hash based resource usable > >> >> > by the phqm and future filters using image hash functionality > >> >> > from OpenCV. > >> >> > > >> >> > C++ to C handling so that full OpenCV functionality and API can > >> >> > be used instead of the C versions (which are incomplete and > >> >> > don't always exist). > >> >> > > >> >> > Example command line: > >> >> > > >> >> > ffmpeg -i encode.mp4 -i reference.mp4 \ > >> >> > -filter_complex "[0:v][1:v]phqm=stats_file=out.log" \ > >> >> > -y -f null /dev/null > >> >> > > >> >> > Signed-off-by: Christopher Kennedy <ckennedy@ellation.com> > >> >> > > >> >> > _______________________________________________ > >> >> > ffmpeg-devel mailing list > >> >> > ffmpeg-devel@ffmpeg.org > >> >> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > >> >> > > >> >> > To unsubscribe, visit link above, or email > >> >> > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". > >> > > >
On Sat, Oct 26, 2019 at 8:50 AM Jan Ekström <jeebjp@gmail.com> wrote: > Hi, > > On Sat, Oct 26, 2019 at 4:35 PM Christopher Kennedy > <ckennedy@ellation.com> wrote: > > The C OpenCV API is not recommended so this does the OpenCV part > > in C++ which allows it to be fully utilized and supported. So that seems > > better to me than using the API OpenCV won't really support and doesn't > > allow usage of img_hash (it is NOT in the C API of OpenCV, impossible to > use). > > Just noting my two cents about the OpenCV C API as I've noticed > regarding people trying to utilize the opencv capabilities already in > FFmpeg on the user IRC channel: > > It seems like upstream OpenCV actively has broken it quite a long time > ago, and the earlier code we have utilizing OpenCV doesn't even > compile with current OpenCV versions. > https://github.com/opencv/opencv/issues/8438#issuecomment-288638915 > > Thus, unless the OpenCV project's view on C APIs changes, I don't see > it being worthwhile adding any new code utilizing the C APIs that only > work with old versions of OpenCV. One way of possibly changing this is > to advocate for C interfaces at OpenCV, but for that the actual > advocates/users of OpenCV usage should raise on the walls regarding > that. > > So while we heavily dislike having C++ APIs as dependencies, I think > if we want to have (current) OpenCV usage around, then modules > utilizing C++ should be permitted (pretty sure we already have some of > those in the code base for other libraries that have been deemed > important enough by someone). > Yes the decklink stuff uses .cpp files too. Which is another good case where it is either have or don't have support over C++. > > Jan > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
On Sat, Oct 26, 2019 at 9:15 AM Paul B Mahol <onemda@gmail.com> wrote: > > On 10/26/19, Christopher Kennedy <ckennedy@ellation.com> wrote: > > On Sat, Oct 26, 2019 at 8:22 AM Paul B Mahol <onemda@gmail.com> wrote: > >> > >> On 10/26/19, Christopher Kennedy <ckennedy@ellation.com> wrote: > >> > This is a reference/encode comparison filter with two files input like > >> > the psnr or vmaf filter. > >> > So it is completely different and uses the C++ OpenCV API since this > >> > img_hash library is not in the C API. > >> > It's unique to what the OCV filter does, and has more research > >> > implications from my talk at Demuxed 2019. > >> > >> I do not see how that is relevant. > >> > >> There should be generic opencv filter which could do this above in > >> generic way, and not by adding yet another filter that uses only some > >> part of opencv. > > > > Is it really possible to do framesync() operations like dual input filters > > like psnr/vmaf and also handle input/output rendering of the frames too? > > This sounds odd to me but I would love to understand how this is possible. > > framesync is nothing special, its just used a lot, there are video > filters that do not use it and still operate on multiple inputs. > > > > > The C OpenCV API is not recommended so this does the OpenCV part > > in C++ which allows it to be fully utilized and supported. So that seems > > better to me than using the API OpenCV won't really support and doesn't > > allow usage of img_hash (it is NOT in the C API of OpenCV, impossible to > > use). > > > > The OpenCV C++ img_hash library is the fastest implementation and does > > work best in OpenCV. So implementing this in C directly isn't a task I > > believe > > is good to do. > > I do not care how generic opencv filter is done, it can be C++ just fine. > > > > > So should the current OpenCV stuff be merged into this filter, is that > > possible? > > Current opencv stuff in libavfilter is pretty dead and not maintained at all. > So I'm not really fond of adding yet another opencv filter that not > gonna be maintained. Well I would maintain it, but maybe I should instead run a Fork of FFmpeg with C++ and OpenCV properly done. Yes the current OCV stuff is dead and not supported by OpenCV or maintained it sounds like. So if we can't put C++ into FFmpeg (yet we already have with decklink filters, so that is unfair to say). And we can't switch to use OpenCV C++ yet the C API is dead and not supported. The only solution I see is to run a fork which is sad to have to do. This just seems really unreasonable overall :(. Thanks, Christopher. > > > If so, then your saying the PSNR filter could also alter frames and output > > them > > too from the reference/encode or one or the other? I need to understand > > this > > better, I want to make it right so am listening. > > I'm not sure what you really want. Native psnr filter does not alter video, > it just takes two inputs and output first one unmodified and reports > via log and metadata psnr values it measured. > > Also, I'm of opinion that hashing images is so trivial operation that > could be done native to libavfilter. > > > > > Thanks > > Christopher > > > >> > >> > > >> > Christopher > >> > > >> > On Sat, Oct 26, 2019 at 7:38 AM Paul B Mahol <onemda@gmail.com> wrote: > >> >> > >> >> Why is this not generic filter like already existing opencv filter? > >> >> > >> >> On 10/26/19, ckennedy@ellation.com <ckennedy@ellation.com> wrote: > >> >> > From: Christopher Kennedy <ckennedy@ellation.com> > >> >> > > >> >> > this adds a phqm filter and OpenCV img_hash based resource usable > >> >> > by the phqm and future filters using image hash functionality > >> >> > from OpenCV. > >> >> > > >> >> > C++ to C handling so that full OpenCV functionality and API can > >> >> > be used instead of the C versions (which are incomplete and > >> >> > don't always exist). > >> >> > > >> >> > Example command line: > >> >> > > >> >> > ffmpeg -i encode.mp4 -i reference.mp4 \ > >> >> > -filter_complex "[0:v][1:v]phqm=stats_file=out.log" \ > >> >> > -y -f null /dev/null > >> >> > > >> >> > Signed-off-by: Christopher Kennedy <ckennedy@ellation.com> > >> >> > > >> >> > _______________________________________________ > >> >> > ffmpeg-devel mailing list > >> >> > ffmpeg-devel@ffmpeg.org > >> >> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > >> >> > > >> >> > To unsubscribe, visit link above, or email > >> >> > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". > >> > > >
On 10/30/19, Christopher Kennedy <ckennedy@ellation.com> wrote: > On Sat, Oct 26, 2019 at 9:15 AM Paul B Mahol <onemda@gmail.com> wrote: >> >> On 10/26/19, Christopher Kennedy <ckennedy@ellation.com> wrote: >> > On Sat, Oct 26, 2019 at 8:22 AM Paul B Mahol <onemda@gmail.com> wrote: >> >> >> >> On 10/26/19, Christopher Kennedy <ckennedy@ellation.com> wrote: >> >> > This is a reference/encode comparison filter with two files input >> >> > like >> >> > the psnr or vmaf filter. >> >> > So it is completely different and uses the C++ OpenCV API since this >> >> > img_hash library is not in the C API. >> >> > It's unique to what the OCV filter does, and has more research >> >> > implications from my talk at Demuxed 2019. >> >> >> >> I do not see how that is relevant. >> >> >> >> There should be generic opencv filter which could do this above in >> >> generic way, and not by adding yet another filter that uses only some >> >> part of opencv. >> > >> > Is it really possible to do framesync() operations like dual input >> > filters >> > like psnr/vmaf and also handle input/output rendering of the frames >> > too? >> > This sounds odd to me but I would love to understand how this is >> > possible. >> >> framesync is nothing special, its just used a lot, there are video >> filters that do not use it and still operate on multiple inputs. >> >> > >> > The C OpenCV API is not recommended so this does the OpenCV part >> > in C++ which allows it to be fully utilized and supported. So that >> > seems >> > better to me than using the API OpenCV won't really support and doesn't >> > allow usage of img_hash (it is NOT in the C API of OpenCV, impossible >> > to >> > use). >> > >> > The OpenCV C++ img_hash library is the fastest implementation and does >> > work best in OpenCV. So implementing this in C directly isn't a task I >> > believe >> > is good to do. >> >> I do not care how generic opencv filter is done, it can be C++ just fine. >> >> > >> > So should the current OpenCV stuff be merged into this filter, is that >> > possible? >> >> Current opencv stuff in libavfilter is pretty dead and not maintained at >> all. >> So I'm not really fond of adding yet another opencv filter that not >> gonna be maintained. > > Well I would maintain it, but maybe I should instead run a Fork of FFmpeg > with > C++ and OpenCV properly done. > > Yes the current OCV stuff is dead and not supported by OpenCV or maintained > it sounds like. > > So if we can't put C++ into FFmpeg (yet we already have with decklink > filters, > so that is unfair to say). > > And we can't switch to use OpenCV C++ yet the C API is dead and not > supported. Who said that? C++ module filter in libavfilter dealing with latest OpenCV and well maintained is always welcome. > > The only solution I see is to run a fork which is sad to have to do. > > This just seems really unreasonable overall :(. > > Thanks, > Christopher. > > >> >> > If so, then your saying the PSNR filter could also alter frames and >> > output >> > them >> > too from the reference/encode or one or the other? I need to understand >> > this >> > better, I want to make it right so am listening. >> >> I'm not sure what you really want. Native psnr filter does not alter >> video, >> it just takes two inputs and output first one unmodified and reports >> via log and metadata psnr values it measured. >> >> Also, I'm of opinion that hashing images is so trivial operation that >> could be done native to libavfilter. >> >> > >> > Thanks >> > Christopher >> > >> >> >> >> > >> >> > Christopher >> >> > >> >> > On Sat, Oct 26, 2019 at 7:38 AM Paul B Mahol <onemda@gmail.com> >> >> > wrote: >> >> >> >> >> >> Why is this not generic filter like already existing opencv filter? >> >> >> >> >> >> On 10/26/19, ckennedy@ellation.com <ckennedy@ellation.com> wrote: >> >> >> > From: Christopher Kennedy <ckennedy@ellation.com> >> >> >> > >> >> >> > this adds a phqm filter and OpenCV img_hash based resource usable >> >> >> > by the phqm and future filters using image hash functionality >> >> >> > from OpenCV. >> >> >> > >> >> >> > C++ to C handling so that full OpenCV functionality and API can >> >> >> > be used instead of the C versions (which are incomplete and >> >> >> > don't always exist). >> >> >> > >> >> >> > Example command line: >> >> >> > >> >> >> > ffmpeg -i encode.mp4 -i reference.mp4 \ >> >> >> > -filter_complex "[0:v][1:v]phqm=stats_file=out.log" \ >> >> >> > -y -f null /dev/null >> >> >> > >> >> >> > Signed-off-by: Christopher Kennedy <ckennedy@ellation.com> >> >> >> > >> >> >> > _______________________________________________ >> >> >> > ffmpeg-devel mailing list >> >> >> > ffmpeg-devel@ffmpeg.org >> >> >> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel >> >> >> > >> >> >> > To unsubscribe, visit link above, or email >> >> >> > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". >> >> > >> > >
On Wed, Oct 30, 2019 at 10:07 AM Paul B Mahol <onemda@gmail.com> wrote: > > On 10/30/19, Christopher Kennedy <ckennedy@ellation.com> wrote: > > On Sat, Oct 26, 2019 at 9:15 AM Paul B Mahol <onemda@gmail.com> wrote: > >> > >> On 10/26/19, Christopher Kennedy <ckennedy@ellation.com> wrote: > >> > On Sat, Oct 26, 2019 at 8:22 AM Paul B Mahol <onemda@gmail.com> wrote: > >> >> > >> >> On 10/26/19, Christopher Kennedy <ckennedy@ellation.com> wrote: > >> >> > This is a reference/encode comparison filter with two files input > >> >> > like > >> >> > the psnr or vmaf filter. > >> >> > So it is completely different and uses the C++ OpenCV API since this > >> >> > img_hash library is not in the C API. > >> >> > It's unique to what the OCV filter does, and has more research > >> >> > implications from my talk at Demuxed 2019. > >> >> > >> >> I do not see how that is relevant. > >> >> > >> >> There should be generic opencv filter which could do this above in > >> >> generic way, and not by adding yet another filter that uses only some > >> >> part of opencv. > >> > > >> > Is it really possible to do framesync() operations like dual input > >> > filters > >> > like psnr/vmaf and also handle input/output rendering of the frames > >> > too? > >> > This sounds odd to me but I would love to understand how this is > >> > possible. > >> > >> framesync is nothing special, its just used a lot, there are video > >> filters that do not use it and still operate on multiple inputs. > >> > >> > > >> > The C OpenCV API is not recommended so this does the OpenCV part > >> > in C++ which allows it to be fully utilized and supported. So that > >> > seems > >> > better to me than using the API OpenCV won't really support and doesn't > >> > allow usage of img_hash (it is NOT in the C API of OpenCV, impossible > >> > to > >> > use). > >> > > >> > The OpenCV C++ img_hash library is the fastest implementation and does > >> > work best in OpenCV. So implementing this in C directly isn't a task I > >> > believe > >> > is good to do. > >> > >> I do not care how generic opencv filter is done, it can be C++ just fine. > >> > >> > > >> > So should the current OpenCV stuff be merged into this filter, is that > >> > possible? > >> > >> Current opencv stuff in libavfilter is pretty dead and not maintained at > >> all. > >> So I'm not really fond of adding yet another opencv filter that not > >> gonna be maintained. > > > > Well I would maintain it, but maybe I should instead run a Fork of FFmpeg > > with > > C++ and OpenCV properly done. > > > > Yes the current OCV stuff is dead and not supported by OpenCV or maintained > > it sounds like. > > > > So if we can't put C++ into FFmpeg (yet we already have with decklink > > filters, > > so that is unfair to say). > > > > And we can't switch to use OpenCV C++ yet the C API is dead and not > > supported. > > Who said that? > C++ module filter in libavfilter dealing with latest OpenCV and well > maintained is always welcome. So I should be moving the functionality of the current ocv filter into a C++ file for filters to use like phqm, the current ocv transforms? Is this a good plan, separate the C++ from the vf_ filter as I have done with img_hash.cpp (yet make it a more generic set of OpenCV C++ helper functions to plug into C vf_filters which can be separated by function yet unified in code used from the C++ base code)? Wanting to clarify exactly what is acceptable so I can do it and not waste time / effort doing stuff that isn't right. Thanks, Christopher > > > > > The only solution I see is to run a fork which is sad to have to do. > > > > This just seems really unreasonable overall :(. > > > > Thanks, > > Christopher. > > > > > >> > >> > If so, then your saying the PSNR filter could also alter frames and > >> > output > >> > them > >> > too from the reference/encode or one or the other? I need to understand > >> > this > >> > better, I want to make it right so am listening. > >> > >> I'm not sure what you really want. Native psnr filter does not alter > >> video, > >> it just takes two inputs and output first one unmodified and reports > >> via log and metadata psnr values it measured. > >> > >> Also, I'm of opinion that hashing images is so trivial operation that > >> could be done native to libavfilter. > >> > >> > > >> > Thanks > >> > Christopher > >> > > >> >> > >> >> > > >> >> > Christopher > >> >> > > >> >> > On Sat, Oct 26, 2019 at 7:38 AM Paul B Mahol <onemda@gmail.com> > >> >> > wrote: > >> >> >> > >> >> >> Why is this not generic filter like already existing opencv filter? > >> >> >> > >> >> >> On 10/26/19, ckennedy@ellation.com <ckennedy@ellation.com> wrote: > >> >> >> > From: Christopher Kennedy <ckennedy@ellation.com> > >> >> >> > > >> >> >> > this adds a phqm filter and OpenCV img_hash based resource usable > >> >> >> > by the phqm and future filters using image hash functionality > >> >> >> > from OpenCV. > >> >> >> > > >> >> >> > C++ to C handling so that full OpenCV functionality and API can > >> >> >> > be used instead of the C versions (which are incomplete and > >> >> >> > don't always exist). > >> >> >> > > >> >> >> > Example command line: > >> >> >> > > >> >> >> > ffmpeg -i encode.mp4 -i reference.mp4 \ > >> >> >> > -filter_complex "[0:v][1:v]phqm=stats_file=out.log" \ > >> >> >> > -y -f null /dev/null > >> >> >> > > >> >> >> > Signed-off-by: Christopher Kennedy <ckennedy@ellation.com> > >> >> >> > > >> >> >> > _______________________________________________ > >> >> >> > ffmpeg-devel mailing list > >> >> >> > ffmpeg-devel@ffmpeg.org > >> >> >> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > >> >> >> > > >> >> >> > To unsubscribe, visit link above, or email > >> >> >> > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". > >> >> > > >> > > >
On 10/30/19, Christopher Kennedy <ckennedy@ellation.com> wrote: > On Wed, Oct 30, 2019 at 10:07 AM Paul B Mahol <onemda@gmail.com> wrote: >> >> On 10/30/19, Christopher Kennedy <ckennedy@ellation.com> wrote: >> > On Sat, Oct 26, 2019 at 9:15 AM Paul B Mahol <onemda@gmail.com> wrote: >> >> >> >> On 10/26/19, Christopher Kennedy <ckennedy@ellation.com> wrote: >> >> > On Sat, Oct 26, 2019 at 8:22 AM Paul B Mahol <onemda@gmail.com> >> >> > wrote: >> >> >> >> >> >> On 10/26/19, Christopher Kennedy <ckennedy@ellation.com> wrote: >> >> >> > This is a reference/encode comparison filter with two files input >> >> >> > like >> >> >> > the psnr or vmaf filter. >> >> >> > So it is completely different and uses the C++ OpenCV API since >> >> >> > this >> >> >> > img_hash library is not in the C API. >> >> >> > It's unique to what the OCV filter does, and has more research >> >> >> > implications from my talk at Demuxed 2019. >> >> >> >> >> >> I do not see how that is relevant. >> >> >> >> >> >> There should be generic opencv filter which could do this above in >> >> >> generic way, and not by adding yet another filter that uses only >> >> >> some >> >> >> part of opencv. >> >> > >> >> > Is it really possible to do framesync() operations like dual input >> >> > filters >> >> > like psnr/vmaf and also handle input/output rendering of the frames >> >> > too? >> >> > This sounds odd to me but I would love to understand how this is >> >> > possible. >> >> >> >> framesync is nothing special, its just used a lot, there are video >> >> filters that do not use it and still operate on multiple inputs. >> >> >> >> > >> >> > The C OpenCV API is not recommended so this does the OpenCV part >> >> > in C++ which allows it to be fully utilized and supported. So that >> >> > seems >> >> > better to me than using the API OpenCV won't really support and >> >> > doesn't >> >> > allow usage of img_hash (it is NOT in the C API of OpenCV, >> >> > impossible >> >> > to >> >> > use). >> >> > >> >> > The OpenCV C++ img_hash library is the fastest implementation and >> >> > does >> >> > work best in OpenCV. So implementing this in C directly isn't a task >> >> > I >> >> > believe >> >> > is good to do. >> >> >> >> I do not care how generic opencv filter is done, it can be C++ just >> >> fine. >> >> >> >> > >> >> > So should the current OpenCV stuff be merged into this filter, is >> >> > that >> >> > possible? >> >> >> >> Current opencv stuff in libavfilter is pretty dead and not maintained >> >> at >> >> all. >> >> So I'm not really fond of adding yet another opencv filter that not >> >> gonna be maintained. >> > >> > Well I would maintain it, but maybe I should instead run a Fork of >> > FFmpeg >> > with >> > C++ and OpenCV properly done. >> > >> > Yes the current OCV stuff is dead and not supported by OpenCV or >> > maintained >> > it sounds like. >> > >> > So if we can't put C++ into FFmpeg (yet we already have with decklink >> > filters, >> > so that is unfair to say). >> > >> > And we can't switch to use OpenCV C++ yet the C API is dead and not >> > supported. >> >> Who said that? >> C++ module filter in libavfilter dealing with latest OpenCV and well >> maintained is always welcome. > > > So I should be moving the functionality of the current ocv filter into a > C++ file for filters to use like phqm, the current ocv transforms? > > Is this a good plan, separate the C++ from the vf_ filter as I have done > with img_hash.cpp (yet make it a more generic set of OpenCV C++ > helper functions to plug into C vf_filters which can be separated by > function yet unified in code used from the C++ base code)? IMHO filter file can be cpp file without issues. And even if not use ff_ prefixed functions, > > Wanting to clarify exactly what is acceptable so I can do it and not waste > time / effort doing stuff that isn't right. > > Thanks, > Christopher > > >> >> > >> > The only solution I see is to run a fork which is sad to have to do. >> > >> > This just seems really unreasonable overall :(. >> > >> > Thanks, >> > Christopher. >> > >> > >> >> >> >> > If so, then your saying the PSNR filter could also alter frames and >> >> > output >> >> > them >> >> > too from the reference/encode or one or the other? I need to >> >> > understand >> >> > this >> >> > better, I want to make it right so am listening. >> >> >> >> I'm not sure what you really want. Native psnr filter does not alter >> >> video, >> >> it just takes two inputs and output first one unmodified and reports >> >> via log and metadata psnr values it measured. >> >> >> >> Also, I'm of opinion that hashing images is so trivial operation that >> >> could be done native to libavfilter. >> >> >> >> > >> >> > Thanks >> >> > Christopher >> >> > >> >> >> >> >> >> > >> >> >> > Christopher >> >> >> > >> >> >> > On Sat, Oct 26, 2019 at 7:38 AM Paul B Mahol <onemda@gmail.com> >> >> >> > wrote: >> >> >> >> >> >> >> >> Why is this not generic filter like already existing opencv >> >> >> >> filter? >> >> >> >> >> >> >> >> On 10/26/19, ckennedy@ellation.com <ckennedy@ellation.com> >> >> >> >> wrote: >> >> >> >> > From: Christopher Kennedy <ckennedy@ellation.com> >> >> >> >> > >> >> >> >> > this adds a phqm filter and OpenCV img_hash based resource >> >> >> >> > usable >> >> >> >> > by the phqm and future filters using image hash functionality >> >> >> >> > from OpenCV. >> >> >> >> > >> >> >> >> > C++ to C handling so that full OpenCV functionality and API >> >> >> >> > can >> >> >> >> > be used instead of the C versions (which are incomplete and >> >> >> >> > don't always exist). >> >> >> >> > >> >> >> >> > Example command line: >> >> >> >> > >> >> >> >> > ffmpeg -i encode.mp4 -i reference.mp4 \ >> >> >> >> > -filter_complex "[0:v][1:v]phqm=stats_file=out.log" >> >> >> >> > \ >> >> >> >> > -y -f null /dev/null >> >> >> >> > >> >> >> >> > Signed-off-by: Christopher Kennedy <ckennedy@ellation.com> >> >> >> >> > >> >> >> >> > _______________________________________________ >> >> >> >> > ffmpeg-devel mailing list >> >> >> >> > ffmpeg-devel@ffmpeg.org >> >> >> >> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel >> >> >> >> > >> >> >> >> > To unsubscribe, visit link above, or email >> >> >> >> > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". >> >> >> > >> >> > >> > >
On Wed, Oct 30, 2019 at 10:57 AM Paul B Mahol <onemda@gmail.com> wrote: > > On 10/30/19, Christopher Kennedy <ckennedy@ellation.com> wrote: > > On Wed, Oct 30, 2019 at 10:07 AM Paul B Mahol <onemda@gmail.com> wrote: > >> > >> On 10/30/19, Christopher Kennedy <ckennedy@ellation.com> wrote: > >> > On Sat, Oct 26, 2019 at 9:15 AM Paul B Mahol <onemda@gmail.com> wrote: > >> >> > >> >> On 10/26/19, Christopher Kennedy <ckennedy@ellation.com> wrote: > >> >> > On Sat, Oct 26, 2019 at 8:22 AM Paul B Mahol <onemda@gmail.com> > >> >> > wrote: > >> >> >> > >> >> >> On 10/26/19, Christopher Kennedy <ckennedy@ellation.com> wrote: > >> >> >> > This is a reference/encode comparison filter with two files input > >> >> >> > like > >> >> >> > the psnr or vmaf filter. > >> >> >> > So it is completely different and uses the C++ OpenCV API since > >> >> >> > this > >> >> >> > img_hash library is not in the C API. > >> >> >> > It's unique to what the OCV filter does, and has more research > >> >> >> > implications from my talk at Demuxed 2019. > >> >> >> > >> >> >> I do not see how that is relevant. > >> >> >> > >> >> >> There should be generic opencv filter which could do this above in > >> >> >> generic way, and not by adding yet another filter that uses only > >> >> >> some > >> >> >> part of opencv. > >> >> > > >> >> > Is it really possible to do framesync() operations like dual input > >> >> > filters > >> >> > like psnr/vmaf and also handle input/output rendering of the frames > >> >> > too? > >> >> > This sounds odd to me but I would love to understand how this is > >> >> > possible. > >> >> > >> >> framesync is nothing special, its just used a lot, there are video > >> >> filters that do not use it and still operate on multiple inputs. > >> >> > >> >> > > >> >> > The C OpenCV API is not recommended so this does the OpenCV part > >> >> > in C++ which allows it to be fully utilized and supported. So that > >> >> > seems > >> >> > better to me than using the API OpenCV won't really support and > >> >> > doesn't > >> >> > allow usage of img_hash (it is NOT in the C API of OpenCV, > >> >> > impossible > >> >> > to > >> >> > use). > >> >> > > >> >> > The OpenCV C++ img_hash library is the fastest implementation and > >> >> > does > >> >> > work best in OpenCV. So implementing this in C directly isn't a task > >> >> > I > >> >> > believe > >> >> > is good to do. > >> >> > >> >> I do not care how generic opencv filter is done, it can be C++ just > >> >> fine. > >> >> > >> >> > > >> >> > So should the current OpenCV stuff be merged into this filter, is > >> >> > that > >> >> > possible? > >> >> > >> >> Current opencv stuff in libavfilter is pretty dead and not maintained > >> >> at > >> >> all. > >> >> So I'm not really fond of adding yet another opencv filter that not > >> >> gonna be maintained. > >> > > >> > Well I would maintain it, but maybe I should instead run a Fork of > >> > FFmpeg > >> > with > >> > C++ and OpenCV properly done. > >> > > >> > Yes the current OCV stuff is dead and not supported by OpenCV or > >> > maintained > >> > it sounds like. > >> > > >> > So if we can't put C++ into FFmpeg (yet we already have with decklink > >> > filters, > >> > so that is unfair to say). > >> > > >> > And we can't switch to use OpenCV C++ yet the C API is dead and not > >> > supported. > >> > >> Who said that? > >> C++ module filter in libavfilter dealing with latest OpenCV and well > >> maintained is always welcome. > > > > > > So I should be moving the functionality of the current ocv filter into a > > C++ file for filters to use like phqm, the current ocv transforms? > > > > Is this a good plan, separate the C++ from the vf_ filter as I have done > > with img_hash.cpp (yet make it a more generic set of OpenCV C++ > > helper functions to plug into C vf_filters which can be separated by > > function yet unified in code used from the C++ base code)? > > IMHO filter file can be cpp file without issues. And even if not use > ff_ prefixed functions, Okay how about this layout... libavfilter/opencv.cpp (all OpenCV code, using C++ API, helper functions for use by the filters) libafilter/vf_libopencv.c current filter, but uses opencv.cpp functions for OpenCV code libavfilter/vf_phqm.c uses OpenCV functions from opencv.cpp Since OpenCV ranges in varying functions we may need, and doing it as C++ cleanly separately in one place sharing with the C filters seems nice. I agree the duplication of OpenCV in C and C++ APIs seems bad, and using just the C++ API is what I see as best according to the OpenCV devs statements against using the C API. Is that acceptable or is a better layout suggested? Thanks, Christopher > > > > > Wanting to clarify exactly what is acceptable so I can do it and not waste > > time / effort doing stuff that isn't right. > > > >> > > >
On 11/1/19, Christopher Kennedy <ckennedy@ellation.com> wrote: > On Wed, Oct 30, 2019 at 10:57 AM Paul B Mahol <onemda@gmail.com> wrote: >> >> On 10/30/19, Christopher Kennedy <ckennedy@ellation.com> wrote: >> > On Wed, Oct 30, 2019 at 10:07 AM Paul B Mahol <onemda@gmail.com> wrote: >> >> >> >> On 10/30/19, Christopher Kennedy <ckennedy@ellation.com> wrote: >> >> > On Sat, Oct 26, 2019 at 9:15 AM Paul B Mahol <onemda@gmail.com> >> >> > wrote: >> >> >> >> >> >> On 10/26/19, Christopher Kennedy <ckennedy@ellation.com> wrote: >> >> >> > On Sat, Oct 26, 2019 at 8:22 AM Paul B Mahol <onemda@gmail.com> >> >> >> > wrote: >> >> >> >> >> >> >> >> On 10/26/19, Christopher Kennedy <ckennedy@ellation.com> wrote: >> >> >> >> > This is a reference/encode comparison filter with two files >> >> >> >> > input >> >> >> >> > like >> >> >> >> > the psnr or vmaf filter. >> >> >> >> > So it is completely different and uses the C++ OpenCV API >> >> >> >> > since >> >> >> >> > this >> >> >> >> > img_hash library is not in the C API. >> >> >> >> > It's unique to what the OCV filter does, and has more research >> >> >> >> > implications from my talk at Demuxed 2019. >> >> >> >> >> >> >> >> I do not see how that is relevant. >> >> >> >> >> >> >> >> There should be generic opencv filter which could do this above >> >> >> >> in >> >> >> >> generic way, and not by adding yet another filter that uses only >> >> >> >> some >> >> >> >> part of opencv. >> >> >> > >> >> >> > Is it really possible to do framesync() operations like dual >> >> >> > input >> >> >> > filters >> >> >> > like psnr/vmaf and also handle input/output rendering of the >> >> >> > frames >> >> >> > too? >> >> >> > This sounds odd to me but I would love to understand how this is >> >> >> > possible. >> >> >> >> >> >> framesync is nothing special, its just used a lot, there are video >> >> >> filters that do not use it and still operate on multiple inputs. >> >> >> >> >> >> > >> >> >> > The C OpenCV API is not recommended so this does the OpenCV part >> >> >> > in C++ which allows it to be fully utilized and supported. So >> >> >> > that >> >> >> > seems >> >> >> > better to me than using the API OpenCV won't really support and >> >> >> > doesn't >> >> >> > allow usage of img_hash (it is NOT in the C API of OpenCV, >> >> >> > impossible >> >> >> > to >> >> >> > use). >> >> >> > >> >> >> > The OpenCV C++ img_hash library is the fastest implementation and >> >> >> > does >> >> >> > work best in OpenCV. So implementing this in C directly isn't a >> >> >> > task >> >> >> > I >> >> >> > believe >> >> >> > is good to do. >> >> >> >> >> >> I do not care how generic opencv filter is done, it can be C++ just >> >> >> fine. >> >> >> >> >> >> > >> >> >> > So should the current OpenCV stuff be merged into this filter, is >> >> >> > that >> >> >> > possible? >> >> >> >> >> >> Current opencv stuff in libavfilter is pretty dead and not >> >> >> maintained >> >> >> at >> >> >> all. >> >> >> So I'm not really fond of adding yet another opencv filter that not >> >> >> gonna be maintained. >> >> > >> >> > Well I would maintain it, but maybe I should instead run a Fork of >> >> > FFmpeg >> >> > with >> >> > C++ and OpenCV properly done. >> >> > >> >> > Yes the current OCV stuff is dead and not supported by OpenCV or >> >> > maintained >> >> > it sounds like. >> >> > >> >> > So if we can't put C++ into FFmpeg (yet we already have with >> >> > decklink >> >> > filters, >> >> > so that is unfair to say). >> >> > >> >> > And we can't switch to use OpenCV C++ yet the C API is dead and not >> >> > supported. >> >> >> >> Who said that? >> >> C++ module filter in libavfilter dealing with latest OpenCV and well >> >> maintained is always welcome. >> > >> > >> > So I should be moving the functionality of the current ocv filter into >> > a >> > C++ file for filters to use like phqm, the current ocv transforms? >> > >> > Is this a good plan, separate the C++ from the vf_ filter as I have >> > done >> > with img_hash.cpp (yet make it a more generic set of OpenCV C++ >> > helper functions to plug into C vf_filters which can be separated by >> > function yet unified in code used from the C++ base code)? >> >> IMHO filter file can be cpp file without issues. And even if not use >> ff_ prefixed functions, > > Okay how about this layout... > > libavfilter/opencv.cpp (all OpenCV code, using C++ API, helper > functions for use by the filters) > > libafilter/vf_libopencv.c current filter, but uses opencv.cpp > functions for OpenCV code > > libavfilter/vf_phqm.c uses OpenCV functions from opencv.cpp This also should be in libopencv filter. > > Since OpenCV ranges in varying functions we may need, and doing > it as C++ cleanly separately in one place sharing with the C filters seems > nice. > > I agree the duplication of OpenCV in C and C++ APIs seems bad, and > using just the C++ > API is what I see as best according to the OpenCV devs statements > against using the C API. > > > Is that acceptable or is a better layout suggested? > > Thanks, > Christopher > >> >> > >> > Wanting to clarify exactly what is acceptable so I can do it and not >> > waste >> > time / effort doing stuff that isn't right. >> > > >> >> > >> > >
diff --git a/Changelog b/Changelog index 316589e336..4a22f77d37 100644 --- a/Changelog +++ b/Changelog @@ -17,6 +17,7 @@ version <next>: - anlms filter - arnndn filter - bilateral filter +- phqm perceptual hash filter using OpenCV img_lib version 4.2: diff --git a/configure b/configure index 8413826f9e..e231d359bb 100755 --- a/configure +++ b/configure @@ -3497,6 +3497,8 @@ nlmeans_opencl_filter_deps="opencl" nnedi_filter_deps="gpl" ocr_filter_deps="libtesseract" ocv_filter_deps="libopencv" +phqm_filter_deps="libopencv" +phqm_filter_extralibs="-lstdc++ -lopencv_img_hash" openclsrc_filter_deps="opencl" overlay_opencl_filter_deps="opencl" overlay_qsv_filter_deps="libmfx" diff --git a/libavfilter/Makefile b/libavfilter/Makefile index 63d2fba861..645e232b3e 100644 --- a/libavfilter/Makefile +++ b/libavfilter/Makefile @@ -325,6 +325,7 @@ OBJS-$(CONFIG_PERMS_FILTER) += f_perms.o OBJS-$(CONFIG_PERSPECTIVE_FILTER) += vf_perspective.o OBJS-$(CONFIG_PHASE_FILTER) += vf_phase.o OBJS-$(CONFIG_PHOTOSENSITIVITY_FILTER) += vf_photosensitivity.o +OBJS-$(CONFIG_PHQM_FILTER) += vf_phqm.o img_hash.o OBJS-$(CONFIG_PIXDESCTEST_FILTER) += vf_pixdesctest.o OBJS-$(CONFIG_PIXSCOPE_FILTER) += vf_datascope.o OBJS-$(CONFIG_PP_FILTER) += vf_pp.o @@ -498,6 +499,7 @@ OBJS-$(CONFIG_SHARED) += log2_tab.o SKIPHEADERS-$(CONFIG_QSVVPP) += qsvvpp.h SKIPHEADERS-$(CONFIG_OPENCL) += opencl.h SKIPHEADERS-$(CONFIG_VAAPI) += vaapi_vpp.h +SKIPHEADERS-$(CONFIG_LIBOPENCV) += img_hash.h TOOLS = graph2dot TESTPROGS = drawutils filtfmts formats integral diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c index e4186f93db..f0fcaad235 100644 --- a/libavfilter/allfilters.c +++ b/libavfilter/allfilters.c @@ -309,6 +309,7 @@ extern AVFilter ff_vf_perms; extern AVFilter ff_vf_perspective; extern AVFilter ff_vf_phase; extern AVFilter ff_vf_photosensitivity; +extern AVFilter ff_vf_phqm; extern AVFilter ff_vf_pixdesctest; extern AVFilter ff_vf_pixscope; extern AVFilter ff_vf_pp; diff --git a/libavfilter/img_hash.cpp b/libavfilter/img_hash.cpp new file mode 100644 index 0000000000..4d5843da22 --- /dev/null +++ b/libavfilter/img_hash.cpp @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2019 Christopher Kennedy + * + * OpenCV img_hash + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <opencv2/core.hpp> +#include <opencv2/core/ocl.hpp> +#include <opencv2/highgui.hpp> +#include <opencv2/img_hash.hpp> +#include <opencv2/imgproc.hpp> + +#include <iostream> + +#include "img_hash.h" +#include "libavutil/pixdesc.h" +extern "C" { +#include "avfilter.h" +} + +// convert from avframe to iplimage format +static int fill_iplimage_from_frame(IplImage *img, const AVFrame *frame, enum AVPixelFormat pixfmt) +{ + IplImage *tmpimg; + int depth = IPL_DEPTH_8U, channels_nb; + + switch (pixfmt) { + case AV_PIX_FMT_GRAY8: channels_nb = 1; break; + case AV_PIX_FMT_BGRA: channels_nb = 4; break; + case AV_PIX_FMT_BGR24: channels_nb = 3; break; + default: return -1; + } + + tmpimg = cvCreateImageHeader((CvSize){frame->width, frame->height}, depth, channels_nb); + *img = *tmpimg; + img->imageData = img->imageDataOrigin = (char *) frame->data[0]; + img->dataOrder = IPL_DATA_ORDER_PIXEL; + img->origin = IPL_ORIGIN_TL; + img->widthStep = frame->linesize[0]; + + return 0; +} + +// Get the score of two Video Frames by comparing the perceptual hashes and deriving a hamming distance +// showing how similar they are or different. lower score is better for most algorithms +extern "C" double getScore(const AVFrame *frame1, const AVFrame *frame2, enum AVPixelFormat pixfmt, int hash_type) { + cv::Ptr<cv::img_hash::ImgHashBase> algo; + IplImage ipl1, ipl2; + cv::Mat h1; + cv::Mat h2; + cv::Mat m1; + cv::Mat m2; + + // Take FFmpeg video frame and convert into an IplImage for OpenCV + if (fill_iplimage_from_frame(&ipl1, frame1, pixfmt) != 0 || + fill_iplimage_from_frame(&ipl2, frame2, pixfmt) != 0) + return DBL_MAX; // Return an invalid value if either fails + + // Convert an IplImage to an Mat Image for OpenCV (newer format) + m1 = cv::cvarrToMat(&ipl1); + m2 = cv::cvarrToMat(&ipl2); + + // substantiate the hash type algorithm + switch (hash_type) { + case PHASH: algo = cv::img_hash::PHash::create(); break; + case AVERAGE: algo = cv::img_hash::AverageHash::create(); break; + case MARRHILDRETH: algo = cv::img_hash::MarrHildrethHash::create(); break; + case RADIALVARIANCE: algo = cv::img_hash::RadialVarianceHash::create(); break; + // BlockMeanHash support mode 0 and mode 1, they associate to + // mode 1 and mode 2 of PHash library + case BLOCKMEAN1: algo = cv::img_hash::BlockMeanHash::create(0); break; + case BLOCKMEAN2: algo = cv::img_hash::BlockMeanHash::create(1); break; + case COLORMOMENT: algo = cv::img_hash::ColorMomentHash::create(); break; + } + + // Compute the hash + algo->compute(m1, h1); + algo->compute(m2, h2); + + // Compare the hashes and return the hamming distance + return algo->compare(h1, h2); +} diff --git a/libavfilter/img_hash.h b/libavfilter/img_hash.h new file mode 100644 index 0000000000..76f55c3013 --- /dev/null +++ b/libavfilter/img_hash.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2019 Christopher Kennedy + * + * PHQM Perceptual Hash Quality Metric + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef AVFILTER_IMG_HASH_H +#define AVFILTER_IMG_HASH_H + +#include "avfilter.h" + +#if defined(__cplusplus) +extern "C" +{ +#endif + +#define AVERAGE 0 +#define BLOCKMEAN1 1 +#define BLOCKMEAN2 2 +#define COLORMOMENT 3 +#define MARRHILDRETH 4 +#define PHASH 5 +#define RADIALVARIANCE 6 + +double getScore(const AVFrame *frame1, const AVFrame *frame2, enum AVPixelFormat pixfmt, int hash_type); +#if defined(__cplusplus) +} +#endif + +#endif diff --git a/libavfilter/vf_phqm.c b/libavfilter/vf_phqm.c new file mode 100644 index 0000000000..0930386b10 --- /dev/null +++ b/libavfilter/vf_phqm.c @@ -0,0 +1,334 @@ +/* + * Copyright (c) 2019 Christopher Kennedy + * + * PHQM Perceptual Hash Quality Metric + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * PHQM: Calculate the Image Hash Hamming Difference between two input videos. + */ + +#include <float.h> +#include "libavutil/avstring.h" +#include "libavutil/opt.h" +#include "libavutil/pixdesc.h" +#include "avfilter.h" +#include "drawutils.h" +#include "formats.h" +#include "framesync.h" +#include "internal.h" +#include "video.h" + +#include "img_hash.h" +#include "scene_sad.h" + +typedef struct PHQMContext { + const AVClass *class; + FFFrameSync fs; + double shd, hd, min_hd, max_hd, smin_hd, smax_hd; + uint64_t nb_shd; + uint64_t nb_frames; + FILE *stats_file; + char *stats_file_str; + int hash_type; + ff_scene_sad_fn sad; ///< Sum of the absolute difference function (scene detect only) + double prev_mafd; ///< previous MAFD (scene detect only) + AVFrame *prev_picref; ///< previous frame (scene detect only) + double scd_thresh; + double scene_score; +} PHQMContext; + +#define OFFSET(x) offsetof(PHQMContext, x) +#define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM + +static const AVOption phqm_options[] = { + { "stats_file", "Set file where to store per-frame difference information.", OFFSET(stats_file_str), AV_OPT_TYPE_STRING, {.str=NULL}, 0, 0, FLAGS }, + { "f", "Set file where to store per-frame difference information.", OFFSET(stats_file_str), AV_OPT_TYPE_STRING, {.str=NULL}, 0, 0, FLAGS }, + { "scd_thresh", "Scene Change Detection Threshold.", OFFSET(scd_thresh), AV_OPT_TYPE_DOUBLE, {.dbl=0.5}, 0, 1, FLAGS }, + { "hash_type", "Type of Image Hash to use from OpenCV.", OFFSET(hash_type), AV_OPT_TYPE_INT, {.i64 = PHASH}, 0, 6, FLAGS, "hash_type" }, + { "average", "Average Hash", 0, AV_OPT_TYPE_CONST, {.i64 = AVERAGE}, 0, 0, FLAGS, "hash_type" }, + { "blockmean1", "Block Mean Hash 1", 0, AV_OPT_TYPE_CONST, {.i64 = BLOCKMEAN1}, 0, 0, FLAGS, "hash_type" }, + { "blockmean2", "Block Mean Hash 2", 0, AV_OPT_TYPE_CONST, {.i64 = BLOCKMEAN2}, 0, 0, FLAGS, "hash_type" }, + { "colormoment", "Color Moment Hash", 0, AV_OPT_TYPE_CONST, {.i64 = COLORMOMENT}, 0, 0, FLAGS, "hash_type" }, + { "marrhildreth", "Marr Hildreth Hash", 0, AV_OPT_TYPE_CONST, {.i64 = MARRHILDRETH}, 0, 0, FLAGS, "hash_type" }, + { "phash", "Perceptual Hash (PHash)", 0, AV_OPT_TYPE_CONST, {.i64 = PHASH}, 0, 0, FLAGS, "hash_type" }, + { "radialvariance", "Radial Variance Hash", 0, AV_OPT_TYPE_CONST, {.i64 = RADIALVARIANCE}, 0, 0, FLAGS, "hash_type" }, + { NULL } +}; + +FRAMESYNC_DEFINE_CLASS(phqm, PHQMContext, fs); + +static void set_meta(AVDictionary **metadata, const char *key, char comp, float d) +{ + char value[128]; + snprintf(value, sizeof(value), "%0.2f", d); + if (comp) { + char key2[128]; + snprintf(key2, sizeof(key2), "%s%c", key, comp); + av_dict_set(metadata, key2, value, 0); + } else { + av_dict_set(metadata, key, value, 0); + } +} + +static double get_scene_score(AVFilterContext *ctx, AVFrame *frame) +{ + double ret = 0.; + PHQMContext *s = ctx->priv; + AVFrame *prev_picref = s->prev_picref; + + if (prev_picref && + frame->height == prev_picref->height && + frame->width == prev_picref->width) { + uint64_t sad; + double mafd, diff; + + s->sad(prev_picref->data[0], prev_picref->linesize[0], frame->data[0], frame->linesize[0], frame->width * 3, frame->height, &sad); + emms_c(); + mafd = (double)sad / (frame->width * 3 * frame->height); + diff = fabs(mafd - s->prev_mafd); + ret = av_clipf(FFMIN(mafd, diff) / 100., 0, 1); + s->prev_mafd = mafd; + av_frame_free(&prev_picref); + } + s->prev_picref = av_frame_clone(frame); + return ret; +} + +static int do_phqm(FFFrameSync *fs) +{ + AVFilterContext *ctx = fs->parent; + PHQMContext *s = ctx->priv; + AVFrame *master, *ref; + double hd = 0.; + int ret; + double hd_limit = 1000000.; + AVDictionary **metadata; + + ret = ff_framesync_dualinput_get(fs, &master, &ref); + if (ret < 0) + return ret; + if (!ref) + return ff_filter_frame(ctx->outputs[0], master); + metadata = &master->metadata; + + + s->nb_frames++; + + /* scene change detection score */ + s->scene_score = get_scene_score(ctx, ref); + if (s->scene_score >= s->scd_thresh && s->nb_shd >= 48) { + av_log(s, AV_LOG_INFO, "ImgHashScene: n:%"PRId64"-%"PRId64" hd_avg:%0.3lf hd_min:%0.3lf hd_max:%0.3lf scd:%0.2lf\n", + (s->nb_frames - s->nb_shd), s->nb_frames - 1, (s->shd / s->nb_shd), s->smin_hd, s->smax_hd, s->scene_score); + s->shd = 0.; + s->nb_shd = 0; + s->smin_hd = 0.; + s->smax_hd = 0.; + } + + /* limit the highest value so we cut off at perceptual difference match */ + switch (s->hash_type) { + case PHASH: + case AVERAGE: hd_limit = 5; break; + case MARRHILDRETH: hd_limit = 30; break; + case RADIALVARIANCE: hd_limit = 0.9; break; + case BLOCKMEAN1: hd_limit = 12; break; + case BLOCKMEAN2: hd_limit = 48; break; + case COLORMOMENT: hd_limit = 8; break; + } + + /* get ref / enc perceptual hashes and calc hamming distance difference value */ + hd = getScore(ref, master, ref->format, s->hash_type); + if (hd == DBL_MAX) { + av_log(s, AV_LOG_ERROR, "Failure with handling pix_fmt of AVFrame for conversion to IPLimage.\n"); + return AVERROR(EINVAL); + } + s->hd += FFMIN(hd, hd_limit); + set_meta(metadata, "lavfi.phqm.phqm", 0, hd); + + /* scene hamming distance avg */ + s->shd += FFMIN(hd, hd_limit); + s->nb_shd++; + av_log(s, AV_LOG_DEBUG, "ImgHashFrame: hd:%0.3lf scd:%0.2lf\n", hd, s->scene_score); + + s->min_hd = FFMIN(s->min_hd, hd); + s->max_hd = FFMAX(s->max_hd, hd); + s->smin_hd = FFMIN(s->smin_hd, hd); + s->smax_hd = FFMAX(s->smax_hd, hd); + + if (s->stats_file) { + fprintf(s->stats_file, + "n:%"PRId64" phqm:%0.3f phqm_min:%0.3f phqm_max:%0.3f sad:%0.2f", + s->nb_frames, hd, s->min_hd, s->max_hd, s->scene_score); + fprintf(s->stats_file, "\n"); + } + + return ff_filter_frame(ctx->outputs[0], master); +} + +static av_cold int init(AVFilterContext *ctx) +{ + PHQMContext *s = ctx->priv; + + if (s->stats_file_str) { + if (!strcmp(s->stats_file_str, "-")) { + s->stats_file = stdout; + } else { + s->stats_file = fopen(s->stats_file_str, "w"); + if (!s->stats_file) { + int err = AVERROR(errno); + char buf[128]; + av_strerror(err, buf, sizeof(buf)); + av_log(ctx, AV_LOG_ERROR, "Could not open stats file %s: %s\n", + s->stats_file_str, buf); + return err; + } + } + } + + s->sad = ff_scene_sad_get_fn(8); + if (!s->sad) + return AVERROR(EINVAL); + + s->fs.on_event = do_phqm; + return 0; +} + +static int query_formats(AVFilterContext *ctx) +{ + PHQMContext *s = ctx->priv; + AVFilterFormats *fmts_list = NULL; + static const enum AVPixelFormat gray8_pix_fmts[] = { + AV_PIX_FMT_GRAY8, + AV_PIX_FMT_NONE + }; + static const enum AVPixelFormat bgr24_pix_fmts[] = { + AV_PIX_FMT_BGR24, + AV_PIX_FMT_NONE + }; + static const enum AVPixelFormat bgra_pix_fmts[] = { + AV_PIX_FMT_BGRA, + AV_PIX_FMT_NONE + }; + + switch (s->hash_type) { + case COLORMOMENT: fmts_list = ff_make_format_list(bgr24_pix_fmts); break; + case MARRHILDRETH: fmts_list = ff_make_format_list(bgra_pix_fmts); break; + /* all other hashes take the gray8 format */ + default: fmts_list = ff_make_format_list(gray8_pix_fmts); break; + } + if (!fmts_list) + return AVERROR(ENOMEM); + return ff_set_common_formats(ctx, fmts_list); +} + +static int config_input_ref(AVFilterLink *inlink) +{ + AVFilterContext *ctx = inlink->dst; + + if (ctx->inputs[0]->w != ctx->inputs[1]->w || + ctx->inputs[0]->h != ctx->inputs[1]->h) { + av_log(ctx, AV_LOG_ERROR, "Width and height of input videos must be same.\n"); + return AVERROR(EINVAL); + } + if (ctx->inputs[0]->format != ctx->inputs[1]->format) { + av_log(ctx, AV_LOG_ERROR, "Inputs must be of same pixel format.\n"); + return AVERROR(EINVAL); + } + + return 0; +} + +static int config_output(AVFilterLink *outlink) +{ + AVFilterContext *ctx = outlink->src; + PHQMContext *s = ctx->priv; + AVFilterLink *mainlink = ctx->inputs[0]; + int ret; + + ret = ff_framesync_init_dualinput(&s->fs, ctx); + if (ret < 0) + return ret; + outlink->w = mainlink->w; + outlink->h = mainlink->h; + outlink->time_base = mainlink->time_base; + outlink->sample_aspect_ratio = mainlink->sample_aspect_ratio; + outlink->frame_rate = mainlink->frame_rate; + if ((ret = ff_framesync_configure(&s->fs)) < 0) + return ret; + + return 0; +} + +static int activate(AVFilterContext *ctx) +{ + PHQMContext *s = ctx->priv; + return ff_framesync_activate(&s->fs); +} + +static av_cold void uninit(AVFilterContext *ctx) +{ + PHQMContext *s = ctx->priv; + + if (s->nb_frames > 0) + av_log(ctx, AV_LOG_WARNING, "PHQM average:%f min:%f max:%f\n", + s->hd / s->nb_frames, s->min_hd, s->max_hd); + + ff_framesync_uninit(&s->fs); + + if (s->stats_file && s->stats_file != stdout) + fclose(s->stats_file); + av_frame_free(&s->prev_picref); +} + +static const AVFilterPad phqm_inputs[] = { + { + .name = "main", + .type = AVMEDIA_TYPE_VIDEO, + },{ + .name = "reference", + .type = AVMEDIA_TYPE_VIDEO, + .config_props = config_input_ref, + }, + { NULL } +}; + +static const AVFilterPad phqm_outputs[] = { + { + .name = "default", + .type = AVMEDIA_TYPE_VIDEO, + .config_props = config_output, + }, + { NULL } +}; + +AVFilter ff_vf_phqm= { + .name = "phqm", + .description = NULL_IF_CONFIG_SMALL("PHQM: Calculate the Perceptual Hash Hamming Difference between two video streams."), + .preinit = phqm_framesync_preinit, + .init = init, + .uninit = uninit, + .query_formats = query_formats, + .activate = activate, + .priv_size = sizeof(PHQMContext), + .priv_class = &phqm_class, + .inputs = phqm_inputs, + .outputs = phqm_outputs, +};