| Message ID | CAC7Kmr3MwotiqBA8SXUt2owo29O1whqDOFAdE-24_oSXE8NeTQ@mail.gmail.com |
|---|---|
| State | New |
| Headers | show |
On Fri, Nov 04, 2016 at 05:27:34PM +0530, Pallavi Kumari wrote: > patch attached. > > On Fri, Nov 4, 2016 at 3:29 PM, Michael Niedermayer <michael@niedermayer.cc> > wrote: > > > On Fri, Nov 04, 2016 at 07:46:34AM +0530, Pallavi Kumari wrote: > > > Updated patch attached. PFA > > [...] > > > > > Makefile | 1 > > > af_peakpoints.c | 263 ++++++++++++++++++++++++++++++ > > ++++++++++++++++++++++++++ > > > allfilters.c | 1 > > > version.h | 2 > > > 4 files changed, 266 insertions(+), 1 deletion(-) > > > 7f6affa8dcb632cdea162553a18ab1489f0783ce 0001-avfilter-added- > > peakpoints-filter.patch > > > From 46be941c87713f8afee686eed0262ca59a2896fd Mon Sep 17 00:00:00 2001 > > > From: Atana <atana@openmailbox.org> > > > Date: Fri, 4 Nov 2016 07:43:29 +0530 > > > Subject: [PATCH] avfilter: added peakpoints filter > > > > > > --- > > > libavfilter/Makefile | 1 + > > > libavfilter/af_peakpoints.c | 263 ++++++++++++++++++++++++++++++ > > ++++++++++++++ > > > libavfilter/allfilters.c | 1 + > > > libavfilter/version.h | 2 +- > > > 4 files changed, 266 insertions(+), 1 deletion(-) > > > create mode 100644 libavfilter/af_peakpoints.c > > > > > > diff --git a/libavfilter/Makefile b/libavfilter/Makefile > > > index 7ed4696..1a18902 100644 > > > --- a/libavfilter/Makefile > > > +++ b/libavfilter/Makefile > > > @@ -96,6 +96,7 @@ OBJS-$(CONFIG_LADSPA_FILTER) += > > af_ladspa.o > > > OBJS-$(CONFIG_LOUDNORM_FILTER) += af_loudnorm.o > > > OBJS-$(CONFIG_LOWPASS_FILTER) += af_biquads.o > > > OBJS-$(CONFIG_PAN_FILTER) += af_pan.o > > > +OBJS-$(CONFIG_PEAKPOINTS_FILTER) += af_peakpoints.o > > > OBJS-$(CONFIG_REPLAYGAIN_FILTER) += af_replaygain.o > > > OBJS-$(CONFIG_RESAMPLE_FILTER) += af_resample.o > > > OBJS-$(CONFIG_RUBBERBAND_FILTER) += af_rubberband.o > > > diff --git a/libavfilter/af_peakpoints.c b/libavfilter/af_peakpoints.c > > > new file mode 100644 > > > index 0000000..da108ca > > > --- /dev/null > > > +++ b/libavfilter/af_peakpoints.c > > > @@ -0,0 +1,263 @@ > > > +/* > > > + * Copyright (c) 2016 Atana > > > + * > > > + * This file is part of FFmpeg. > > > + * > > > + * FFmpeg is free software; you can redistribute it and/or > > > + * modify it under the terms of the GNU Lesser General Public > > > + * License as published by the Free Software Foundation; either > > > + * version 2.1 of the License, or (at your option) any later version. > > > + * > > > + * FFmpeg is distributed in the hope that it will be useful, > > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > > + * Lesser General Public License for more details. > > > + * > > > + * You should have received a copy of the GNU Lesser General Public > > > + * License along with FFmpeg; if not, write to the Free Software > > > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA > > 02110-1301 USA > > > + */ > > > + > > > +#include "libavcodec/avcodec.h" > > > +#include "libavcodec/avfft.h" > > > +#include "libavformat/avformat.h" > > > +#include "libswscale/swscale.h" > > > +#include "avfilter.h" > > > +#include "audio.h" > > > +#include "libavutil/opt.h" > > > + > > > +#define SIZECHECK 4096 > > > + > > > +/* Structure to contain peak points context */ > > > +typedef struct { > > > + const AVClass *class; > > > + double *data; > > > + int nsamples; > > > + int index; > > > + int isOnce; > > > + int buffFlag; > > > + double *peaks; > > > + int size; // number of peaks > > > + int windowSize; > > > + //char *inputFile; > > > +} PeakPointsContext; > > > + > > > +/* returns maximum value from an array conditioned on start and end > > index */ > > > +static double getMax(double *res_arr, int startIndex, int endIndex) { > > > + int i; > > > + double max = res_arr[startIndex]; > > > + for (i = startIndex; i <= endIndex; i++) { > > > + if (res_arr[i] > max) { > > > + max = res_arr[i]; > > > + } > > > + } > > > > tabs are forbidden in ffmpeg git > > > > > > > + return max; > > > +} > > > + > > > +/* Stores peak frequency for each window(of chunkSize) in peaks array */ > > > +static void getPeakPointInChunk(int chunkSize, double *res_arr, int > > size, double *peaks) { > > > + int i = 0, peakIndex = 0; > > > + int startIndex = 0; > > > + double max; > > > + // get a chunk and find max value in it > > > + while (i < size) { > > > + if (i % chunkSize-1 == 0) { > > > + max = getMax(res_arr, startIndex, i); > > > + peaks[peakIndex++] = max; > > > + startIndex = startIndex + chunkSize; > > > + } > > > + i += 1; > > > + } > > > +} > > > + > > > +/* Get peaks points from windowed frequency domain data*/ > > > +static int getPeakPoints(PeakPointsContext *ppc) { > > > + int i, m, k, size, chunkSize, pSize, chunkSampleSize, resSize; > > > + double *fft_res; > > > + void *avc; > > > + RDFTContext *rdftC; > > > + FFTSample *data; > > > + > > > + size = ppc->index; > > > + m = log2(ppc->windowSize); > > > + chunkSize = ppc->windowSize; > > > + chunkSampleSize = size/chunkSize; > > > + resSize = chunkSize * chunkSampleSize; > > > + > > > > > + fft_res = av_malloc_array(resSize, sizeof(double)); > > > > This seems never freed > > > > > > > + > > > + if (!fft_res) { > > > + av_log(avc, AV_LOG_ERROR, "Cann't allocate memmory for storing > > fft data\n"); > > > + return 0; > > > + } > > > + > > > + > > > + rdftC = av_rdft_init(m, DFT_R2C); > > > + data = av_malloc_array(chunkSize, sizeof(FFTSample)); > > > + > > > + if (!data) { > > > + av_log(avc, AV_LOG_ERROR, "Cann't allocate memmory for chunk > > fft data\n"); > > > + return 0; > > > + } > > > + // FFT transform for windowed time domain data > > > + // window is of size chunkSize > > > + k = 0; > > > + while (k < resSize) { > > > + //copy data > > > + for (i = 0; i < chunkSize; i++) { > > > + data[i] = ppc->data[i+k]; > > > + } > > > + //calculate FFT > > > + av_rdft_calc(rdftC, data); > > > + for (i = 0; i < chunkSize; i++) { > > > + fft_res[i+k] = data[i]; > > > + } > > > + k = k + chunkSize; > > > + } > > > + > > > + av_rdft_end(rdftC); > > > + pSize = resSize/chunkSize; > > > + ppc->size = pSize; > > > > > + ppc->peaks = av_malloc_array(pSize, sizeof(double)); > > > > This is allocated potentially durng each filter_frame > > but only deallocated once at the end, all except the last thus leak > > > > > > [...] > > > +static int filter_frame(AVFilterLink *inlink, AVFrame *samples) > > > +{ > > > + AVFilterContext *ctx = inlink->dst; > > > + PeakPointsContext *p = ctx->priv; > > > + int i, nb_samples = samples->nb_samples; > > > + > > > + // store audio data > > > + for (i = 0; i < nb_samples; i++) { > > > > > + p->data[p->index] = (double)samples->data[0][i]; > > > > This reads the double values as 8bit integers, you have to cast to > > a pointer to double and dereference it to read the double values > > > > > > [...] > > > > > > -- > > Michael GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB > > > > Breaking DRM is a little like attempting to break through a door even > > though the window is wide open and the only thing in the house is a bunch > > of things you dont want and which you would get tomorrow for free anyway > > > > _______________________________________________ > > ffmpeg-devel mailing list > > ffmpeg-devel@ffmpeg.org > > http://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > > > > Makefile | 1 > af_peakpoints.c | 267 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ > allfilters.c | 1 > version.h | 2 > 4 files changed, 270 insertions(+), 1 deletion(-) > 0428e6738c1ea6db9e228a4a4c286e70c138db6b 0001-avfilter-added-peakpoints-filter.patch > From 5b7efa32b1e3970d6accbb23fee02331278b118b Mon Sep 17 00:00:00 2001 > From: Atana <atana@openmailbox.org> > Date: Fri, 4 Nov 2016 07:43:29 +0530 > Subject: [PATCH] avfilter: added peakpoints filter > > --- > libavfilter/Makefile | 1 + > libavfilter/af_peakpoints.c | 267 ++++++++++++++++++++++++++++++++++++++++++++ > libavfilter/allfilters.c | 1 + > libavfilter/version.h | 2 +- > 4 files changed, 270 insertions(+), 1 deletion(-) > create mode 100644 libavfilter/af_peakpoints.c > > diff --git a/libavfilter/Makefile b/libavfilter/Makefile > index 7ed4696..1a18902 100644 > --- a/libavfilter/Makefile > +++ b/libavfilter/Makefile > @@ -96,6 +96,7 @@ OBJS-$(CONFIG_LADSPA_FILTER) += af_ladspa.o > OBJS-$(CONFIG_LOUDNORM_FILTER) += af_loudnorm.o > OBJS-$(CONFIG_LOWPASS_FILTER) += af_biquads.o > OBJS-$(CONFIG_PAN_FILTER) += af_pan.o > +OBJS-$(CONFIG_PEAKPOINTS_FILTER) += af_peakpoints.o > OBJS-$(CONFIG_REPLAYGAIN_FILTER) += af_replaygain.o > OBJS-$(CONFIG_RESAMPLE_FILTER) += af_resample.o > OBJS-$(CONFIG_RUBBERBAND_FILTER) += af_rubberband.o > diff --git a/libavfilter/af_peakpoints.c b/libavfilter/af_peakpoints.c > new file mode 100644 > index 0000000..cb64b82 > --- /dev/null > +++ b/libavfilter/af_peakpoints.c > @@ -0,0 +1,267 @@ > +/* > + * Copyright (c) 2016 Atana > + * > + * This file is part of FFmpeg. > + * > + * FFmpeg is free software; you can redistribute it and/or > + * modify it under the terms of the GNU Lesser General Public > + * License as published by the Free Software Foundation; either > + * version 2.1 of the License, or (at your option) any later version. > + * > + * FFmpeg is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + * Lesser General Public License for more details. > + * > + * You should have received a copy of the GNU Lesser General Public > + * License along with FFmpeg; if not, write to the Free Software > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > + */ > + > +#include "libavcodec/avcodec.h" > +#include "libavcodec/avfft.h" > +#include "libavformat/avformat.h" > +#include "libswscale/swscale.h" > +#include "avfilter.h" > +#include "audio.h" > +#include "libavutil/opt.h" > + > +#define SIZECHECK 4096 > + > +/* Structure to contain peak points context */ > +typedef struct { > + const AVClass *class; > + double *data; > + int nsamples; > + int index; > + int isOnce; > + int buffFlag; > + double *peaks; > + int size; // number of peaks > + int windowSize; > + //char *inputFile; > +} PeakPointsContext; > + > +/* returns maximum value from an array conditioned on start and end index */ > +static double getMax(double *res_arr, int startIndex, int endIndex) { > + int i; > + double max = res_arr[startIndex]; > + for (i = startIndex; i <= endIndex; i++) { > + if (res_arr[i] > max) { > + max = res_arr[i]; > + } > + } > + return max; > +} > + > +/* Stores peak frequency for each window(of chunkSize) in peaks array */ > +static void getPeakPointInChunk(int chunkSize, double *res_arr, int size, double *peaks) { > + int i = 0, peakIndex = 0; > + int startIndex = 0; > + double max; > + // get a chunk and find max value in it > + while (i < size) { > + if (i % chunkSize-1 == 0) { > + max = getMax(res_arr, startIndex, i); > + peaks[peakIndex++] = max; > + startIndex = startIndex + chunkSize; > + } > + i += 1; > + } > +} The frequency components after a R->C FFT are complex numbers the peak values of complex numbers are not the same as the maxima of their real components did the paper this is based on say anything about what is recommanded to take here for the maxima? It would seem logic to me to take the magnitude of the complex numbers instead of the real components individually, but that may or may not be what the paper meant not that we have to do what the paper says, we can in fact do something entirely different if it works better [...] > +static void ppointsStats(AVFilterContext *ctx, PeakPointsContext *p) { > + int i, ret; > + ret = getPeakPoints(p); > + > + if (ret && p->size) { > + // print peaks > + if (p->isOnce) { > + av_log(ctx, AV_LOG_INFO, "######## Peak points are ########\n"); > + p->isOnce = 0; > + } > + for (i = 0; i < p->size; i++) { > + av_log(ctx, AV_LOG_INFO, "%f\n", p->peaks[i]); > + } > + // free peaks and set size to zero > + av_freep(&p->peaks); > + p->size = 0; > + } else if (p->size || !ret) { > + av_log(ctx, AV_LOG_ERROR, "Peak points not retrieved\n"); > + return; > + } > +} This prints the amplitude of the peaks, so if the volume is changed to half, all points change to half Wasnt the paper about the locations of the peaks (which would not change if the volume is changed) ? [...]
>> The frequency components after a R->C FFT are complex numbers >> the peak values of complex numbers are not the same as the maxima of >> their real components >> did the paper this is based on say anything about what is recommanded >> to take here for the maxima? >> It would seem logic to me to take the magnitude of the complex numbers >> instead of the real components individually, but that may or may not >> be what the paper meant >> not that we have to do what the paper says, we can in fact do something >> entirely different if it works better We need points who have high peak intensity(amplitude). Hence, we require amplitude value at frequency points (say x-axis is freq and y is amplitude.) so, I guess taking magnitude of complex numbers is logical. That's how people are using. >> This prints the amplitude of the peaks, so if the volume is changed >> to half, all points change to half >> Wasnt the paper about the locations of the peaks (which would not >> change if the volume is changed) ? (say x-axis is freq and y is amplitude.) We will be creating bins for ex: 0-40, 40-80, 80-120, 120-180, 180-300 frequency values. And for each freq bin interest is to find a points with highest amplitude. A example set(interest point) is 37, 72, 120, 158 From each window an interest point is extracted. On Fri, Nov 4, 2016 at 6:41 PM, Michael Niedermayer <michael@niedermayer.cc> wrote: > On Fri, Nov 04, 2016 at 05:27:34PM +0530, Pallavi Kumari wrote: > > patch attached. > > > > On Fri, Nov 4, 2016 at 3:29 PM, Michael Niedermayer > <michael@niedermayer.cc> > > wrote: > > > > > On Fri, Nov 04, 2016 at 07:46:34AM +0530, Pallavi Kumari wrote: > > > > Updated patch attached. PFA > > > [...] > > > > > > > Makefile | 1 > > > > af_peakpoints.c | 263 ++++++++++++++++++++++++++++++ > > > ++++++++++++++++++++++++++ > > > > allfilters.c | 1 > > > > version.h | 2 > > > > 4 files changed, 266 insertions(+), 1 deletion(-) > > > > 7f6affa8dcb632cdea162553a18ab1489f0783ce 0001-avfilter-added- > > > peakpoints-filter.patch > > > > From 46be941c87713f8afee686eed0262ca59a2896fd Mon Sep 17 00:00:00 > 2001 > > > > From: Atana <atana@openmailbox.org> > > > > Date: Fri, 4 Nov 2016 07:43:29 +0530 > > > > Subject: [PATCH] avfilter: added peakpoints filter > > > > > > > > --- > > > > libavfilter/Makefile | 1 + > > > > libavfilter/af_peakpoints.c | 263 ++++++++++++++++++++++++++++++ > > > ++++++++++++++ > > > > libavfilter/allfilters.c | 1 + > > > > libavfilter/version.h | 2 +- > > > > 4 files changed, 266 insertions(+), 1 deletion(-) > > > > create mode 100644 libavfilter/af_peakpoints.c > > > > > > > > diff --git a/libavfilter/Makefile b/libavfilter/Makefile > > > > index 7ed4696..1a18902 100644 > > > > --- a/libavfilter/Makefile > > > > +++ b/libavfilter/Makefile > > > > @@ -96,6 +96,7 @@ OBJS-$(CONFIG_LADSPA_FILTER) += > > > af_ladspa.o > > > > OBJS-$(CONFIG_LOUDNORM_FILTER) += af_loudnorm.o > > > > OBJS-$(CONFIG_LOWPASS_FILTER) += af_biquads.o > > > > OBJS-$(CONFIG_PAN_FILTER) += af_pan.o > > > > +OBJS-$(CONFIG_PEAKPOINTS_FILTER) += af_peakpoints.o > > > > OBJS-$(CONFIG_REPLAYGAIN_FILTER) += af_replaygain.o > > > > OBJS-$(CONFIG_RESAMPLE_FILTER) += af_resample.o > > > > OBJS-$(CONFIG_RUBBERBAND_FILTER) += af_rubberband.o > > > > diff --git a/libavfilter/af_peakpoints.c > b/libavfilter/af_peakpoints.c > > > > new file mode 100644 > > > > index 0000000..da108ca > > > > --- /dev/null > > > > +++ b/libavfilter/af_peakpoints.c > > > > @@ -0,0 +1,263 @@ > > > > +/* > > > > + * Copyright (c) 2016 Atana > > > > + * > > > > + * This file is part of FFmpeg. > > > > + * > > > > + * FFmpeg is free software; you can redistribute it and/or > > > > + * modify it under the terms of the GNU Lesser General Public > > > > + * License as published by the Free Software Foundation; either > > > > + * version 2.1 of the License, or (at your option) any later > version. > > > > + * > > > > + * FFmpeg is distributed in the hope that it will be useful, > > > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > > > + * Lesser General Public License for more details. > > > > + * > > > > + * You should have received a copy of the GNU Lesser General Public > > > > + * License along with FFmpeg; if not, write to the Free Software > > > > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA > > > 02110-1301 USA > > > > + */ > > > > + > > > > +#include "libavcodec/avcodec.h" > > > > +#include "libavcodec/avfft.h" > > > > +#include "libavformat/avformat.h" > > > > +#include "libswscale/swscale.h" > > > > +#include "avfilter.h" > > > > +#include "audio.h" > > > > +#include "libavutil/opt.h" > > > > + > > > > +#define SIZECHECK 4096 > > > > + > > > > +/* Structure to contain peak points context */ > > > > +typedef struct { > > > > + const AVClass *class; > > > > + double *data; > > > > + int nsamples; > > > > + int index; > > > > + int isOnce; > > > > + int buffFlag; > > > > + double *peaks; > > > > + int size; // number of peaks > > > > + int windowSize; > > > > + //char *inputFile; > > > > +} PeakPointsContext; > > > > + > > > > +/* returns maximum value from an array conditioned on start and end > > > index */ > > > > +static double getMax(double *res_arr, int startIndex, int endIndex) > { > > > > + int i; > > > > + double max = res_arr[startIndex]; > > > > + for (i = startIndex; i <= endIndex; i++) { > > > > + if (res_arr[i] > max) { > > > > + max = res_arr[i]; > > > > + } > > > > + } > > > > > > tabs are forbidden in ffmpeg git > > > > > > > > > > + return max; > > > > +} > > > > + > > > > +/* Stores peak frequency for each window(of chunkSize) in peaks > array */ > > > > +static void getPeakPointInChunk(int chunkSize, double *res_arr, int > > > size, double *peaks) { > > > > + int i = 0, peakIndex = 0; > > > > + int startIndex = 0; > > > > + double max; > > > > + // get a chunk and find max value in it > > > > + while (i < size) { > > > > + if (i % chunkSize-1 == 0) { > > > > + max = getMax(res_arr, startIndex, i); > > > > + peaks[peakIndex++] = max; > > > > + startIndex = startIndex + chunkSize; > > > > + } > > > > + i += 1; > > > > + } > > > > +} > > > > + > > > > +/* Get peaks points from windowed frequency domain data*/ > > > > +static int getPeakPoints(PeakPointsContext *ppc) { > > > > + int i, m, k, size, chunkSize, pSize, chunkSampleSize, resSize; > > > > + double *fft_res; > > > > + void *avc; > > > > + RDFTContext *rdftC; > > > > + FFTSample *data; > > > > + > > > > + size = ppc->index; > > > > + m = log2(ppc->windowSize); > > > > + chunkSize = ppc->windowSize; > > > > + chunkSampleSize = size/chunkSize; > > > > + resSize = chunkSize * chunkSampleSize; > > > > + > > > > > > > + fft_res = av_malloc_array(resSize, sizeof(double)); > > > > > > This seems never freed > > > > > > > > > > + > > > > + if (!fft_res) { > > > > + av_log(avc, AV_LOG_ERROR, "Cann't allocate memmory for > storing > > > fft data\n"); > > > > + return 0; > > > > + } > > > > + > > > > + > > > > + rdftC = av_rdft_init(m, DFT_R2C); > > > > + data = av_malloc_array(chunkSize, sizeof(FFTSample)); > > > > + > > > > + if (!data) { > > > > + av_log(avc, AV_LOG_ERROR, "Cann't allocate memmory for chunk > > > fft data\n"); > > > > + return 0; > > > > + } > > > > + // FFT transform for windowed time domain data > > > > + // window is of size chunkSize > > > > + k = 0; > > > > + while (k < resSize) { > > > > + //copy data > > > > + for (i = 0; i < chunkSize; i++) { > > > > + data[i] = ppc->data[i+k]; > > > > + } > > > > + //calculate FFT > > > > + av_rdft_calc(rdftC, data); > > > > + for (i = 0; i < chunkSize; i++) { > > > > + fft_res[i+k] = data[i]; > > > > + } > > > > + k = k + chunkSize; > > > > + } > > > > + > > > > + av_rdft_end(rdftC); > > > > + pSize = resSize/chunkSize; > > > > + ppc->size = pSize; > > > > > > > + ppc->peaks = av_malloc_array(pSize, sizeof(double)); > > > > > > This is allocated potentially durng each filter_frame > > > but only deallocated once at the end, all except the last thus leak > > > > > > > > > [...] > > > > +static int filter_frame(AVFilterLink *inlink, AVFrame *samples) > > > > +{ > > > > + AVFilterContext *ctx = inlink->dst; > > > > + PeakPointsContext *p = ctx->priv; > > > > + int i, nb_samples = samples->nb_samples; > > > > + > > > > + // store audio data > > > > + for (i = 0; i < nb_samples; i++) { > > > > > > > + p->data[p->index] = (double)samples->data[0][i]; > > > > > > This reads the double values as 8bit integers, you have to cast to > > > a pointer to double and dereference it to read the double values > > > > > > > > > [...] > > > > > > > > > -- > > > Michael GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC7 > 87040B0FAB > > > > > > Breaking DRM is a little like attempting to break through a door even > > > though the window is wide open and the only thing in the house is a > bunch > > > of things you dont want and which you would get tomorrow for free > anyway > > > > > > _______________________________________________ > > > ffmpeg-devel mailing list > > > ffmpeg-devel@ffmpeg.org > > > http://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > > > > > > > > Makefile | 1 > > af_peakpoints.c | 267 ++++++++++++++++++++++++++++++ > ++++++++++++++++++++++++++ > > allfilters.c | 1 > > version.h | 2 > > 4 files changed, 270 insertions(+), 1 deletion(-) > > 0428e6738c1ea6db9e228a4a4c286e70c138db6b 0001-avfilter-added- > peakpoints-filter.patch > > From 5b7efa32b1e3970d6accbb23fee02331278b118b Mon Sep 17 00:00:00 2001 > > From: Atana <atana@openmailbox.org> > > Date: Fri, 4 Nov 2016 07:43:29 +0530 > > Subject: [PATCH] avfilter: added peakpoints filter > > > > --- > > libavfilter/Makefile | 1 + > > libavfilter/af_peakpoints.c | 267 ++++++++++++++++++++++++++++++ > ++++++++++++++ > > libavfilter/allfilters.c | 1 + > > libavfilter/version.h | 2 +- > > 4 files changed, 270 insertions(+), 1 deletion(-) > > create mode 100644 libavfilter/af_peakpoints.c > > > > diff --git a/libavfilter/Makefile b/libavfilter/Makefile > > index 7ed4696..1a18902 100644 > > --- a/libavfilter/Makefile > > +++ b/libavfilter/Makefile > > @@ -96,6 +96,7 @@ OBJS-$(CONFIG_LADSPA_FILTER) += > af_ladspa.o > > OBJS-$(CONFIG_LOUDNORM_FILTER) += af_loudnorm.o > > OBJS-$(CONFIG_LOWPASS_FILTER) += af_biquads.o > > OBJS-$(CONFIG_PAN_FILTER) += af_pan.o > > +OBJS-$(CONFIG_PEAKPOINTS_FILTER) += af_peakpoints.o > > OBJS-$(CONFIG_REPLAYGAIN_FILTER) += af_replaygain.o > > OBJS-$(CONFIG_RESAMPLE_FILTER) += af_resample.o > > OBJS-$(CONFIG_RUBBERBAND_FILTER) += af_rubberband.o > > diff --git a/libavfilter/af_peakpoints.c b/libavfilter/af_peakpoints.c > > new file mode 100644 > > index 0000000..cb64b82 > > --- /dev/null > > +++ b/libavfilter/af_peakpoints.c > > @@ -0,0 +1,267 @@ > > +/* > > + * Copyright (c) 2016 Atana > > + * > > + * This file is part of FFmpeg. > > + * > > + * FFmpeg is free software; you can redistribute it and/or > > + * modify it under the terms of the GNU Lesser General Public > > + * License as published by the Free Software Foundation; either > > + * version 2.1 of the License, or (at your option) any later version. > > + * > > + * FFmpeg is distributed in the hope that it will be useful, > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > + * Lesser General Public License for more details. > > + * > > + * You should have received a copy of the GNU Lesser General Public > > + * License along with FFmpeg; if not, write to the Free Software > > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA > 02110-1301 USA > > + */ > > + > > +#include "libavcodec/avcodec.h" > > +#include "libavcodec/avfft.h" > > +#include "libavformat/avformat.h" > > +#include "libswscale/swscale.h" > > +#include "avfilter.h" > > +#include "audio.h" > > +#include "libavutil/opt.h" > > + > > +#define SIZECHECK 4096 > > + > > +/* Structure to contain peak points context */ > > +typedef struct { > > + const AVClass *class; > > + double *data; > > + int nsamples; > > + int index; > > + int isOnce; > > + int buffFlag; > > + double *peaks; > > + int size; // number of peaks > > + int windowSize; > > + //char *inputFile; > > +} PeakPointsContext; > > + > > +/* returns maximum value from an array conditioned on start and end > index */ > > +static double getMax(double *res_arr, int startIndex, int endIndex) { > > + int i; > > + double max = res_arr[startIndex]; > > + for (i = startIndex; i <= endIndex; i++) { > > + if (res_arr[i] > max) { > > + max = res_arr[i]; > > + } > > + } > > + return max; > > +} > > + > > +/* Stores peak frequency for each window(of chunkSize) in peaks array */ > > +static void getPeakPointInChunk(int chunkSize, double *res_arr, int > size, double *peaks) { > > + int i = 0, peakIndex = 0; > > + int startIndex = 0; > > + double max; > > + // get a chunk and find max value in it > > + while (i < size) { > > + if (i % chunkSize-1 == 0) { > > + max = getMax(res_arr, startIndex, i); > > + peaks[peakIndex++] = max; > > + startIndex = startIndex + chunkSize; > > + } > > + i += 1; > > + } > > +} > > The frequency components after a R->C FFT are complex numbers > the peak values of complex numbers are not the same as the maxima of > their real components > did the paper this is based on say anything about what is recommanded > to take here for the maxima? > > It would seem logic to me to take the magnitude of the complex numbers > instead of the real components individually, but that may or may not > be what the paper meant > not that we have to do what the paper says, we can in fact do something > entirely different if it works better > > [...] > > > +static void ppointsStats(AVFilterContext *ctx, PeakPointsContext *p) { > > + int i, ret; > > + ret = getPeakPoints(p); > > + > > + if (ret && p->size) { > > + // print peaks > > + if (p->isOnce) { > > + av_log(ctx, AV_LOG_INFO, "######## Peak points are > ########\n"); > > + p->isOnce = 0; > > + } > > + for (i = 0; i < p->size; i++) { > > + av_log(ctx, AV_LOG_INFO, "%f\n", p->peaks[i]); > > + } > > + // free peaks and set size to zero > > + av_freep(&p->peaks); > > + p->size = 0; > > + } else if (p->size || !ret) { > > + av_log(ctx, AV_LOG_ERROR, "Peak points not retrieved\n"); > > + return; > > + } > > +} > > This prints the amplitude of the peaks, so if the volume is changed > to half, all points change to half > Wasnt the paper about the locations of the peaks (which would not > change if the volume is changed) ? > > [...] > -- > Michael GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB > > If you think the mosad wants you dead since a long time then you are either > wrong or dead since a long time. > > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > http://ffmpeg.org/mailman/listinfo/ffmpeg-devel > >
On Fri, Nov 04, 2016 at 09:01:22PM +0530, Pallavi Kumari wrote: > >> The frequency components after a R->C FFT are complex numbers > >> the peak values of complex numbers are not the same as the maxima of > >> their real components > >> did the paper this is based on say anything about what is recommanded > >> to take here for the maxima? > > >> It would seem logic to me to take the magnitude of the complex numbers > >> instead of the real components individually, but that may or may not > >> be what the paper meant > >> not that we have to do what the paper says, we can in fact do something > >> entirely different if it works better > > We need points who have high peak intensity(amplitude). Hence, we require > amplitude value at frequency points (say x-axis is freq and y is > amplitude.) so, I guess taking magnitude of complex numbers is logical. > That's how people are using. ok, do you know how to compute that or should i explain it ? [...]
>> ok, do you know how to compute that or should i explain it ? Explanation will help. Is there any function in FFmpeg to get absolute value of FFT? and how to get frequency data? On Fri, Nov 4, 2016 at 11:04 PM, Michael Niedermayer <michael@niedermayer.cc > wrote: > On Fri, Nov 04, 2016 at 09:01:22PM +0530, Pallavi Kumari wrote: > > >> The frequency components after a R->C FFT are complex numbers > > >> the peak values of complex numbers are not the same as the maxima of > > >> their real components > > >> did the paper this is based on say anything about what is recommanded > > >> to take here for the maxima? > > > > >> It would seem logic to me to take the magnitude of the complex numbers > > >> instead of the real components individually, but that may or may not > > >> be what the paper meant > > >> not that we have to do what the paper says, we can in fact do > something > > >> entirely different if it works better > > > > We need points who have high peak intensity(amplitude). Hence, we require > > amplitude value at frequency points (say x-axis is freq and y is > > amplitude.) so, I guess taking magnitude of complex numbers is logical. > > That's how people are using. > > ok, do you know how to compute that or should i explain it ? > > [...] > -- > Michael GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB > > What does censorship reveal? It reveals fear. -- Julian Assange > > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > http://ffmpeg.org/mailman/listinfo/ffmpeg-devel > >
On Fri, Nov 04, 2016 at 11:37:24PM +0530, Pallavi Kumari wrote: > >> ok, do you know how to compute that or should i explain it ? > > Explanation will help. Is there any function in FFmpeg to get absolute > value of FFT? hypot() of the 2 real components of each complex value should work complex numbers are like dots on a paper, if you declare the middle of the paper as 0, the 2 components are the x and y coordinates of a complex number and the distance to the 0 point is the absolute value > and how to get frequency data? the data[i] array after the FFT is so that data[i] are the values for the frequency i. You may need to permute them though see libavcodec/tests/fft.c for how exactly the fft relates to a reference fft [...]
From 5b7efa32b1e3970d6accbb23fee02331278b118b Mon Sep 17 00:00:00 2001 From: Atana <atana@openmailbox.org> Date: Fri, 4 Nov 2016 07:43:29 +0530 Subject: [PATCH] avfilter: added peakpoints filter --- libavfilter/Makefile | 1 + libavfilter/af_peakpoints.c | 267 ++++++++++++++++++++++++++++++++++++++++++++ libavfilter/allfilters.c | 1 + libavfilter/version.h | 2 +- 4 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 libavfilter/af_peakpoints.c diff --git a/libavfilter/Makefile b/libavfilter/Makefile index 7ed4696..1a18902 100644 --- a/libavfilter/Makefile +++ b/libavfilter/Makefile @@ -96,6 +96,7 @@ OBJS-$(CONFIG_LADSPA_FILTER) += af_ladspa.o OBJS-$(CONFIG_LOUDNORM_FILTER) += af_loudnorm.o OBJS-$(CONFIG_LOWPASS_FILTER) += af_biquads.o OBJS-$(CONFIG_PAN_FILTER) += af_pan.o +OBJS-$(CONFIG_PEAKPOINTS_FILTER) += af_peakpoints.o OBJS-$(CONFIG_REPLAYGAIN_FILTER) += af_replaygain.o OBJS-$(CONFIG_RESAMPLE_FILTER) += af_resample.o OBJS-$(CONFIG_RUBBERBAND_FILTER) += af_rubberband.o diff --git a/libavfilter/af_peakpoints.c b/libavfilter/af_peakpoints.c new file mode 100644 index 0000000..cb64b82 --- /dev/null +++ b/libavfilter/af_peakpoints.c @@ -0,0 +1,267 @@ +/* + * Copyright (c) 2016 Atana + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavcodec/avcodec.h" +#include "libavcodec/avfft.h" +#include "libavformat/avformat.h" +#include "libswscale/swscale.h" +#include "avfilter.h" +#include "audio.h" +#include "libavutil/opt.h" + +#define SIZECHECK 4096 + +/* Structure to contain peak points context */ +typedef struct { + const AVClass *class; + double *data; + int nsamples; + int index; + int isOnce; + int buffFlag; + double *peaks; + int size; // number of peaks + int windowSize; + //char *inputFile; +} PeakPointsContext; + +/* returns maximum value from an array conditioned on start and end index */ +static double getMax(double *res_arr, int startIndex, int endIndex) { + int i; + double max = res_arr[startIndex]; + for (i = startIndex; i <= endIndex; i++) { + if (res_arr[i] > max) { + max = res_arr[i]; + } + } + return max; +} + +/* Stores peak frequency for each window(of chunkSize) in peaks array */ +static void getPeakPointInChunk(int chunkSize, double *res_arr, int size, double *peaks) { + int i = 0, peakIndex = 0; + int startIndex = 0; + double max; + // get a chunk and find max value in it + while (i < size) { + if (i % chunkSize-1 == 0) { + max = getMax(res_arr, startIndex, i); + peaks[peakIndex++] = max; + startIndex = startIndex + chunkSize; + } + i += 1; + } +} + +/* Get peaks points from windowed frequency domain data*/ +static int getPeakPoints(PeakPointsContext *ppc) { + int i, m, k, size, chunkSize, pSize, chunkSampleSize, resSize; + double *fft_res; + void *avc; + RDFTContext *rdftC; + FFTSample *data; + + size = ppc->index; + m = log2(ppc->windowSize); + chunkSize = ppc->windowSize; + chunkSampleSize = size/chunkSize; + resSize = chunkSize * chunkSampleSize; + + fft_res = av_malloc_array(resSize, sizeof(double)); + + if (!fft_res) { + av_log(avc, AV_LOG_ERROR, "Cann't allocate memmory for storing fft data\n"); + return 0; + } + + + rdftC = av_rdft_init(m, DFT_R2C); + data = av_malloc_array(chunkSize, sizeof(FFTSample)); + + if (!data) { + av_log(avc, AV_LOG_ERROR, "Cann't allocate memmory for chunk fft data\n"); + return 0; + } + // FFT transform for windowed time domain data + // window is of size chunkSize + k = 0; + while (k < resSize) { + //copy data + for (i = 0; i < chunkSize; i++) { + data[i] = ppc->data[i+k]; + } + //calculate FFT + av_rdft_calc(rdftC, data); + for (i = 0; i < chunkSize; i++) { + fft_res[i+k] = data[i]; + } + k = k + chunkSize; + } + + av_rdft_end(rdftC); + pSize = resSize/chunkSize; + ppc->size = pSize; + ppc->peaks = av_malloc_array(pSize, sizeof(double)); + + if (!ppc->peaks) { + av_log(avc, AV_LOG_ERROR, "Cann't allocate memory for peak storage\n"); + return 0; + } + + getPeakPointInChunk(chunkSize, fft_res, resSize, ppc->peaks); + av_freep(&data); + av_freep(&fft_res); + return 1; +} + + +#define OFFSET(x) offsetof(PeakPointsContext, x) + +static const AVOption peakpoints_options[] = { + { "wsize", "set window size", OFFSET(windowSize), AV_OPT_TYPE_INT, {.i64=16}, 0, INT_MAX}, + { NULL }, +}; + +AVFILTER_DEFINE_CLASS(peakpoints); + +static av_cold int init(AVFilterContext *ctx) +{ + PeakPointsContext *p = ctx->priv; + + if (p->windowSize < 16 || p->windowSize > SIZECHECK) { + av_log(ctx, AV_LOG_ERROR, "window size must be in range 16 to %d\n", SIZECHECK); + return AVERROR(EINVAL); + } + + p->index = 0; + p->size = 0; + p->isOnce = 1; + p->data = av_malloc_array(SIZECHECK, sizeof(double)); + + if (!p->data) { + av_log(ctx, AV_LOG_ERROR, "Cann't allocate memmory for audio data\n"); + return AVERROR(EINVAL); + } + + return 0; +} + +static void ppointsStats(AVFilterContext *ctx, PeakPointsContext *p) { + int i, ret; + ret = getPeakPoints(p); + + if (ret && p->size) { + // print peaks + if (p->isOnce) { + av_log(ctx, AV_LOG_INFO, "######## Peak points are ########\n"); + p->isOnce = 0; + } + for (i = 0; i < p->size; i++) { + av_log(ctx, AV_LOG_INFO, "%f\n", p->peaks[i]); + } + // free peaks and set size to zero + av_freep(&p->peaks); + p->size = 0; + } else if (p->size || !ret) { + av_log(ctx, AV_LOG_ERROR, "Peak points not retrieved\n"); + return; + } +} + +static int query_formats(AVFilterContext *ctx) +{ + static const enum AVSampleFormat sample_fmts[] = { + AV_SAMPLE_FMT_DBL, + AV_SAMPLE_FMT_DBLP, + AV_SAMPLE_FMT_NONE + }; + AVFilterFormats *formats; + + if (!(formats = ff_make_format_list(sample_fmts))) + return AVERROR(ENOMEM); + return ff_set_common_formats(ctx, formats); +} + +static int filter_frame(AVFilterLink *inlink, AVFrame *samples) +{ + AVFilterContext *ctx = inlink->dst; + PeakPointsContext *p = ctx->priv; + int i, nb_samples = samples->nb_samples; + + // store audio data + for (i = 0; i < nb_samples; i++) { + p->data[p->index] = ((double*)(samples->data[0]))[i]; + p->buffFlag = 1; + p->index = p->index + 1; + + // size check + if (p->index == SIZECHECK) { + // get peak points stats + ppointsStats(ctx, p); + p->index = 0; + p->buffFlag = 1; + } + } + + return ff_filter_frame(inlink->dst->outputs[0], samples); +} + +static av_cold void uninit(AVFilterContext *ctx) +{ + PeakPointsContext *p = ctx->priv; + + // if audio data in buffer get peak points + if (p->buffFlag) { + ppointsStats(ctx, p); + } + + // free allocated memories + av_freep(&p->data); + av_freep(&p->peaks); +} + +static const AVFilterPad peakpoints_inputs[] = { + { + .name = "default", + .type = AVMEDIA_TYPE_AUDIO, + .filter_frame = filter_frame, + }, + { NULL } +}; + +static const AVFilterPad peakpoints_outputs[] = { + { + .name = "default", + .type = AVMEDIA_TYPE_AUDIO, + }, + { NULL } +}; + +AVFilter ff_af_peakpoints = { + .name = "peakpoints", + .description = NULL_IF_CONFIG_SMALL("peak points from frequency domain windowed data."), + .init = init, + .uninit = uninit, + .query_formats = query_formats, + .priv_size = sizeof(PeakPointsContext), + .inputs = peakpoints_inputs, + .outputs = peakpoints_outputs, + .priv_class = &peakpoints_class, +}; diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c index 82a65ee..b3b0330 100644 --- a/libavfilter/allfilters.c +++ b/libavfilter/allfilters.c @@ -114,6 +114,7 @@ void avfilter_register_all(void) REGISTER_FILTER(LOUDNORM, loudnorm, af); REGISTER_FILTER(LOWPASS, lowpass, af); REGISTER_FILTER(PAN, pan, af); + REGISTER_FILTER(PEAKPOINTS, peakpoints, af); REGISTER_FILTER(REPLAYGAIN, replaygain, af); REGISTER_FILTER(RESAMPLE, resample, af); REGISTER_FILTER(RUBBERBAND, rubberband, af); diff --git a/libavfilter/version.h b/libavfilter/version.h index 82a5f63..b8c9b81 100644 --- a/libavfilter/version.h +++ b/libavfilter/version.h @@ -30,7 +30,7 @@ #include "libavutil/version.h" #define LIBAVFILTER_VERSION_MAJOR 6 -#define LIBAVFILTER_VERSION_MINOR 66 +#define LIBAVFILTER_VERSION_MINOR 67 #define LIBAVFILTER_VERSION_MICRO 100 #define LIBAVFILTER_VERSION_INT AV_VERSION_INT(LIBAVFILTER_VERSION_MAJOR, \ -- 1.9.1