| Message ID | 20201204170345.11668-1-onemda@gmail.com |
|---|---|
| State | Accepted |
| Headers | show |
| Series | [FFmpeg-devel] avfilter/af_earwax: fix filter behavior | expand |
| Context | Check | Description |
|---|---|---|
| andriy/x86_make | success | Make finished |
| andriy/x86_make_fate | success | Make fate finished |
Will apply soon with few leaks on the allocation error fixed. On Fri, Dec 4, 2020 at 6:03 PM Paul B Mahol <onemda@gmail.com> wrote: > Previous filter output was incorrect. New one actually follows > graph in comments described on side of filter taps. > > Signed-off-by: Paul B Mahol <onemda@gmail.com> > --- > libavfilter/af_earwax.c | 119 +++++++++++++++++++++++++++-------- > tests/fate/filter-audio.mak | 2 +- > tests/ref/fate/filter-earwax | 40 ++++++------ > 3 files changed, 113 insertions(+), 48 deletions(-) > > diff --git a/libavfilter/af_earwax.c b/libavfilter/af_earwax.c > index cdd2b4fc49..951aaccb5d 100644 > --- a/libavfilter/af_earwax.c > +++ b/libavfilter/af_earwax.c > @@ -34,9 +34,9 @@ > #include "audio.h" > #include "formats.h" > > -#define NUMTAPS 64 > +#define NUMTAPS 32 > > -static const int8_t filt[NUMTAPS] = { > +static const int8_t filt[NUMTAPS * 2] = { > /* 30° 330° */ > 4, -6, /* 32 tap stereo FIR filter. */ > 4, -11, /* One side filters as if the */ > @@ -72,7 +72,10 @@ static const int8_t filt[NUMTAPS] = { > 4, 0}; > > typedef struct EarwaxContext { > - int16_t taps[NUMTAPS * 2]; > + int16_t filter[2][NUMTAPS]; > + int16_t taps[4][NUMTAPS * 2]; > + > + AVFrame *frame[2]; > } EarwaxContext; > > static int query_formats(AVFilterContext *ctx) > @@ -83,7 +86,7 @@ static int query_formats(AVFilterContext *ctx) > AVFilterFormats *formats = NULL; > AVFilterChannelLayouts *layout = NULL; > > - if ((ret = ff_add_format (&formats, > AV_SAMPLE_FMT_S16 )) < 0 || > + if ((ret = ff_add_format (&formats, > AV_SAMPLE_FMT_S16P )) < 0 || > (ret = ff_set_common_formats (ctx , formats > )) < 0 || > (ret = ff_add_channel_layout (&layout , > AV_CH_LAYOUT_STEREO )) < 0 || > (ret = ff_set_common_channel_layouts (ctx , layout > )) < 0 || > @@ -94,7 +97,8 @@ static int query_formats(AVFilterContext *ctx) > } > > //FIXME: replace with DSPContext.scalarproduct_int16 > -static inline int16_t *scalarproduct(const int16_t *in, const int16_t > *endin, int16_t *out) > +static inline int16_t *scalarproduct(const int16_t *in, const int16_t > *endin, > + const int16_t *filt, int16_t *out) > { > int32_t sample; > int16_t j; > @@ -111,40 +115,99 @@ static inline int16_t *scalarproduct(const int16_t > *in, const int16_t *endin, in > return out; > } > > -static int filter_frame(AVFilterLink *inlink, AVFrame *insamples) > +static int config_input(AVFilterLink *inlink) > { > - AVFilterLink *outlink = inlink->dst->outputs[0]; > - int16_t *taps, *endin, *in, *out; > - AVFrame *outsamples = ff_get_audio_buffer(outlink, > insamples->nb_samples); > - int len; > + EarwaxContext *s = inlink->dst->priv; > > - if (!outsamples) { > - av_frame_free(&insamples); > - return AVERROR(ENOMEM); > + for (int i = 0; i < NUMTAPS; i++) { > + s->filter[0][i] = filt[i * 2]; > + s->filter[1][i] = filt[i * 2 + 1]; > } > - av_frame_copy_props(outsamples, insamples); > > - taps = ((EarwaxContext *)inlink->dst->priv)->taps; > - out = (int16_t *)outsamples->data[0]; > - in = (int16_t *)insamples ->data[0]; > + return 0; > +} > + > +static void convolve(AVFilterContext *ctx, AVFrame *in, > + int input_ch, int output_ch, > + int filter_ch, int tap_ch) > +{ > + EarwaxContext *s = ctx->priv; > + int16_t *taps, *endin, *dst, *src; > + int len; > + > + taps = s->taps[tap_ch]; > + dst = (int16_t *)s->frame[input_ch]->data[output_ch]; > + src = (int16_t *)in->data[input_ch]; > > - len = FFMIN(NUMTAPS, 2*insamples->nb_samples); > + len = FFMIN(NUMTAPS, in->nb_samples); > // copy part of new input and process with saved input > - memcpy(taps+NUMTAPS, in, len * sizeof(*taps)); > - out = scalarproduct(taps, taps + len, out); > + memcpy(taps+NUMTAPS, src, len * sizeof(*taps)); > + dst = scalarproduct(taps, taps + len, s->filter[filter_ch], dst); > > // process current input > - if (2*insamples->nb_samples >= NUMTAPS ){ > - endin = in + insamples->nb_samples * 2 - NUMTAPS; > - scalarproduct(in, endin, out); > + if (2*in->nb_samples >= NUMTAPS ){ > + endin = src + in->nb_samples - NUMTAPS; > + scalarproduct(src, endin, s->filter[filter_ch], dst); > > // save part of input for next round > memcpy(taps, endin, NUMTAPS * sizeof(*taps)); > - } else > - memmove(taps, taps + 2*insamples->nb_samples, NUMTAPS * > sizeof(*taps)); > + } else { > + memmove(taps, taps + in->nb_samples, NUMTAPS * sizeof(*taps)); > + } > +} > + > +static void mix(AVFilterContext *ctx, AVFrame *out, > + int output_ch, int f0, int f1, int i0, int i1) > +{ > + EarwaxContext *s = ctx->priv; > + const int16_t *srcl = (const int16_t *)s->frame[f0]->data[i0]; > + const int16_t *srcr = (const int16_t *)s->frame[f1]->data[i1]; > + int16_t *dst = (int16_t *)out->data[output_ch]; > + > + for (int n = 0; n < out->nb_samples; n++) > + dst[n] = srcl[n] + srcr[n]; > +} > + > +static int filter_frame(AVFilterLink *inlink, AVFrame *in) > +{ > + AVFilterContext *ctx = inlink->dst; > + EarwaxContext *s = ctx->priv; > + AVFilterLink *outlink = ctx->outputs[0]; > + AVFrame *out = ff_get_audio_buffer(outlink, in->nb_samples); > + > + for (int ch = 0; ch < 2; ch++) { > + if (!s->frame[ch] || s->frame[ch]->nb_samples < in->nb_samples) { > + av_frame_free(&s->frame[ch]); > + s->frame[ch] = ff_get_audio_buffer(outlink, in->nb_samples); > + if (!s->frame[ch]) > + return AVERROR(ENOMEM); > + } > + } > + > + if (!out) { > + av_frame_free(&in); > + return AVERROR(ENOMEM); > + } > + av_frame_copy_props(out, in); > + > + convolve(ctx, in, 0, 0, 0, 0); > + convolve(ctx, in, 0, 1, 1, 1); > + convolve(ctx, in, 1, 0, 0, 2); > + convolve(ctx, in, 1, 1, 1, 3); > + > + mix(ctx, out, 0, 0, 1, 1, 0); > + mix(ctx, out, 1, 0, 1, 0, 1); > + > + av_frame_free(&in); > + return ff_filter_frame(outlink, out); > +} > + > +static av_cold void uninit(AVFilterContext *ctx) > +{ > + EarwaxContext *s = ctx->priv; > > - av_frame_free(&insamples); > - return ff_filter_frame(outlink, outsamples); > + av_frame_free(&s->frame[0]); > + av_frame_free(&s->frame[1]); > } > > static const AVFilterPad earwax_inputs[] = { > @@ -152,6 +215,7 @@ static const AVFilterPad earwax_inputs[] = { > .name = "default", > .type = AVMEDIA_TYPE_AUDIO, > .filter_frame = filter_frame, > + .config_props = config_input, > }, > { NULL } > }; > @@ -169,6 +233,7 @@ AVFilter ff_af_earwax = { > .description = NULL_IF_CONFIG_SMALL("Widen the stereo image."), > .query_formats = query_formats, > .priv_size = sizeof(EarwaxContext), > + .uninit = uninit, > .inputs = earwax_inputs, > .outputs = earwax_outputs, > }; > diff --git a/tests/fate/filter-audio.mak b/tests/fate/filter-audio.mak > index ee1a9dfc97..8b38ee5e75 100644 > --- a/tests/fate/filter-audio.mak > +++ b/tests/fate/filter-audio.mak > @@ -112,7 +112,7 @@ fate-filter-dcshift: CMD = framecrc -i $(SRC) > -frames:a 20 -af aresample,dcshift > FATE_AFILTER-$(call FILTERDEMDECENCMUX, EARWAX, WAV, PCM_S16LE, > PCM_S16LE, WAV) += fate-filter-earwax > fate-filter-earwax: tests/data/asynth-44100-2.wav > fate-filter-earwax: SRC = $(TARGET_PATH)/tests/data/asynth-44100-2.wav > -fate-filter-earwax: CMD = framecrc -i $(SRC) -frames:a 20 -af earwax > +fate-filter-earwax: CMD = framecrc -i $(SRC) -frames:a 20 -af > aresample,earwax,aresample > > FATE_AFILTER-$(call FILTERDEMDECENCMUX, EXTRASTEREO, WAV, PCM_S16LE, > PCM_S16LE, WAV) += fate-filter-extrastereo > fate-filter-extrastereo: tests/data/asynth-44100-2.wav > diff --git a/tests/ref/fate/filter-earwax b/tests/ref/fate/filter-earwax > index 855f579cac..b079ea404b 100644 > --- a/tests/ref/fate/filter-earwax > +++ b/tests/ref/fate/filter-earwax > @@ -4,23 +4,23 @@ > #sample_rate 0: 44100 > #channel_layout 0: 3 > #channel_layout_name 0: stereo > -0, 0, 0, 1024, 4096, 0x900af751 > -0, 1024, 1024, 1024, 4096, 0xad570065 > -0, 2048, 2048, 1024, 4096, 0x93d5f494 > -0, 3072, 3072, 1024, 4096, 0x2c65ef7d > -0, 4096, 4096, 1024, 4096, 0xdc8af6d2 > -0, 5120, 5120, 1024, 4096, 0x7ae00249 > -0, 6144, 6144, 1024, 4096, 0xaab5fdd0 > -0, 7168, 7168, 1024, 4096, 0x4373ef39 > -0, 8192, 8192, 1024, 4096, 0x0756eb43 > -0, 9216, 9216, 1024, 4096, 0x494d06e0 > -0, 10240, 10240, 1024, 4096, 0x4393ffae > -0, 11264, 11264, 1024, 4096, 0x6972f97e > -0, 12288, 12288, 1024, 4096, 0xb834ea05 > -0, 13312, 13312, 1024, 4096, 0x39b8f871 > -0, 14336, 14336, 1024, 4096, 0xf032fccd > -0, 15360, 15360, 1024, 4096, 0xefcd0709 > -0, 16384, 16384, 1024, 4096, 0x0590ebc0 > -0, 17408, 17408, 1024, 4096, 0x2e75f264 > -0, 18432, 18432, 1024, 4096, 0xbea1fd03 > -0, 19456, 19456, 1024, 4096, 0x9bbe0434 > +0, 0, 0, 1024, 4096, 0x76eff65b > +0, 1024, 1024, 1024, 4096, 0x2621ffb7 > +0, 2048, 2048, 1024, 4096, 0x40bfeb89 > +0, 3072, 3072, 1024, 4096, 0xd530f217 > +0, 4096, 4096, 1024, 4096, 0x34f6f5fb > +0, 5120, 5120, 1024, 4096, 0x4aaa04c6 > +0, 6144, 6144, 1024, 4096, 0x73d5fb9f > +0, 7168, 7168, 1024, 4096, 0x5726ee2b > +0, 8192, 8192, 1024, 4096, 0x8b40ec57 > +0, 9216, 9216, 1024, 4096, 0xe5c3052a > +0, 10240, 10240, 1024, 4096, 0xd300f85d > +0, 11264, 11264, 1024, 4096, 0xdedaf89d > +0, 12288, 12288, 1024, 4096, 0xeaa2ea0f > +0, 13312, 13312, 1024, 4096, 0xf014fba1 > +0, 14336, 14336, 1024, 4096, 0x59f2fcdd > +0, 15360, 15360, 1024, 4096, 0x1cbc00b0 > +0, 16384, 16384, 1024, 4096, 0x1423ee75 > +0, 17408, 17408, 1024, 4096, 0xdab7ef27 > +0, 18432, 18432, 1024, 4096, 0xdfa20160 > +0, 19456, 19456, 1024, 4096, 0xe435fca9 > -- > 2.17.1 > >
diff --git a/libavfilter/af_earwax.c b/libavfilter/af_earwax.c index cdd2b4fc49..951aaccb5d 100644 --- a/libavfilter/af_earwax.c +++ b/libavfilter/af_earwax.c @@ -34,9 +34,9 @@ #include "audio.h" #include "formats.h" -#define NUMTAPS 64 +#define NUMTAPS 32 -static const int8_t filt[NUMTAPS] = { +static const int8_t filt[NUMTAPS * 2] = { /* 30° 330° */ 4, -6, /* 32 tap stereo FIR filter. */ 4, -11, /* One side filters as if the */ @@ -72,7 +72,10 @@ static const int8_t filt[NUMTAPS] = { 4, 0}; typedef struct EarwaxContext { - int16_t taps[NUMTAPS * 2]; + int16_t filter[2][NUMTAPS]; + int16_t taps[4][NUMTAPS * 2]; + + AVFrame *frame[2]; } EarwaxContext; static int query_formats(AVFilterContext *ctx) @@ -83,7 +86,7 @@ static int query_formats(AVFilterContext *ctx) AVFilterFormats *formats = NULL; AVFilterChannelLayouts *layout = NULL; - if ((ret = ff_add_format (&formats, AV_SAMPLE_FMT_S16 )) < 0 || + if ((ret = ff_add_format (&formats, AV_SAMPLE_FMT_S16P )) < 0 || (ret = ff_set_common_formats (ctx , formats )) < 0 || (ret = ff_add_channel_layout (&layout , AV_CH_LAYOUT_STEREO )) < 0 || (ret = ff_set_common_channel_layouts (ctx , layout )) < 0 || @@ -94,7 +97,8 @@ static int query_formats(AVFilterContext *ctx) } //FIXME: replace with DSPContext.scalarproduct_int16 -static inline int16_t *scalarproduct(const int16_t *in, const int16_t *endin, int16_t *out) +static inline int16_t *scalarproduct(const int16_t *in, const int16_t *endin, + const int16_t *filt, int16_t *out) { int32_t sample; int16_t j; @@ -111,40 +115,99 @@ static inline int16_t *scalarproduct(const int16_t *in, const int16_t *endin, in return out; } -static int filter_frame(AVFilterLink *inlink, AVFrame *insamples) +static int config_input(AVFilterLink *inlink) { - AVFilterLink *outlink = inlink->dst->outputs[0]; - int16_t *taps, *endin, *in, *out; - AVFrame *outsamples = ff_get_audio_buffer(outlink, insamples->nb_samples); - int len; + EarwaxContext *s = inlink->dst->priv; - if (!outsamples) { - av_frame_free(&insamples); - return AVERROR(ENOMEM); + for (int i = 0; i < NUMTAPS; i++) { + s->filter[0][i] = filt[i * 2]; + s->filter[1][i] = filt[i * 2 + 1]; } - av_frame_copy_props(outsamples, insamples); - taps = ((EarwaxContext *)inlink->dst->priv)->taps; - out = (int16_t *)outsamples->data[0]; - in = (int16_t *)insamples ->data[0]; + return 0; +} + +static void convolve(AVFilterContext *ctx, AVFrame *in, + int input_ch, int output_ch, + int filter_ch, int tap_ch) +{ + EarwaxContext *s = ctx->priv; + int16_t *taps, *endin, *dst, *src; + int len; + + taps = s->taps[tap_ch]; + dst = (int16_t *)s->frame[input_ch]->data[output_ch]; + src = (int16_t *)in->data[input_ch]; - len = FFMIN(NUMTAPS, 2*insamples->nb_samples); + len = FFMIN(NUMTAPS, in->nb_samples); // copy part of new input and process with saved input - memcpy(taps+NUMTAPS, in, len * sizeof(*taps)); - out = scalarproduct(taps, taps + len, out); + memcpy(taps+NUMTAPS, src, len * sizeof(*taps)); + dst = scalarproduct(taps, taps + len, s->filter[filter_ch], dst); // process current input - if (2*insamples->nb_samples >= NUMTAPS ){ - endin = in + insamples->nb_samples * 2 - NUMTAPS; - scalarproduct(in, endin, out); + if (2*in->nb_samples >= NUMTAPS ){ + endin = src + in->nb_samples - NUMTAPS; + scalarproduct(src, endin, s->filter[filter_ch], dst); // save part of input for next round memcpy(taps, endin, NUMTAPS * sizeof(*taps)); - } else - memmove(taps, taps + 2*insamples->nb_samples, NUMTAPS * sizeof(*taps)); + } else { + memmove(taps, taps + in->nb_samples, NUMTAPS * sizeof(*taps)); + } +} + +static void mix(AVFilterContext *ctx, AVFrame *out, + int output_ch, int f0, int f1, int i0, int i1) +{ + EarwaxContext *s = ctx->priv; + const int16_t *srcl = (const int16_t *)s->frame[f0]->data[i0]; + const int16_t *srcr = (const int16_t *)s->frame[f1]->data[i1]; + int16_t *dst = (int16_t *)out->data[output_ch]; + + for (int n = 0; n < out->nb_samples; n++) + dst[n] = srcl[n] + srcr[n]; +} + +static int filter_frame(AVFilterLink *inlink, AVFrame *in) +{ + AVFilterContext *ctx = inlink->dst; + EarwaxContext *s = ctx->priv; + AVFilterLink *outlink = ctx->outputs[0]; + AVFrame *out = ff_get_audio_buffer(outlink, in->nb_samples); + + for (int ch = 0; ch < 2; ch++) { + if (!s->frame[ch] || s->frame[ch]->nb_samples < in->nb_samples) { + av_frame_free(&s->frame[ch]); + s->frame[ch] = ff_get_audio_buffer(outlink, in->nb_samples); + if (!s->frame[ch]) + return AVERROR(ENOMEM); + } + } + + if (!out) { + av_frame_free(&in); + return AVERROR(ENOMEM); + } + av_frame_copy_props(out, in); + + convolve(ctx, in, 0, 0, 0, 0); + convolve(ctx, in, 0, 1, 1, 1); + convolve(ctx, in, 1, 0, 0, 2); + convolve(ctx, in, 1, 1, 1, 3); + + mix(ctx, out, 0, 0, 1, 1, 0); + mix(ctx, out, 1, 0, 1, 0, 1); + + av_frame_free(&in); + return ff_filter_frame(outlink, out); +} + +static av_cold void uninit(AVFilterContext *ctx) +{ + EarwaxContext *s = ctx->priv; - av_frame_free(&insamples); - return ff_filter_frame(outlink, outsamples); + av_frame_free(&s->frame[0]); + av_frame_free(&s->frame[1]); } static const AVFilterPad earwax_inputs[] = { @@ -152,6 +215,7 @@ static const AVFilterPad earwax_inputs[] = { .name = "default", .type = AVMEDIA_TYPE_AUDIO, .filter_frame = filter_frame, + .config_props = config_input, }, { NULL } }; @@ -169,6 +233,7 @@ AVFilter ff_af_earwax = { .description = NULL_IF_CONFIG_SMALL("Widen the stereo image."), .query_formats = query_formats, .priv_size = sizeof(EarwaxContext), + .uninit = uninit, .inputs = earwax_inputs, .outputs = earwax_outputs, }; diff --git a/tests/fate/filter-audio.mak b/tests/fate/filter-audio.mak index ee1a9dfc97..8b38ee5e75 100644 --- a/tests/fate/filter-audio.mak +++ b/tests/fate/filter-audio.mak @@ -112,7 +112,7 @@ fate-filter-dcshift: CMD = framecrc -i $(SRC) -frames:a 20 -af aresample,dcshift FATE_AFILTER-$(call FILTERDEMDECENCMUX, EARWAX, WAV, PCM_S16LE, PCM_S16LE, WAV) += fate-filter-earwax fate-filter-earwax: tests/data/asynth-44100-2.wav fate-filter-earwax: SRC = $(TARGET_PATH)/tests/data/asynth-44100-2.wav -fate-filter-earwax: CMD = framecrc -i $(SRC) -frames:a 20 -af earwax +fate-filter-earwax: CMD = framecrc -i $(SRC) -frames:a 20 -af aresample,earwax,aresample FATE_AFILTER-$(call FILTERDEMDECENCMUX, EXTRASTEREO, WAV, PCM_S16LE, PCM_S16LE, WAV) += fate-filter-extrastereo fate-filter-extrastereo: tests/data/asynth-44100-2.wav diff --git a/tests/ref/fate/filter-earwax b/tests/ref/fate/filter-earwax index 855f579cac..b079ea404b 100644 --- a/tests/ref/fate/filter-earwax +++ b/tests/ref/fate/filter-earwax @@ -4,23 +4,23 @@ #sample_rate 0: 44100 #channel_layout 0: 3 #channel_layout_name 0: stereo -0, 0, 0, 1024, 4096, 0x900af751 -0, 1024, 1024, 1024, 4096, 0xad570065 -0, 2048, 2048, 1024, 4096, 0x93d5f494 -0, 3072, 3072, 1024, 4096, 0x2c65ef7d -0, 4096, 4096, 1024, 4096, 0xdc8af6d2 -0, 5120, 5120, 1024, 4096, 0x7ae00249 -0, 6144, 6144, 1024, 4096, 0xaab5fdd0 -0, 7168, 7168, 1024, 4096, 0x4373ef39 -0, 8192, 8192, 1024, 4096, 0x0756eb43 -0, 9216, 9216, 1024, 4096, 0x494d06e0 -0, 10240, 10240, 1024, 4096, 0x4393ffae -0, 11264, 11264, 1024, 4096, 0x6972f97e -0, 12288, 12288, 1024, 4096, 0xb834ea05 -0, 13312, 13312, 1024, 4096, 0x39b8f871 -0, 14336, 14336, 1024, 4096, 0xf032fccd -0, 15360, 15360, 1024, 4096, 0xefcd0709 -0, 16384, 16384, 1024, 4096, 0x0590ebc0 -0, 17408, 17408, 1024, 4096, 0x2e75f264 -0, 18432, 18432, 1024, 4096, 0xbea1fd03 -0, 19456, 19456, 1024, 4096, 0x9bbe0434 +0, 0, 0, 1024, 4096, 0x76eff65b +0, 1024, 1024, 1024, 4096, 0x2621ffb7 +0, 2048, 2048, 1024, 4096, 0x40bfeb89 +0, 3072, 3072, 1024, 4096, 0xd530f217 +0, 4096, 4096, 1024, 4096, 0x34f6f5fb +0, 5120, 5120, 1024, 4096, 0x4aaa04c6 +0, 6144, 6144, 1024, 4096, 0x73d5fb9f +0, 7168, 7168, 1024, 4096, 0x5726ee2b +0, 8192, 8192, 1024, 4096, 0x8b40ec57 +0, 9216, 9216, 1024, 4096, 0xe5c3052a +0, 10240, 10240, 1024, 4096, 0xd300f85d +0, 11264, 11264, 1024, 4096, 0xdedaf89d +0, 12288, 12288, 1024, 4096, 0xeaa2ea0f +0, 13312, 13312, 1024, 4096, 0xf014fba1 +0, 14336, 14336, 1024, 4096, 0x59f2fcdd +0, 15360, 15360, 1024, 4096, 0x1cbc00b0 +0, 16384, 16384, 1024, 4096, 0x1423ee75 +0, 17408, 17408, 1024, 4096, 0xdab7ef27 +0, 18432, 18432, 1024, 4096, 0xdfa20160 +0, 19456, 19456, 1024, 4096, 0xe435fca9
Previous filter output was incorrect. New one actually follows graph in comments described on side of filter taps. Signed-off-by: Paul B Mahol <onemda@gmail.com> --- libavfilter/af_earwax.c | 119 +++++++++++++++++++++++++++-------- tests/fate/filter-audio.mak | 2 +- tests/ref/fate/filter-earwax | 40 ++++++------ 3 files changed, 113 insertions(+), 48 deletions(-)