diff --git a/libavfilter/formats.c b/libavfilter/formats.c
index 3118aa0925..2379be1518 100644
--- a/libavfilter/formats.c
+++ b/libavfilter/formats.c
@@ -327,7 +327,6 @@ AVFilterChannelLayouts *avfilter_make_format64_list(const int64_t *fmts)
 #define ADD_FORMAT(f, fmt, unref_fn, type, list, nb)        \
 do {                                                        \
     type *fmts;                                             \
-    void *oldf = *f;                                        \
                                                             \
     if (!(*f) && !(*f = av_mallocz(sizeof(**f)))) {         \
         return AVERROR(ENOMEM);                             \
@@ -337,8 +336,6 @@ do {                                                        \
                             sizeof(*(*f)->list));           \
     if (!fmts) {                                            \
         unref_fn(f);                                        \
-        if (!oldf)                                          \
-            av_freep(f);                                    \
         return AVERROR(ENOMEM);                             \
     }                                                       \
                                                             \
@@ -499,16 +496,17 @@ do {                                        \
 do {                                                               \
     int idx = -1;                                                  \
                                                                    \
-    if (!ref || !*ref || !(*ref)->refs)                            \
+    if (!ref || !*ref)                                             \
         return;                                                    \
                                                                    \
     FIND_REF_INDEX(ref, idx);                                      \
                                                                    \
-    if (idx >= 0)                                                  \
+    if (idx >= 0) {                                                \
         memmove((*ref)->refs + idx, (*ref)->refs + idx + 1,        \
             sizeof(*(*ref)->refs) * ((*ref)->refcount - idx - 1)); \
-                                                                   \
-    if(!--(*ref)->refcount) {                                      \
+        --(*ref)->refcount;                                        \
+    }                                                              \
+    if (!(*ref)->refcount) {                                       \
         av_free((*ref)->list);                                     \
         av_free((*ref)->refs);                                     \
         av_free(*ref);                                             \
@@ -550,7 +548,7 @@ void ff_formats_changeref(AVFilterFormats **oldref, AVFilterFormats **newref)
     FORMATS_CHANGEREF(oldref, newref);
 }
 
-#define SET_COMMON_FORMATS(ctx, fmts, in_fmts, out_fmts, ref_fn, unref_fn, list) \
+#define SET_COMMON_FORMATS(ctx, fmts, in_fmts, out_fmts, ref_fn, unref_fn) \
     int count = 0, i;                                               \
                                                                     \
     if (!fmts)                                                      \
@@ -560,10 +558,6 @@ void ff_formats_changeref(AVFilterFormats **oldref, AVFilterFormats **newref)
         if (ctx->inputs[i] && !ctx->inputs[i]->out_fmts) {          \
             int ret = ref_fn(fmts, &ctx->inputs[i]->out_fmts);      \
             if (ret < 0) {                                          \
-                unref_fn(&fmts);                                    \
-                if (fmts)                                           \
-                    av_freep(&fmts->list);                          \
-                av_freep(&fmts);                                    \
                 return ret;                                         \
             }                                                       \
             count++;                                                \
@@ -573,10 +567,6 @@ void ff_formats_changeref(AVFilterFormats **oldref, AVFilterFormats **newref)
         if (ctx->outputs[i] && !ctx->outputs[i]->in_fmts) {         \
             int ret = ref_fn(fmts, &ctx->outputs[i]->in_fmts);      \
             if (ret < 0) {                                          \
-                unref_fn(&fmts);                                    \
-                if (fmts)                                           \
-                    av_freep(&fmts->list);                          \
-                av_freep(&fmts);                                    \
                 return ret;                                         \
             }                                                       \
             count++;                                                \
@@ -584,9 +574,7 @@ void ff_formats_changeref(AVFilterFormats **oldref, AVFilterFormats **newref)
     }                                                               \
                                                                     \
     if (!count) {                                                   \
-        av_freep(&fmts->list);                                      \
-        av_freep(&fmts->refs);                                      \
-        av_freep(&fmts);                                            \
+        unref_fn(&fmts);                                            \
     }                                                               \
                                                                     \
     return 0;
@@ -595,14 +583,14 @@ int ff_set_common_channel_layouts(AVFilterContext *ctx,
                                   AVFilterChannelLayouts *layouts)
 {
     SET_COMMON_FORMATS(ctx, layouts, in_channel_layouts, out_channel_layouts,
-                       ff_channel_layouts_ref, ff_channel_layouts_unref, channel_layouts);
+                       ff_channel_layouts_ref, ff_channel_layouts_unref);
 }
 
 int ff_set_common_samplerates(AVFilterContext *ctx,
                               AVFilterFormats *samplerates)
 {
     SET_COMMON_FORMATS(ctx, samplerates, in_samplerates, out_samplerates,
-                       ff_formats_ref, ff_formats_unref, formats);
+                       ff_formats_ref, ff_formats_unref);
 }
 
 /**
@@ -613,7 +601,7 @@ int ff_set_common_samplerates(AVFilterContext *ctx,
 int ff_set_common_formats(AVFilterContext *ctx, AVFilterFormats *formats)
 {
     SET_COMMON_FORMATS(ctx, formats, in_formats, out_formats,
-                       ff_formats_ref, ff_formats_unref, formats);
+                       ff_formats_ref, ff_formats_unref);
 }
 
 static int default_query_formats_common(AVFilterContext *ctx,
