diff --git a/libavfilter/qsvvpp.c b/libavfilter/qsvvpp.c
index 4768f6208b..5b0b30e23c 100644
--- a/libavfilter/qsvvpp.c
+++ b/libavfilter/qsvvpp.c
@@ -647,15 +647,11 @@ static unsigned int qsv_fifo_size(const AVFifoBuffer* fifo)
     return  av_fifo_size(fifo)/qsv_fifo_item_size();
 }
 
-int ff_qsvvpp_create(AVFilterContext *avctx, QSVVPPContext **vpp, QSVVPPParam *param)
+int ff_qsvvpp_init(AVFilterContext *avctx, QSVVPPParam *param)
 {
     int i;
     int ret;
-    QSVVPPContext *s;
-
-    s = av_mallocz(sizeof(*s));
-    if (!s)
-        return AVERROR(ENOMEM);
+    QSVVPPContext *s = avctx->priv;
 
     s->filter_frame  = param->filter_frame;
     if (!s->filter_frame)
@@ -722,14 +718,13 @@ int ff_qsvvpp_create(AVFilterContext *avctx, QSVVPPContext **vpp, QSVVPPParam *p
     s->got_frame = 0;
 
     /** keep fifo size at least 1. Even when async_depth is 0, fifo is used. */
-    s->async_fifo  = av_fifo_alloc((param->async_depth + 1) * qsv_fifo_item_size());
-    s->async_depth = param->async_depth;
+    s->async_fifo  = av_fifo_alloc((s->async_depth + 1) * qsv_fifo_item_size());
     if (!s->async_fifo) {
         ret = AVERROR(ENOMEM);
         goto failed;
     }
 
-    s->vpp_param.AsyncDepth = param->async_depth;
+    s->vpp_param.AsyncDepth = s->async_depth;
 
     if (IS_SYSTEM_MEMORY(s->in_mem_mode))
         s->vpp_param.IOPattern |= MFX_IOPATTERN_IN_SYSTEM_MEMORY;
@@ -756,25 +751,22 @@ int ff_qsvvpp_create(AVFilterContext *avctx, QSVVPPContext **vpp, QSVVPPParam *p
     } else if (ret > 0)
         ff_qsvvpp_print_warning(avctx, ret, "Warning When creating qsvvpp");
 
-    *vpp = s;
     return 0;
 
 failed:
-    ff_qsvvpp_free(&s);
+    ff_qsvvpp_close(avctx);
 
     return ret;
 }
 
-int ff_qsvvpp_free(QSVVPPContext **vpp)
+int ff_qsvvpp_close(AVFilterContext *avctx)
 {
-    QSVVPPContext *s = *vpp;
-
-    if (!s)
-        return 0;
+    QSVVPPContext *s = avctx->priv;
 
     if (s->session) {
         MFXVideoVPP_Close(s->session);
         MFXClose(s->session);
+        s->session = NULL;
     }
 
     /* release all the resources */
@@ -785,7 +777,6 @@ int ff_qsvvpp_free(QSVVPPContext **vpp)
     av_freep(&s->ext_buffers);
     av_freep(&s->frame_infos);
     av_fifo_free(s->async_fifo);
-    av_freep(vpp);
 
     return 0;
 }
diff --git a/libavfilter/qsvvpp.h b/libavfilter/qsvvpp.h
index e0f4c8f5bb..b6fe0d3fa7 100644
--- a/libavfilter/qsvvpp.h
+++ b/libavfilter/qsvvpp.h
@@ -48,6 +48,8 @@ typedef struct QSVFrame {
 } QSVFrame;
 
 typedef struct QSVVPPContext {
+    const AVClass      *class;
+
     mfxSession          session;
     int (*filter_frame) (AVFilterLink *outlink, AVFrame *frame); /**< callback */
     enum AVPixelFormat  out_sw_format;   /**< Real output format */
@@ -95,15 +97,13 @@ typedef struct QSVVPPParam {
     /* Crop information for each input, if needed */
     int num_crop;
     QSVVPPCrop *crop;
-
-   int async_depth;
 } QSVVPPParam;
 
 /* create and initialize the QSV session */
-int ff_qsvvpp_create(AVFilterContext *avctx, QSVVPPContext **vpp, QSVVPPParam *param);
+int ff_qsvvpp_init(AVFilterContext *avctx, QSVVPPParam *param);
 
 /* release the resources (eg.surfaces) */
-int ff_qsvvpp_free(QSVVPPContext **vpp);
+int ff_qsvvpp_close(AVFilterContext *avctx);
 
 /* vpp filter frame and call the cb if needed */
 int ff_qsvvpp_filter_frame(QSVVPPContext *vpp, AVFilterLink *inlink, AVFrame *frame);
diff --git a/libavfilter/vf_overlay_qsv.c b/libavfilter/vf_overlay_qsv.c
index 14c4c37a3c..e7db6adf4b 100644
--- a/libavfilter/vf_overlay_qsv.c
+++ b/libavfilter/vf_overlay_qsv.c
@@ -57,10 +57,9 @@ enum var_name {
 };
 
 typedef struct QSVOverlayContext {
-    const AVClass      *class;
+    QSVVPPContext      qsv;
 
     FFFrameSync fs;
-    QSVVPPContext      *qsv;
     QSVVPPParam        qsv_param;
     mfxExtVPPComposite comp_conf;
     double             var_values[VAR_VARS_NB];
@@ -230,14 +229,14 @@ static int config_overlay_input(AVFilterLink *inlink)
 static int process_frame(FFFrameSync *fs)
 {
     AVFilterContext  *ctx = fs->parent;
-    QSVOverlayContext  *s = fs->opaque;
+    QSVVPPContext    *qsv = fs->opaque;
     AVFrame        *frame = NULL;
     int               ret = 0, i;
 
     for (i = 0; i < ctx->nb_inputs; i++) {
         ret = ff_framesync_get_frame(fs, i, &frame, 0);
         if (ret == 0)
-            ret = ff_qsvvpp_filter_frame(s->qsv, ctx->inputs[i], frame);
+            ret = ff_qsvvpp_filter_frame(qsv, ctx->inputs[i], frame);
         if (ret < 0 && ret != AVERROR(EAGAIN))
             break;
     }
@@ -299,7 +298,7 @@ static int config_output(AVFilterLink *outlink)
     if (ret < 0)
         return ret;
 
-    return ff_qsvvpp_create(ctx, &vpp->qsv, &vpp->qsv_param);
+    return ff_qsvvpp_init(ctx, &vpp->qsv_param);
 }
 
 /*
@@ -348,7 +347,7 @@ static av_cold void overlay_qsv_uninit(AVFilterContext *ctx)
 {
     QSVOverlayContext *vpp = ctx->priv;
 
-    ff_qsvvpp_free(&vpp->qsv);
+    ff_qsvvpp_close(ctx);
     ff_framesync_uninit(&vpp->fs);
     av_freep(&vpp->comp_conf.InputStream);
     av_freep(&vpp->qsv_param.ext_buf);
diff --git a/libavfilter/vf_vpp_qsv.c b/libavfilter/vf_vpp_qsv.c
index 70bd5310c3..5b0b07adf5 100644
--- a/libavfilter/vf_vpp_qsv.c
+++ b/libavfilter/vf_vpp_qsv.c
@@ -48,9 +48,7 @@
 #define QSV_HAVE_SCALING_CONFIG QSV_VERSION_ATLEAST(1, 19)
 
 typedef struct VPPContext{
-    const AVClass *class;
-
-    QSVVPPContext *qsv;
+    QSVVPPContext qsv;
 
     /* Video Enhancement Algorithms */
     mfxExtVPPDeinterlacing  deinterlace_conf;
@@ -99,9 +97,6 @@ typedef struct VPPContext{
     char *cx, *cy, *cw, *ch;
     char *ow, *oh;
     char *output_format_str;
-
-    int async_depth;
-    int eof;
 } VPPContext;
 
 static const AVOption options[] = {
@@ -137,7 +132,7 @@ static const AVOption options[] = {
     { "h",      "Output video height", OFFSET(oh), AV_OPT_TYPE_STRING, { .str="w*ch/cw" }, 0, 255, .flags = FLAGS },
     { "height", "Output video height", OFFSET(oh), AV_OPT_TYPE_STRING, { .str="w*ch/cw" }, 0, 255, .flags = FLAGS },
     { "format", "Output pixel format", OFFSET(output_format_str), AV_OPT_TYPE_STRING, { .str = "same" }, .flags = FLAGS },
-    { "async_depth", "Internal parallelization depth, the higher the value the higher the latency.", OFFSET(async_depth), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, .flags = FLAGS },
+    { "async_depth", "Internal parallelization depth, the higher the value the higher the latency.", OFFSET(qsv.async_depth), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, .flags = FLAGS },
     { "scale_mode", "scale mode: 0=auto, 1=low power, 2=high quality", OFFSET(scale_mode), AV_OPT_TYPE_INT, { .i64 = MFX_SCALING_MODE_DEFAULT }, MFX_SCALING_MODE_DEFAULT, MFX_SCALING_MODE_QUALITY, .flags = FLAGS, "scale mode" },
 
     { NULL }
@@ -314,7 +309,6 @@ static int config_output(AVFilterLink *outlink)
     param.filter_frame  = NULL;
     param.num_ext_buf   = 0;
     param.ext_buf       = ext_buf;
-    param.async_depth   = vpp->async_depth;
 
     if (inlink->format == AV_PIX_FMT_QSV) {
          if (!inlink->hw_frames_ctx || !inlink->hw_frames_ctx->data)
@@ -483,8 +477,9 @@ static int config_output(AVFilterLink *outlink)
     if (vpp->use_frc || vpp->use_crop || vpp->deinterlace || vpp->denoise ||
         vpp->detail || vpp->procamp || vpp->rotate || vpp->hflip ||
         inlink->w != outlink->w || inlink->h != outlink->h || in_format != vpp->out_format)
-        return ff_qsvvpp_create(ctx, &vpp->qsv, &param);
+        return ff_qsvvpp_init(ctx, &param);
     else {
+        /* No MFX session is created in this case */
         av_log(ctx, AV_LOG_VERBOSE, "qsv vpp pass through mode.\n");
         if (inlink->hw_frames_ctx)
             outlink->hw_frames_ctx = av_buffer_ref(inlink->hw_frames_ctx);
@@ -497,33 +492,31 @@ static int activate(AVFilterContext *ctx)
 {
     AVFilterLink *inlink = ctx->inputs[0];
     AVFilterLink *outlink = ctx->outputs[0];
-    VPPContext *s =ctx->priv;
-    QSVVPPContext *qsv = s->qsv;
+    QSVVPPContext *qsv = ctx->priv;
     AVFrame *in = NULL;
     int ret, status;
     int64_t pts;
 
     FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink);
 
-    if (!s->eof) {
+    if (!qsv->eof) {
         ret = ff_inlink_consume_frame(inlink, &in);
         if (ret < 0)
             return ret;
 
         if (ff_inlink_acknowledge_status(inlink, &status, &pts)) {
             if (status == AVERROR_EOF) {
-                s->eof = 1;
+                qsv->eof = 1;
             }
         }
     }
 
-    if (qsv) {
-        if (in || s->eof) {
-            qsv->eof = s->eof;
+    if (qsv->session) {
+        if (in || qsv->eof) {
             ret = ff_qsvvpp_filter_frame(qsv, inlink, in);
             av_frame_free(&in);
 
-            if (s->eof) {
+            if (qsv->eof) {
                 ff_outlink_set_status(outlink, status, pts);
                 return 0;
             }
@@ -534,6 +527,7 @@ static int activate(AVFilterContext *ctx)
             }
         }
     } else {
+        /* No MFX session is created in pass-through mode */
         if (in) {
             if (in->pts != AV_NOPTS_VALUE)
                 in->pts = av_rescale_q(in->pts, inlink->time_base, outlink->time_base);
@@ -543,7 +537,7 @@ static int activate(AVFilterContext *ctx)
         }
     }
 
-    if (s->eof) {
+    if (qsv->eof) {
         ff_outlink_set_status(outlink, status, pts);
         return 0;
     } else {
@@ -581,9 +575,7 @@ static int query_formats(AVFilterContext *ctx)
 
 static av_cold void vpp_uninit(AVFilterContext *ctx)
 {
-    VPPContext *vpp = ctx->priv;
-
-    ff_qsvvpp_free(&vpp->qsv);
+    ff_qsvvpp_close(ctx);
 }
 
 static const AVClass vpp_class = {
