diff --git a/libavcodec/vaapi_decode.c b/libavcodec/vaapi_decode.c
index 69512e1d45..5fc9767802 100644
--- a/libavcodec/vaapi_decode.c
+++ b/libavcodec/vaapi_decode.c
@@ -408,7 +408,8 @@ static const struct {
 static int vaapi_decode_make_config(AVCodecContext *avctx,
                                     AVBufferRef *device_ref,
                                     VAConfigID *va_config,
-                                    AVBufferRef *frames_ref)
+                                    AVBufferRef *frames_ref,
+                                    int dpb_size)
 {
     AVVAAPIHWConfig       *hwconfig    = NULL;
     AVHWFramesConstraints *constraints = NULL;
@@ -549,22 +550,8 @@ static int vaapi_decode_make_config(AVCodecContext *avctx,
         if (err < 0)
             goto fail;
 
-        frames->initial_pool_size = 1;
-        // Add per-codec number of surfaces used for storing reference frames.
-        switch (avctx->codec_id) {
-        case AV_CODEC_ID_H264:
-        case AV_CODEC_ID_HEVC:
-            frames->initial_pool_size += 16;
-            break;
-        case AV_CODEC_ID_VP9:
-            frames->initial_pool_size += 8;
-            break;
-        case AV_CODEC_ID_VP8:
-            frames->initial_pool_size += 3;
-            break;
-        default:
-            frames->initial_pool_size += 2;
-        }
+        if (dpb_size > 0)
+            frames->initial_pool_size = dpb_size + 1;
     }
 
     av_hwframe_constraints_free(&constraints);
@@ -583,8 +570,9 @@ fail:
     return err;
 }
 
-int ff_vaapi_common_frame_params(AVCodecContext *avctx,
-                                 AVBufferRef *hw_frames_ctx)
+int ff_vaapi_frame_params_with_dpb_size(AVCodecContext *avctx,
+                                        AVBufferRef *hw_frames_ctx,
+                                        int dpb_size)
 {
     AVHWFramesContext *hw_frames = (AVHWFramesContext *)hw_frames_ctx->data;
     AVHWDeviceContext *device_ctx = hw_frames->device_ctx;
@@ -597,7 +585,7 @@ int ff_vaapi_common_frame_params(AVCodecContext *avctx,
     hwctx = device_ctx->hwctx;
 
     err = vaapi_decode_make_config(avctx, hw_frames->device_ref, &va_config,
-                                   hw_frames_ctx);
+                                   hw_frames_ctx, dpb_size);
     if (err)
         return err;
 
@@ -607,6 +595,13 @@ int ff_vaapi_common_frame_params(AVCodecContext *avctx,
     return 0;
 }
 
+int ff_vaapi_common_frame_params(AVCodecContext *avctx,
+                                 AVBufferRef *hw_frames_ctx)
+{
+    // Set common dpb_size for vc1/mjpeg/mpeg2/mpeg4.
+    return ff_vaapi_frame_params_with_dpb_size(avctx, hw_frames_ctx, 2);
+}
+
 int ff_vaapi_decode_init(AVCodecContext *avctx)
 {
     VAAPIDecodeContext *ctx = avctx->internal->hwaccel_priv_data;
@@ -666,7 +661,9 @@ int ff_vaapi_decode_init(AVCodecContext *avctx)
     ctx->hwctx  = ctx->device->hwctx;
 
     err = vaapi_decode_make_config(avctx, ctx->frames->device_ref,
-                                   &ctx->va_config, avctx->hw_frames_ctx);
+                                   &ctx->va_config, avctx->hw_frames_ctx,
+                                   0);
+
     if (err)
         goto fail;
 
diff --git a/libavcodec/vaapi_decode.h b/libavcodec/vaapi_decode.h
index 6b415dd1d3..c3e74bf9c7 100644
--- a/libavcodec/vaapi_decode.h
+++ b/libavcodec/vaapi_decode.h
@@ -98,6 +98,9 @@ int ff_vaapi_decode_cancel(AVCodecContext *avctx,
 int ff_vaapi_decode_init(AVCodecContext *avctx);
 int ff_vaapi_decode_uninit(AVCodecContext *avctx);
 
+int ff_vaapi_frame_params_with_dpb_size(AVCodecContext *avctx,
+                                        AVBufferRef *hw_frames_ctx,
+                                        int dpb_size);
 int ff_vaapi_common_frame_params(AVCodecContext *avctx,
                                  AVBufferRef *hw_frames_ctx);
 
diff --git a/libavcodec/vaapi_h264.c b/libavcodec/vaapi_h264.c
index dd2a657160..8d7f5c2004 100644
--- a/libavcodec/vaapi_h264.c
+++ b/libavcodec/vaapi_h264.c
@@ -385,6 +385,15 @@ static int vaapi_h264_decode_slice(AVCodecContext *avctx,
     return 0;
 }
 
+static int ff_vaapi_h264_frame_params(AVCodecContext *avctx,
+                                   AVBufferRef *hw_frames_ctx)
+{
+    const H264Context *h = avctx->priv_data;
+    const SPS       *sps = h->ps.sps;
+    return ff_vaapi_frame_params_with_dpb_size(avctx, hw_frames_ctx,
+                                               sps->ref_frame_count + sps->num_reorder_frames);
+}
+
 const AVHWAccel ff_h264_vaapi_hwaccel = {
     .name                 = "h264_vaapi",
     .type                 = AVMEDIA_TYPE_VIDEO,
@@ -396,7 +405,7 @@ const AVHWAccel ff_h264_vaapi_hwaccel = {
     .frame_priv_data_size = sizeof(VAAPIDecodePicture),
     .init                 = &ff_vaapi_decode_init,
     .uninit               = &ff_vaapi_decode_uninit,
-    .frame_params         = &ff_vaapi_common_frame_params,
+    .frame_params         = &ff_vaapi_h264_frame_params,
     .priv_data_size       = sizeof(VAAPIDecodeContext),
     .caps_internal        = HWACCEL_CAP_ASYNC_SAFE,
 };
diff --git a/libavcodec/vaapi_hevc.c b/libavcodec/vaapi_hevc.c
index c69d63d8ec..41e973626c 100644
--- a/libavcodec/vaapi_hevc.c
+++ b/libavcodec/vaapi_hevc.c
@@ -421,6 +421,15 @@ static int vaapi_hevc_decode_slice(AVCodecContext *avctx,
     return 0;
 }
 
+static int ff_vaapi_hevc_frame_params(AVCodecContext *avctx,
+                                   AVBufferRef *hw_frames_ctx)
+{
+    const HEVCContext *s = avctx->priv_data;
+    const HEVCSPS   *sps = s->ps.sps;
+    return ff_vaapi_frame_params_with_dpb_size(avctx, hw_frames_ctx,
+                                               sps->temporal_layer[sps->max_sub_layers - 1].max_dec_pic_buffering);
+}
+
 const AVHWAccel ff_hevc_vaapi_hwaccel = {
     .name                 = "hevc_vaapi",
     .type                 = AVMEDIA_TYPE_VIDEO,
@@ -432,7 +441,7 @@ const AVHWAccel ff_hevc_vaapi_hwaccel = {
     .frame_priv_data_size = sizeof(VAAPIDecodePictureHEVC),
     .init                 = ff_vaapi_decode_init,
     .uninit               = ff_vaapi_decode_uninit,
-    .frame_params         = ff_vaapi_common_frame_params,
+    .frame_params         = ff_vaapi_hevc_frame_params,
     .priv_data_size       = sizeof(VAAPIDecodeContext),
     .caps_internal        = HWACCEL_CAP_ASYNC_SAFE,
 };
diff --git a/libavcodec/vaapi_vp8.c b/libavcodec/vaapi_vp8.c
index 2426b30f13..4b74b9e0ad 100644
--- a/libavcodec/vaapi_vp8.c
+++ b/libavcodec/vaapi_vp8.c
@@ -220,6 +220,12 @@ fail:
     return err;
 }
 
+static int ff_vaapi_vp8_frame_params(AVCodecContext *avctx,
+                                  AVBufferRef *hw_frames_ctx)
+{
+    return ff_vaapi_frame_params_with_dpb_size(avctx, hw_frames_ctx, 3);
+}
+
 const AVHWAccel ff_vp8_vaapi_hwaccel = {
     .name                 = "vp8_vaapi",
     .type                 = AVMEDIA_TYPE_VIDEO,
@@ -231,7 +237,7 @@ const AVHWAccel ff_vp8_vaapi_hwaccel = {
     .frame_priv_data_size = sizeof(VAAPIDecodePicture),
     .init                 = &ff_vaapi_decode_init,
     .uninit               = &ff_vaapi_decode_uninit,
-    .frame_params         = &ff_vaapi_common_frame_params,
+    .frame_params         = &ff_vaapi_vp8_frame_params,
     .priv_data_size       = sizeof(VAAPIDecodeContext),
     .caps_internal        = HWACCEL_CAP_ASYNC_SAFE,
 };
diff --git a/libavcodec/vaapi_vp9.c b/libavcodec/vaapi_vp9.c
index f384ba7873..ce15701405 100644
--- a/libavcodec/vaapi_vp9.c
+++ b/libavcodec/vaapi_vp9.c
@@ -168,6 +168,12 @@ static int vaapi_vp9_decode_slice(AVCodecContext *avctx,
     return 0;
 }
 
+static int ff_vaapi_vp9_frame_params(AVCodecContext *avctx,
+                                  AVBufferRef *hw_frames_ctx)
+{
+    return ff_vaapi_frame_params_with_dpb_size(avctx, hw_frames_ctx, 8);
+}
+
 const AVHWAccel ff_vp9_vaapi_hwaccel = {
     .name                 = "vp9_vaapi",
     .type                 = AVMEDIA_TYPE_VIDEO,
@@ -179,7 +185,7 @@ const AVHWAccel ff_vp9_vaapi_hwaccel = {
     .frame_priv_data_size = sizeof(VAAPIDecodePicture),
     .init                 = ff_vaapi_decode_init,
     .uninit               = ff_vaapi_decode_uninit,
-    .frame_params         = ff_vaapi_common_frame_params,
+    .frame_params         = ff_vaapi_vp9_frame_params,
     .priv_data_size       = sizeof(VAAPIDecodeContext),
     .caps_internal        = HWACCEL_CAP_ASYNC_SAFE,
 };
