diff --git a/libavcodec/vaapi_encode_h265.c b/libavcodec/vaapi_encode_h265.c
index 3ae92a7a09..85350380ca 100644
--- a/libavcodec/vaapi_encode_h265.c
+++ b/libavcodec/vaapi_encode_h265.c
@@ -34,6 +34,8 @@
 #include "vaapi_encode.h"
 
 
+static const char *picture_type_name[] = { "IDR", "I", "P", "B" };
+
 typedef struct VAAPIEncodeH265Context {
     unsigned int ctu_width;
     unsigned int ctu_height;
@@ -58,6 +60,11 @@ typedef struct VAAPIEncodeH265Context {
     CodedBitstreamContext *cbc;
     CodedBitstreamFragment current_access_unit;
     int aud_needed;
+
+    // reference frames param
+    struct VAAPIEncodePicture *references[MAX_PICTURE_REFERENCES];
+    int ref_nr;
+    int max_ref_nr;
 } VAAPIEncodeH265Context;
 
 typedef struct VAAPIEncodeH265Options {
@@ -224,7 +231,7 @@ static int vaapi_encode_h265_init_sequence_params(AVCodecContext *avctx)
     vps->profile_tier_level.general_profile_compatibility_flag[avctx->profile & 31] = 1;
 
     vps->vps_sub_layer_ordering_info_present_flag = 0;
-    vps->vps_max_dec_pic_buffering_minus1[0]      = (ctx->b_per_p > 0) + 1;
+    vps->vps_max_dec_pic_buffering_minus1[0]      = priv->max_ref_nr;
     vps->vps_max_num_reorder_pics[0]              = (ctx->b_per_p > 0);
     vps->vps_max_latency_increase_plus1[0]        = 0;
 
@@ -677,59 +684,44 @@ static int vaapi_encode_h265_init_slice_params(AVCodecContext *avctx,
 
     if (pic->type != PICTURE_TYPE_IDR) {
         H265RawSTRefPicSet *rps;
-        VAAPIEncodePicture *st;
-        int used;
 
         sh->short_term_ref_pic_set_sps_flag = 0;
 
         rps = &sh->short_term_ref_pic_set;
         memset(rps, 0, sizeof(*rps));
 
-        for (st = ctx->pic_start; st; st = st->next) {
-            if (st->encode_order >= pic->encode_order) {
-                // Not yet in DPB.
-                continue;
+        if (pic->type != PICTURE_TYPE_B) {
+            rps->num_negative_pics = pic->nb_refs;
+            for (i = 0 ; i < rps->num_negative_pics; i++) {
+                rps->used_by_curr_pic_s0_flag[i] = 1;
+                if (i == 0) {
+                    rps->delta_poc_s0_minus1[i] =  priv->pic_order_cnt -
+			               vpic->reference_frames[pic->nb_refs - 1 - i].pic_order_cnt - 1;
+                } else {
+                    rps->delta_poc_s0_minus1[i] = vpic->reference_frames[pic->nb_refs - i].pic_order_cnt -
+                                        vpic->reference_frames[pic->nb_refs - 1 - i].pic_order_cnt - 1;
+                }
             }
-            used = 0;
-            for (i = 0; i < pic->nb_refs; i++) {
-                if (pic->refs[i] == st)
-                    used = 1;
+        } else {
+            rps->num_positive_pics = 1;
+            for (i = 0 ; i < rps->num_positive_pics; i++) {
+                rps->used_by_curr_pic_s1_flag[i] = 1;
+                rps->delta_poc_s1_minus1[i] =
+                vpic->reference_frames[pic->nb_refs- 1 - i].pic_order_cnt - priv->pic_order_cnt - 1;
             }
-            if (!used) {
-                // Usually each picture always uses all of the others in the
-                // DPB as references.  The one case we have to treat here is
-                // a non-IDR IRAP picture, which may need to hold unused
-                // references across itself to be used for the decoding of
-                // following RASL pictures.  This looks for such an RASL
-                // picture, and keeps the reference if there is one.
-                VAAPIEncodePicture *rp;
-                for (rp = ctx->pic_start; rp; rp = rp->next) {
-                    if (rp->encode_order < pic->encode_order)
-                        continue;
-                    if (rp->type != PICTURE_TYPE_B)
-                        continue;
-                    if (rp->refs[0] == st && rp->refs[1] == pic)
-                        break;
+            rps->num_negative_pics = pic->nb_refs - 1 ;
+            for (i = 0 ; i < rps->num_negative_pics; i++) {
+                rps->used_by_curr_pic_s0_flag[i] = 1;
+                if (i == 0) {
+                    rps->delta_poc_s0_minus1[i] =
+                        priv->pic_order_cnt - vpic->reference_frames[pic->nb_refs - 2 - i].pic_order_cnt - 1;
+                } else {
+                    rps->delta_poc_s0_minus1[i] =
+                         vpic->reference_frames[pic->nb_refs - 1 - i].pic_order_cnt -
+                         vpic->reference_frames[pic->nb_refs - 2 - i].pic_order_cnt - 1;
                 }
-                if (!rp)
-                    continue;
-            }
-            // This only works for one instance of each (delta_poc_sN_minus1
-            // is relative to the previous frame in the list, not relative to
-            // the current frame directly).
-            if (st->display_order < pic->display_order) {
-                rps->delta_poc_s0_minus1[rps->num_negative_pics] =
-                    pic->display_order - st->display_order - 1;
-                rps->used_by_curr_pic_s0_flag[rps->num_negative_pics] = used;
-                ++rps->num_negative_pics;
-            } else {
-                rps->delta_poc_s1_minus1[rps->num_positive_pics] =
-                    st->display_order - pic->display_order - 1;
-                rps->used_by_curr_pic_s1_flag[rps->num_positive_pics] = used;
-                ++rps->num_positive_pics;
             }
         }
-
         sh->num_long_term_sps  = 0;
         sh->num_long_term_pics = 0;
 
@@ -740,9 +732,13 @@ static int vaapi_encode_h265_init_slice_params(AVCodecContext *avctx,
             sh->collocated_ref_idx      = 0;
         }
 
-        sh->num_ref_idx_active_override_flag = 0;
-        sh->num_ref_idx_l0_active_minus1 = pps->num_ref_idx_l0_default_active_minus1;
-        sh->num_ref_idx_l1_active_minus1 = pps->num_ref_idx_l1_default_active_minus1;
+        sh->num_ref_idx_active_override_flag = 1;
+        if (pic->type == PICTURE_TYPE_P) {
+            sh->num_ref_idx_l0_active_minus1 = pic->nb_refs - 1;
+        } else if (pic->type == PICTURE_TYPE_B) {
+            sh->num_ref_idx_l0_active_minus1 = pic->nb_refs - 2;
+            sh->num_ref_idx_l1_active_minus1 = 0;
+        }
     }
 
     sh->slice_sao_luma_flag = sh->slice_sao_chroma_flag =
@@ -765,8 +761,6 @@ static int vaapi_encode_h265_init_slice_params(AVCodecContext *avctx,
 
         .num_ref_idx_l0_active_minus1 = sh->num_ref_idx_l0_active_minus1,
         .num_ref_idx_l1_active_minus1 = sh->num_ref_idx_l1_active_minus1,
-        .ref_pic_list0[0]             = vpic->reference_frames[0],
-        .ref_pic_list1[0]             = vpic->reference_frames[1],
 
         .luma_log2_weight_denom         = sh->luma_log2_weight_denom,
         .delta_chroma_log2_weight_denom = sh->delta_chroma_log2_weight_denom,
@@ -800,7 +794,16 @@ static int vaapi_encode_h265_init_slice_params(AVCodecContext *avctx,
         },
     };
 
-
+    if (pic->type == PICTURE_TYPE_P) {
+        for (i = 0; i < pic->nb_refs; i++) {
+            vslice->ref_pic_list0[i] = vpic->reference_frames[pic->nb_refs - 1 - i];
+        }
+    }
+    if (pic->type == PICTURE_TYPE_B) {
+        for (i = 0; i < pic->nb_refs - 1; i++)
+            vslice->ref_pic_list0[i] = vpic->reference_frames[pic->nb_refs - 2 - i];
+        vslice->ref_pic_list1[0] = vpic->reference_frames[pic->nb_refs - 1];
+    }
     return 0;
 }
 
@@ -854,6 +857,34 @@ static av_cold int vaapi_encode_h265_configure(AVCodecContext *avctx)
         av_assert0(0 && "Invalid RC mode.");
     }
 
+    priv->max_ref_nr = avctx->refs;
+
+    if (priv->max_ref_nr > ctx->max_ref_l0 + ctx->max_ref_l1) {
+        av_log(avctx, AV_LOG_WARNING, "Warning: " \
+               "reference frame number exceeds %d" \
+               "correct to %d\n", \
+               ctx->max_ref_l0 + ctx->max_ref_l1,
+               ctx->max_ref_l0 + ctx->max_ref_l1);
+        priv->max_ref_nr = ctx->max_ref_l0 + ctx->max_ref_l1;
+    }
+    if (avctx->max_b_frames && priv->max_ref_nr < 2) {
+        av_log(avctx, AV_LOG_WARNING, "Warning: " \
+               "reference frame number is 1 but b frame encoding is setted," \
+               "correct to 2\n");
+        priv->max_ref_nr = 2;
+    }
+    if (!avctx->max_b_frames && priv->max_ref_nr > ctx->max_ref_l0) {
+        av_log(avctx, AV_LOG_WARNING, "Warning: " \
+               "no b frame, but ref_nr > max_ref_l0" \
+               "correct to %d\n", ctx->max_ref_l0);
+        priv->max_ref_nr = ctx->max_ref_l0;
+    }
+    if (priv->max_ref_nr < 1 && avctx->gop_size) {
+        av_log(avctx, AV_LOG_WARNING, "Warning: " \
+               "reference frame number is %d but gop_size > 0," \
+               "correct to 1\n", priv->max_ref_nr);
+        priv->max_ref_nr = 1;
+    }
     return 0;
 }
 
@@ -936,6 +967,374 @@ static av_cold int vaapi_encode_h265_close(AVCodecContext *avctx)
     return ff_vaapi_encode_close(avctx);
 }
 
+static void vaapi_encode_h265_add_reference (AVCodecContext *avctx,
+                                             VAAPIEncodePicture *pic)
+{
+    VAAPIEncodeContext *ctx = avctx->priv_data;
+    VAAPIEncodeH265Context *priv = ctx->priv_data;
+    int i;
+
+    av_assert0 (pic->type != PICTURE_TYPE_B);
+
+    if (pic->type == PICTURE_TYPE_IDR) {
+        // clear the reference frame list
+        for (i = 0 ; i < priv->ref_nr; i++) {
+           priv->references[i]->ref_count --;
+           priv->references[i] = NULL;
+        }
+        priv->ref_nr = 0;
+    }
+
+    if (priv->ref_nr == priv->max_ref_nr) {
+        // remove the oldest reference frame
+        for (i = 0 ; i < priv->ref_nr - 1; i++) {
+           priv->references[i]->ref_count --;
+           priv->references[i] = priv->references[i+1];
+           priv->references[i]->ref_count ++;
+        }
+        priv->references[priv->ref_nr-1]->ref_count--;
+        priv->ref_nr --;
+    }
+
+    priv->references[priv->ref_nr] = pic;
+    priv->references[priv->ref_nr]->ref_count ++;
+    priv->ref_nr++;
+}
+
+static int vaapi_encode_h265_get_next(AVCodecContext *avctx,
+                                      VAAPIEncodePicture **pic_out)
+{
+    VAAPIEncodeContext *ctx = avctx->priv_data;
+    VAAPIEncodeH265Context *priv = ctx->priv_data;
+    VAAPIEncodePicture *start, *end, *pic;
+    int i,j;
+
+    for (pic = ctx->pic_start; pic; pic = pic->next) {
+        if (pic->next)
+            av_assert0(pic->display_order < pic->next->display_order);
+        if (pic->display_order == ctx->input_order) {
+            *pic_out = pic;
+            return 0;
+        }
+    }
+
+    pic = vaapi_encode_alloc();
+    if (!pic)
+        return AVERROR(ENOMEM);
+
+    if (ctx->input_order == 0 || ctx->force_idr ||
+        ctx->gop_counter >= avctx->gop_size) {
+        pic->type = PICTURE_TYPE_IDR;
+        ctx->force_idr = 0;
+        ctx->gop_counter = 1;
+        ctx->p_counter = 0;
+    } else if (ctx->p_counter >= ctx->p_per_i) {
+        pic->type = PICTURE_TYPE_I;
+        ++ctx->gop_counter;
+        ctx->p_counter = 0;
+    } else {
+        pic->type = PICTURE_TYPE_P;
+        for (i = 0 ; i < priv->ref_nr; i++) {
+            pic->refs[i] = priv->references[i];
+            pic->refs[i]->ref_count++;
+        }
+        pic->nb_refs = priv->ref_nr;
+        ++ctx->gop_counter;
+        ++ctx->p_counter;
+    }
+    start = end = pic;
+    vaapi_encode_h265_add_reference(avctx, pic);
+
+    if (pic->type != PICTURE_TYPE_IDR) {
+        // If that was not an IDR frame, add B-frames display-before and
+        // encode-after it, but not exceeding the GOP size.
+
+        for (i = 0; i < ctx->b_per_p &&
+            ctx->gop_counter < avctx->gop_size; i++) {
+            pic = vaapi_encode_alloc();
+            if (!pic)
+                goto fail;
+
+            pic->type = PICTURE_TYPE_B;
+            for (j = 0 ; j < priv->ref_nr; j++) {
+                pic->refs[j] = priv->references[j];
+                pic->refs[j]->ref_count++;
+            }
+            pic->nb_refs = priv->ref_nr;
+            pic->next = start;
+            pic->display_order = ctx->input_order + ctx->b_per_p - i - 1;
+            pic->encode_order  = pic->display_order + 1;
+            start = pic;
+
+            ++ctx->gop_counter;
+        }
+    }
+
+    if (ctx->input_order == 0) {
+        pic->display_order = 0;
+        pic->encode_order  = 0;
+
+        ctx->pic_start = ctx->pic_end = pic;
+
+    } else {
+        for (i = 0, pic = start; pic; i++, pic = pic->next) {
+            pic->display_order = ctx->input_order + i;
+            if (end->type == PICTURE_TYPE_IDR)
+                pic->encode_order = ctx->input_order + i;
+            else if (pic == end)
+                pic->encode_order = ctx->input_order;
+            else
+                pic->encode_order = ctx->input_order + i + 1;
+        }
+
+        av_assert0(ctx->pic_end);
+        ctx->pic_end->next = start;
+        ctx->pic_end = end;
+    }
+    *pic_out = start;
+
+    av_log(avctx, AV_LOG_DEBUG, "Pictures:");
+    for (pic = ctx->pic_start; pic; pic = pic->next) {
+        av_log(avctx, AV_LOG_DEBUG, " %s (%"PRId64"/%"PRId64")",
+               picture_type_name[pic->type],
+               pic->display_order, pic->encode_order);
+    }
+    av_log(avctx, AV_LOG_DEBUG, "\n");
+
+    return 0;
+
+fail:
+    while (start) {
+        pic = start->next;
+        vaapi_encode_free(avctx, start);
+        start = pic;
+    }
+    return AVERROR(ENOMEM);
+}
+
+static int vaapi_encode_h265_clear_old(AVCodecContext *avctx)
+{
+    VAAPIEncodeContext *ctx = avctx->priv_data;
+    VAAPIEncodePicture *pic, *next;
+    pic = ctx->pic_start;
+    while (pic && pic->next) {
+        if (pic->encode_order > ctx->output_order)
+            break;
+
+        if (pic->ref_count == 0 && pic == ctx->pic_start) {
+            ctx->pic_start = pic->next;
+            vaapi_encode_free(avctx, pic);
+            pic = ctx->pic_start;
+            continue;
+        }
+        next = pic->next;
+
+        if (next->encode_order > ctx->output_order)
+            break;
+        if (next->ref_count == 0) {
+            pic->next = next->next;
+            vaapi_encode_free(avctx, next);
+        }
+        pic = pic->next;
+    }
+    return 0;
+}
+
+static int vaapi_encode_h265_truncate_gop(AVCodecContext *avctx)
+{
+    VAAPIEncodeContext *ctx = avctx->priv_data;
+    VAAPIEncodeH265Context *priv = ctx->priv_data;
+    VAAPIEncodePicture *pic, *last_pic, *next;
+
+    // Find the last picture we actually have input for.
+    for (pic = ctx->pic_start; pic; pic = pic->next) {
+        if (!pic->input_available)
+            break;
+        last_pic = pic;
+    }
+
+    if (pic) {
+        av_assert0(last_pic);
+
+        if (last_pic->type == PICTURE_TYPE_B) {
+            // Some fixing up is required.  Change the type of this
+            // picture to P, then modify preceding B references which
+            // point beyond it to point at it instead.
+            int last_ref = last_pic->nb_refs - 1;
+
+            last_pic->type = PICTURE_TYPE_P;
+            last_pic->encode_order = last_pic->refs[last_ref]->encode_order;
+
+            for (pic = ctx->pic_start; pic != last_pic; pic = pic->next) {
+                 if (pic->type == PICTURE_TYPE_B &&
+                        pic->refs[last_ref] == last_pic->refs[last_ref]) {
+                        if (last_pic->refs[last_ref])
+                           pic->refs[last_ref]->ref_count --;
+                        pic->refs[last_ref] = last_pic;
+                        pic->refs[last_ref]->ref_count ++;
+                 }
+            }
+
+            last_pic->nb_refs = last_pic->refs[last_ref] ?  last_pic->nb_refs - 1 :  last_pic->nb_refs;
+
+            if (last_pic->refs[last_ref])
+                last_pic->refs[last_ref]->ref_count--;
+            last_pic->refs[last_ref] = NULL;
+        } else {
+            // We can use the current structure (no references point
+            // beyond the end), but there are unused pics to discard.
+        }
+
+        // Discard all following pics, they will never be used.
+        for (pic = last_pic->next; pic; pic = next) {
+            int i;
+            int ref_nr = priv->ref_nr;
+            next = pic->next;
+
+            for (i = 0; i < pic->nb_refs; i++) {
+                pic->refs[i]->ref_count--;
+            }
+            for (i = 0 ; i < ref_nr; i++) {
+                if (priv->references[i] == pic) {
+                    priv->references[i]->ref_count--;
+                    priv->references[i] = NULL;
+                    priv->ref_nr --;
+                }
+            }
+            vaapi_encode_free(avctx, pic);
+        }
+
+        last_pic->next = NULL;
+        ctx->pic_end = last_pic;
+
+    } else {
+        // Input is available for all pictures, so we don't need to
+        // mangle anything.
+    }
+
+    av_log(avctx, AV_LOG_DEBUG, "Pictures ending truncated GOP:");
+    for (pic = ctx->pic_start; pic; pic = pic->next) {
+        av_log(avctx, AV_LOG_DEBUG, " %s (%"PRId64"/%"PRId64")",
+               picture_type_name[pic->type],
+               pic->display_order, pic->encode_order);
+    }
+    av_log(avctx, AV_LOG_DEBUG, "\n");
+
+    return 0;
+}
+
+static av_cold int vaapi_encode_h265_encode(AVCodecContext *avctx,
+                                            AVPacket *pkt,
+                                            const AVFrame *input_image,
+                                            int *got_packet)
+{
+    VAAPIEncodeContext *ctx = avctx->priv_data;
+    VAAPIEncodePicture *pic;
+    int err;
+
+    if (input_image) {
+        av_log(avctx, AV_LOG_DEBUG, "Encode frame: %ux%u (%"PRId64").\n",
+               input_image->width, input_image->height, input_image->pts);
+
+        if (input_image->pict_type == AV_PICTURE_TYPE_I) {
+            err = vaapi_encode_h265_truncate_gop(avctx);
+            if (err < 0)
+                goto fail;
+            ctx->force_idr = 1;
+        }
+
+        err = vaapi_encode_h265_get_next(avctx, &pic);
+        if (err) {
+            av_log(avctx, AV_LOG_ERROR, "Input setup failed: %d.\n", err);
+            return err;
+        }
+
+        pic->input_image = av_frame_alloc();
+        if (!pic->input_image) {
+            err = AVERROR(ENOMEM);
+            goto fail;
+        }
+        err = av_frame_ref(pic->input_image, input_image);
+        if (err < 0)
+            goto fail;
+        pic->input_surface = (VASurfaceID)(uintptr_t)input_image->data[3];
+        pic->pts = input_image->pts;
+
+        if (ctx->input_order == 0)
+            ctx->first_pts = pic->pts;
+        if (ctx->input_order == ctx->decode_delay)
+            ctx->dts_pts_diff = pic->pts - ctx->first_pts;
+        if (ctx->output_delay > 0)
+            ctx->ts_ring[ctx->input_order % (3 * ctx->output_delay)] = pic->pts;
+
+        pic->input_available = 1;
+
+    } else {
+        if (!ctx->end_of_stream) {
+            err = vaapi_encode_h265_truncate_gop(avctx);
+            if (err < 0)
+                goto fail;
+            ctx->end_of_stream = 1;
+        }
+    }
+
+    ++ctx->input_order;
+    ++ctx->output_order;
+    av_assert0(ctx->output_order + ctx->output_delay + 1 == ctx->input_order);
+
+    for (pic = ctx->pic_start; pic; pic = pic->next)
+        if (pic->encode_order == ctx->output_order)
+            break;
+
+    // pic can be null here if we don't have a specific target in this
+    // iteration.  We might still issue encodes if things can be overlapped,
+    // even though we don't intend to output anything.
+
+    err = vaapi_encode_step(avctx, pic);
+    if (err < 0) {
+        av_log(avctx, AV_LOG_ERROR, "Encode failed: %d.\n", err);
+        goto fail;
+    }
+
+    if (!pic) {
+        *got_packet = 0;
+    } else {
+        err = vaapi_encode_output(avctx, pic, pkt);
+        if (err < 0) {
+            av_log(avctx, AV_LOG_ERROR, "Output failed: %d.\n", err);
+            goto fail;
+        }
+
+        if (ctx->output_delay == 0) {
+            pkt->dts = pkt->pts;
+        } else if (ctx->output_order < ctx->decode_delay) {
+            if (ctx->ts_ring[ctx->output_order] < INT64_MIN + ctx->dts_pts_diff)
+                pkt->dts = INT64_MIN;
+            else
+                pkt->dts = ctx->ts_ring[ctx->output_order] - ctx->dts_pts_diff;
+        } else {
+            pkt->dts = ctx->ts_ring[(ctx->output_order - ctx->decode_delay) %
+                                    (3 * ctx->output_delay)];
+        }
+
+        *got_packet = 1;
+    }
+
+    err = vaapi_encode_h265_clear_old(avctx);
+    if (err < 0) {
+        av_log(avctx, AV_LOG_ERROR, "List clearing failed: %d.\n", err);
+        goto fail;
+    }
+
+    return 0;
+
+fail:
+    // Unclear what to clean up on failure.  There are probably some things we
+    // could do usefully clean up here, but for now just leave them for uninit()
+    // to do instead.
+    return err;
+}
+
 #define OFFSET(x) (offsetof(VAAPIEncodeContext, codec_options_data) + \
                    offsetof(VAAPIEncodeH265Options, x))
 #define FLAGS (AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM)
@@ -977,7 +1376,7 @@ AVCodec ff_hevc_vaapi_encoder = {
     .priv_data_size = (sizeof(VAAPIEncodeContext) +
                        sizeof(VAAPIEncodeH265Options)),
     .init           = &vaapi_encode_h265_init,
-    .encode2        = &ff_vaapi_encode2,
+    .encode2        = &vaapi_encode_h265_encode,
     .close          = &vaapi_encode_h265_close,
     .priv_class     = &vaapi_encode_h265_class,
     .capabilities   = AV_CODEC_CAP_DELAY,
