diff --git a/libavcodec/Makefile b/libavcodec/Makefile
index 2443d2c6fd..998f6b7e12 100644
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -165,7 +165,7 @@ OBJS-$(CONFIG_STARTCODE)               += startcode.o
 OBJS-$(CONFIG_TEXTUREDSP)              += texturedsp.o
 OBJS-$(CONFIG_TEXTUREDSPENC)           += texturedspenc.o
 OBJS-$(CONFIG_TPELDSP)                 += tpeldsp.o
-OBJS-$(CONFIG_VAAPI_ENCODE)            += vaapi_encode.o
+OBJS-$(CONFIG_VAAPI_ENCODE)            += vaapi_encode.o hw_base_encode.o
 OBJS-$(CONFIG_AV1_AMF_ENCODER)         += amfenc_av1.o
 OBJS-$(CONFIG_VC1DSP)                  += vc1dsp.o
 OBJS-$(CONFIG_VIDEODSP)                += videodsp.o
diff --git a/libavcodec/hw_base_encode.c b/libavcodec/hw_base_encode.c
new file mode 100644
index 0000000000..16afaa37be
--- /dev/null
+++ b/libavcodec/hw_base_encode.c
@@ -0,0 +1,594 @@
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "libavutil/avassert.h"
+#include "libavutil/common.h"
+#include "libavutil/internal.h"
+#include "libavutil/log.h"
+#include "libavutil/mem.h"
+#include "libavutil/pixdesc.h"
+
+#include "encode.h"
+#include "avcodec.h"
+#include "hw_base_encode.h"
+
+static void hw_base_encode_add_ref(FFHWBaseEncodePicture *pic,
+                                   FFHWBaseEncodePicture *target,
+                                   int is_ref, int in_dpb, int prev)
+{
+    int refs = 0;
+
+    if (is_ref) {
+        av_assert0(pic != target);
+        av_assert0(pic->nb_refs[0] < MAX_PICTURE_REFERENCES &&
+                   pic->nb_refs[1] < MAX_PICTURE_REFERENCES);
+        if (target->display_order < pic->display_order)
+            pic->refs[0][pic->nb_refs[0]++] = target;
+        else
+            pic->refs[1][pic->nb_refs[1]++] = target;
+        ++refs;
+    }
+
+    if (in_dpb) {
+        av_assert0(pic->nb_dpb_pics < MAX_DPB_SIZE);
+        pic->dpb[pic->nb_dpb_pics++] = target;
+        ++refs;
+    }
+
+    if (prev) {
+        av_assert0(!pic->prev);
+        pic->prev = target;
+        ++refs;
+    }
+
+    target->ref_count[0] += refs;
+    target->ref_count[1] += refs;
+}
+
+static void hw_base_encode_remove_refs(FFHWBaseEncodePicture *pic, int level)
+{
+    int i;
+
+    if (pic->ref_removed[level])
+        return;
+
+    for (i = 0; i < pic->nb_refs[0]; i++) {
+        av_assert0(pic->refs[0][i]);
+        --pic->refs[0][i]->ref_count[level];
+        av_assert0(pic->refs[0][i]->ref_count[level] >= 0);
+    }
+
+    for (i = 0; i < pic->nb_refs[1]; i++) {
+        av_assert0(pic->refs[1][i]);
+        --pic->refs[1][i]->ref_count[level];
+        av_assert0(pic->refs[1][i]->ref_count[level] >= 0);
+    }
+
+    for (i = 0; i < pic->nb_dpb_pics; i++) {
+        av_assert0(pic->dpb[i]);
+        --pic->dpb[i]->ref_count[level];
+        av_assert0(pic->dpb[i]->ref_count[level] >= 0);
+    }
+
+    av_assert0(pic->prev || pic->type == FF_HW_PICTURE_TYPE_IDR);
+    if (pic->prev) {
+        --pic->prev->ref_count[level];
+        av_assert0(pic->prev->ref_count[level] >= 0);
+    }
+
+    pic->ref_removed[level] = 1;
+}
+
+static void hw_base_encode_set_b_pictures(AVCodecContext *avctx,
+                                          FFHWBaseEncodePicture *start,
+                                          FFHWBaseEncodePicture *end,
+                                          FFHWBaseEncodePicture *prev,
+                                          int current_depth,
+                                          FFHWBaseEncodePicture **last)
+{
+    FFHWBaseEncodeContext *ctx = avctx->priv_data;
+    FFHWBaseEncodePicture *pic, *next, *ref;
+    int i, len;
+
+    av_assert0(start && end && start != end && start->next != end);
+
+    // If we are at the maximum depth then encode all pictures as
+    // non-referenced B-pictures.  Also do this if there is exactly one
+    // picture left, since there will be nothing to reference it.
+    if (current_depth == ctx->max_b_depth || start->next->next == end) {
+        for (pic = start->next; pic; pic = pic->next) {
+            if (pic == end)
+                break;
+            pic->type    = FF_HW_PICTURE_TYPE_B;
+            pic->b_depth = current_depth;
+
+            hw_base_encode_add_ref(pic, start, 1, 1, 0);
+            hw_base_encode_add_ref(pic, end,   1, 1, 0);
+            hw_base_encode_add_ref(pic, prev,  0, 0, 1);
+
+            for (ref = end->refs[1][0]; ref; ref = ref->refs[1][0])
+                hw_base_encode_add_ref(pic, ref, 0, 1, 0);
+        }
+        *last = prev;
+
+    } else {
+        // Split the current list at the midpoint with a referenced
+        // B-picture, then descend into each side separately.
+        len = 0;
+        for (pic = start->next; pic != end; pic = pic->next)
+            ++len;
+        for (pic = start->next, i = 1; 2 * i < len; pic = pic->next, i++);
+
+        pic->type    = FF_HW_PICTURE_TYPE_B;
+        pic->b_depth = current_depth;
+
+        pic->is_reference = 1;
+
+        hw_base_encode_add_ref(pic, pic,   0, 1, 0);
+        hw_base_encode_add_ref(pic, start, 1, 1, 0);
+        hw_base_encode_add_ref(pic, end,   1, 1, 0);
+        hw_base_encode_add_ref(pic, prev,  0, 0, 1);
+
+        for (ref = end->refs[1][0]; ref; ref = ref->refs[1][0])
+            hw_base_encode_add_ref(pic, ref, 0, 1, 0);
+
+        if (i > 1)
+            hw_base_encode_set_b_pictures(avctx, start, pic, pic,
+                                          current_depth + 1, &next);
+        else
+            next = pic;
+
+        hw_base_encode_set_b_pictures(avctx, pic, end, next,
+                                      current_depth + 1, last);
+    }
+}
+
+static void hw_base_encode_add_next_prev(AVCodecContext *avctx,
+                                         FFHWBaseEncodePicture *pic)
+{
+    FFHWBaseEncodeContext *ctx = avctx->priv_data;
+    int i;
+
+    if (!pic)
+        return;
+
+    if (pic->type == FF_HW_PICTURE_TYPE_IDR) {
+        for (i = 0; i < ctx->nb_next_prev; i++) {
+            --ctx->next_prev[i]->ref_count[0];
+            ctx->next_prev[i] = NULL;
+        }
+        ctx->next_prev[0] = pic;
+        ++pic->ref_count[0];
+        ctx->nb_next_prev = 1;
+
+        return;
+    }
+
+    if (ctx->nb_next_prev < MAX_PICTURE_REFERENCES) {
+        ctx->next_prev[ctx->nb_next_prev++] = pic;
+        ++pic->ref_count[0];
+    } else {
+        --ctx->next_prev[0]->ref_count[0];
+        for (i = 0; i < MAX_PICTURE_REFERENCES - 1; i++)
+            ctx->next_prev[i] = ctx->next_prev[i + 1];
+        ctx->next_prev[i] = pic;
+        ++pic->ref_count[0];
+    }
+}
+
+static int hw_base_encode_pick_next(AVCodecContext *avctx,
+                                    FFHWBaseEncodePicture **pic_out)
+{
+    FFHWBaseEncodeContext *ctx = avctx->priv_data;
+    FFHWBaseEncodePicture *pic = NULL, *prev = NULL, *next, *start;
+    int i, b_counter, closed_gop_end;
+
+    // If there are any B-frames already queued, the next one to encode
+    // is the earliest not-yet-issued frame for which all references are
+    // available.
+    for (pic = ctx->pic_start; pic; pic = pic->next) {
+        if (pic->encode_issued)
+            continue;
+        if (pic->type != FF_HW_PICTURE_TYPE_B)
+            continue;
+        for (i = 0; i < pic->nb_refs[0]; i++) {
+            if (!pic->refs[0][i]->encode_issued)
+                break;
+        }
+        if (i != pic->nb_refs[0])
+            continue;
+
+        for (i = 0; i < pic->nb_refs[1]; i++) {
+            if (!pic->refs[1][i]->encode_issued)
+                break;
+        }
+        if (i == pic->nb_refs[1])
+            break;
+    }
+
+    if (pic) {
+        av_log(avctx, AV_LOG_DEBUG, "Pick B-picture at depth %d to "
+               "encode next.\n", pic->b_depth);
+        *pic_out = pic;
+        return 0;
+    }
+
+    // Find the B-per-Pth available picture to become the next picture
+    // on the top layer.
+    start = NULL;
+    b_counter = 0;
+    closed_gop_end = ctx->closed_gop ||
+                     ctx->idr_counter == ctx->gop_per_idr;
+    for (pic = ctx->pic_start; pic; pic = next) {
+        next = pic->next;
+        if (pic->encode_issued) {
+            start = pic;
+            continue;
+        }
+        // If the next available picture is force-IDR, encode it to start
+        // a new GOP immediately.
+        if (pic->force_idr)
+            break;
+        if (b_counter == ctx->b_per_p)
+            break;
+        // If this picture ends a closed GOP or starts a new GOP then it
+        // needs to be in the top layer.
+        if (ctx->gop_counter + b_counter + closed_gop_end >= ctx->gop_size)
+            break;
+        // If the picture after this one is force-IDR, we need to encode
+        // this one in the top layer.
+        if (next && next->force_idr)
+            break;
+        ++b_counter;
+    }
+
+    // At the end of the stream the last picture must be in the top layer.
+    if (!pic && ctx->end_of_stream) {
+        --b_counter;
+        pic = ctx->pic_end;
+        if (pic->encode_complete)
+            return AVERROR_EOF;
+        else if (pic->encode_issued)
+            return AVERROR(EAGAIN);
+    }
+
+    if (!pic) {
+        av_log(avctx, AV_LOG_DEBUG, "Pick nothing to encode next - "
+               "need more input for reference pictures.\n");
+        return AVERROR(EAGAIN);
+    }
+    if (ctx->input_order <= ctx->decode_delay && !ctx->end_of_stream) {
+        av_log(avctx, AV_LOG_DEBUG, "Pick nothing to encode next - "
+               "need more input for timestamps.\n");
+        return AVERROR(EAGAIN);
+    }
+
+    if (pic->force_idr) {
+        av_log(avctx, AV_LOG_DEBUG, "Pick forced IDR-picture to "
+               "encode next.\n");
+        pic->type = FF_HW_PICTURE_TYPE_IDR;
+        ctx->idr_counter = 1;
+        ctx->gop_counter = 1;
+
+    } else if (ctx->gop_counter + b_counter >= ctx->gop_size) {
+        if (ctx->idr_counter == ctx->gop_per_idr) {
+            av_log(avctx, AV_LOG_DEBUG, "Pick new-GOP IDR-picture to "
+                   "encode next.\n");
+            pic->type = FF_HW_PICTURE_TYPE_IDR;
+            ctx->idr_counter = 1;
+        } else {
+            av_log(avctx, AV_LOG_DEBUG, "Pick new-GOP I-picture to "
+                   "encode next.\n");
+            pic->type = FF_HW_PICTURE_TYPE_I;
+            ++ctx->idr_counter;
+        }
+        ctx->gop_counter = 1;
+
+    } else {
+        if (ctx->gop_counter + b_counter + closed_gop_end == ctx->gop_size) {
+            av_log(avctx, AV_LOG_DEBUG, "Pick group-end P-picture to "
+                   "encode next.\n");
+        } else {
+            av_log(avctx, AV_LOG_DEBUG, "Pick normal P-picture to "
+                   "encode next.\n");
+        }
+        pic->type = FF_HW_PICTURE_TYPE_P;
+        av_assert0(start);
+        ctx->gop_counter += 1 + b_counter;
+    }
+    pic->is_reference = 1;
+    *pic_out = pic;
+
+    hw_base_encode_add_ref(pic, pic, 0, 1, 0);
+    if (pic->type != FF_HW_PICTURE_TYPE_IDR) {
+        // TODO: apply both previous and forward multi reference for all vaapi encoders.
+        // And L0/L1 reference frame number can be set dynamically through query
+        // VAConfigAttribEncMaxRefFrames attribute.
+        if (avctx->codec_id == AV_CODEC_ID_AV1) {
+            for (i = 0; i < ctx->nb_next_prev; i++)
+                hw_base_encode_add_ref(pic, ctx->next_prev[i],
+                                       pic->type == FF_HW_PICTURE_TYPE_P,
+                                       b_counter > 0, 0);
+        } else
+            hw_base_encode_add_ref(pic, start,
+                                   pic->type == FF_HW_PICTURE_TYPE_P,
+                                   b_counter > 0, 0);
+
+        hw_base_encode_add_ref(pic, ctx->next_prev[ctx->nb_next_prev - 1], 0, 0, 1);
+    }
+
+    if (b_counter > 0) {
+        hw_base_encode_set_b_pictures(avctx, start, pic, pic, 1,
+                                      &prev);
+    } else {
+        prev = pic;
+    }
+    hw_base_encode_add_next_prev(avctx, prev);
+
+    return 0;
+}
+
+static int hw_base_encode_clear_old(AVCodecContext *avctx)
+{
+    FFHWBaseEncodeContext *ctx = avctx->priv_data;
+    FFHWBaseEncodePicture *pic, *prev, *next;
+
+    av_assert0(ctx->pic_start);
+
+    // Remove direct references once each picture is complete.
+    for (pic = ctx->pic_start; pic; pic = pic->next) {
+        if (pic->encode_complete && pic->next)
+            hw_base_encode_remove_refs(pic, 0);
+    }
+
+    // Remove indirect references once a picture has no direct references.
+    for (pic = ctx->pic_start; pic; pic = pic->next) {
+        if (pic->encode_complete && pic->ref_count[0] == 0)
+            hw_base_encode_remove_refs(pic, 1);
+    }
+
+    // Clear out all complete pictures with no remaining references.
+    prev = NULL;
+    for (pic = ctx->pic_start; pic; pic = next) {
+        next = pic->next;
+        if (pic->encode_complete && pic->ref_count[1] == 0) {
+            av_assert0(pic->ref_removed[0] && pic->ref_removed[1]);
+            if (prev)
+                prev->next = next;
+            else
+                ctx->pic_start = next;
+            ctx->op->free(avctx, pic);
+        } else {
+            prev = pic;
+        }
+    }
+
+    return 0;
+}
+
+static int hw_base_encode_check_frame(AVCodecContext *avctx,
+                                      const AVFrame *frame)
+{
+    FFHWBaseEncodeContext *ctx = avctx->priv_data;
+
+    if ((frame->crop_top  || frame->crop_bottom ||
+         frame->crop_left || frame->crop_right) && !ctx->crop_warned) {
+        av_log(avctx, AV_LOG_WARNING, "Cropping information on input "
+               "frames ignored due to lack of API support.\n");
+        ctx->crop_warned = 1;
+    }
+
+    if (!ctx->roi_allowed) {
+        AVFrameSideData *sd =
+            av_frame_get_side_data(frame, AV_FRAME_DATA_REGIONS_OF_INTEREST);
+
+        if (sd && !ctx->roi_warned) {
+            av_log(avctx, AV_LOG_WARNING, "ROI side data on input "
+                   "frames ignored due to lack of driver support.\n");
+            ctx->roi_warned = 1;
+        }
+    }
+
+    return 0;
+}
+
+static int hw_base_encode_send_frame(AVCodecContext *avctx, AVFrame *frame)
+{
+    FFHWBaseEncodeContext *ctx = avctx->priv_data;
+    FFHWBaseEncodePicture *pic;
+    int err;
+
+    if (frame) {
+        av_log(avctx, AV_LOG_DEBUG, "Input frame: %ux%u (%"PRId64").\n",
+               frame->width, frame->height, frame->pts);
+
+        err = hw_base_encode_check_frame(avctx, frame);
+        if (err < 0)
+            return err;
+
+        pic = ctx->op->alloc(avctx, frame);
+        if (!pic)
+            return AVERROR(ENOMEM);
+
+        pic->input_image = av_frame_alloc();
+        if (!pic->input_image) {
+            err = AVERROR(ENOMEM);
+            goto fail;
+        }
+
+        pic->recon_image = av_frame_alloc();
+        if (!pic->recon_image) {
+            err = AVERROR(ENOMEM);
+            goto fail;
+        }
+
+        if (ctx->input_order == 0 || frame->pict_type == AV_PICTURE_TYPE_I)
+            pic->force_idr = 1;
+
+        pic->pts = frame->pts;
+        pic->duration = frame->duration;
+
+        if (avctx->flags & AV_CODEC_FLAG_COPY_OPAQUE) {
+            err = av_buffer_replace(&pic->opaque_ref, frame->opaque_ref);
+            if (err < 0)
+                goto fail;
+
+            pic->opaque = frame->opaque;
+        }
+
+        av_frame_move_ref(pic->input_image, frame);
+
+        if (ctx->input_order == 0)
+            ctx->first_pts = pic->pts;
+        if (ctx->input_order == ctx->decode_delay)
+            ctx->dts_pts_diff = pic->pts - ctx->first_pts;
+        if (ctx->output_delay > 0)
+            ctx->ts_ring[ctx->input_order %
+                        (3 * ctx->output_delay + ctx->async_depth)] = pic->pts;
+
+        pic->display_order = ctx->input_order;
+        ++ctx->input_order;
+
+        if (ctx->pic_start) {
+            ctx->pic_end->next = pic;
+            ctx->pic_end       = pic;
+        } else {
+            ctx->pic_start     = pic;
+            ctx->pic_end       = pic;
+        }
+
+    } else {
+        ctx->end_of_stream = 1;
+
+        // Fix timestamps if we hit end-of-stream before the initial decode
+        // delay has elapsed.
+        if (ctx->input_order < ctx->decode_delay)
+            ctx->dts_pts_diff = ctx->pic_end->pts - ctx->first_pts;
+    }
+
+    return 0;
+
+fail:
+    ctx->op->free(avctx, pic);
+    return err;
+}
+
+int ff_hw_base_encode_receive_packet(AVCodecContext *avctx, AVPacket *pkt)
+{
+    FFHWBaseEncodeContext *ctx = avctx->priv_data;
+    FFHWBaseEncodePicture *pic = NULL;
+    AVFrame *frame = ctx->frame;
+    int err;
+
+    av_assert0(ctx->op && ctx->op->alloc && ctx->op->issue &&
+               ctx->op->output && ctx->op->free);
+
+start:
+    /** if no B frame before repeat P frame, sent repeat P frame out. */
+    if (ctx->tail_pkt->size) {
+        for (FFHWBaseEncodePicture *tmp = ctx->pic_start; tmp; tmp = tmp->next) {
+            if (tmp->type == FF_HW_PICTURE_TYPE_B && tmp->pts < ctx->tail_pkt->pts)
+                break;
+            else if (!tmp->next) {
+                av_packet_move_ref(pkt, ctx->tail_pkt);
+                goto end;
+            }
+        }
+    }
+
+    err = ff_encode_get_frame(avctx, frame);
+    if (err < 0 && err != AVERROR_EOF)
+        return err;
+
+    if (err == AVERROR_EOF)
+        frame = NULL;
+
+    err = hw_base_encode_send_frame(avctx, frame);
+    if (err < 0)
+        return err;
+
+    if (!ctx->pic_start) {
+        if (ctx->end_of_stream)
+            return AVERROR_EOF;
+        else
+            return AVERROR(EAGAIN);
+    }
+
+    if (ctx->async_encode) {
+        if (av_fifo_can_write(ctx->encode_fifo)) {
+            err = hw_base_encode_pick_next(avctx, &pic);
+            if (!err) {
+                av_assert0(pic);
+                pic->encode_order = ctx->encode_order +
+                    av_fifo_can_read(ctx->encode_fifo);
+                err = ctx->op->issue(avctx, pic);
+                if (err < 0) {
+                    av_log(avctx, AV_LOG_ERROR, "Encode failed: %d.\n", err);
+                    return err;
+                }
+                pic->encode_issued = 1;
+                av_fifo_write(ctx->encode_fifo, &pic, 1);
+            }
+        }
+
+        if (!av_fifo_can_read(ctx->encode_fifo))
+            return err;
+
+        // More frames can be buffered
+        if (av_fifo_can_write(ctx->encode_fifo) && !ctx->end_of_stream)
+            return AVERROR(EAGAIN);
+
+        av_fifo_read(ctx->encode_fifo, &pic, 1);
+        ctx->encode_order = pic->encode_order + 1;
+    } else {
+        err = hw_base_encode_pick_next(avctx, &pic);
+        if (err < 0)
+            return err;
+        av_assert0(pic);
+
+        pic->encode_order = ctx->encode_order++;
+
+        err = ctx->op->issue(avctx, pic);
+        if (err < 0) {
+            av_log(avctx, AV_LOG_ERROR, "Encode failed: %d.\n", err);
+            return err;
+        }
+
+        pic->encode_issued = 1;
+    }
+
+    err = ctx->op->output(avctx, pic, pkt);
+    if (err < 0) {
+        av_log(avctx, AV_LOG_ERROR, "Output failed: %d.\n", err);
+        return err;
+    }
+
+    ctx->output_order = pic->encode_order;
+    hw_base_encode_clear_old(avctx);
+
+    /** loop to get an available pkt in encoder flushing. */
+    if (ctx->end_of_stream && !pkt->size)
+        goto start;
+
+end:
+    if (pkt->size)
+        av_log(avctx, AV_LOG_DEBUG, "Output packet: pts %"PRId64", dts %"PRId64", "
+               "size %d bytes.\n", pkt->pts, pkt->dts, pkt->size);
+
+    return 0;
+}
diff --git a/libavcodec/hw_base_encode.h b/libavcodec/hw_base_encode.h
index 858450afa8..81e6c87036 100644
--- a/libavcodec/hw_base_encode.h
+++ b/libavcodec/hw_base_encode.h
@@ -19,6 +19,8 @@
 #ifndef AVCODEC_HW_BASE_ENCODE_H
 #define AVCODEC_HW_BASE_ENCODE_H
 
+#include "libavutil/fifo.h"
+
 #define MAX_DPB_SIZE 16
 #define MAX_PICTURE_REFERENCES 2
 #define MAX_REORDER_DELAY 16
@@ -54,13 +56,135 @@ enum {
     FF_HW_FLAG_NON_IDR_KEY_PICTURES  = 1 << 5,
 };
 
+typedef struct FFHWBaseEncodePicture {
+    struct FFHWBaseEncodePicture *next;
+
+    int64_t         display_order;
+    int64_t         encode_order;
+    int64_t         pts;
+    int64_t         duration;
+    int             force_idr;
+
+    void           *opaque;
+    AVBufferRef    *opaque_ref;
+
+    int             type;
+    int             b_depth;
+    int             encode_issued;
+    int             encode_complete;
+
+    AVFrame        *input_image;
+    AVFrame        *recon_image;
+
+    void           *priv_data;
+
+    // Whether this picture is a reference picture.
+    int             is_reference;
+
+    // The contents of the DPB after this picture has been decoded.
+    // This will contain the picture itself if it is a reference picture,
+    // but not if it isn't.
+    int                     nb_dpb_pics;
+    struct FFHWBaseEncodePicture *dpb[MAX_DPB_SIZE];
+    // The reference pictures used in decoding this picture. If they are
+    // used by later pictures they will also appear in the DPB. ref[0][] for
+    // previous reference frames. ref[1][] for future reference frames.
+    int                     nb_refs[MAX_REFERENCE_LIST_NUM];
+    struct FFHWBaseEncodePicture *refs[MAX_REFERENCE_LIST_NUM][MAX_PICTURE_REFERENCES];
+    // The previous reference picture in encode order.  Must be in at least
+    // one of the reference list and DPB list.
+    struct FFHWBaseEncodePicture *prev;
+    // Reference count for other pictures referring to this one through
+    // the above pointers, directly from incomplete pictures and indirectly
+    // through completed pictures.
+    int             ref_count[2];
+    int             ref_removed[2];
+} FFHWBaseEncodePicture;
+
+typedef struct FFHWEncodePictureOperation {
+    // Alloc memory for the picture structure and initialize the API-specific internals
+    // based of the given frame.
+    FFHWBaseEncodePicture * (*alloc)(AVCodecContext *avctx, const AVFrame *frame);
+    // Issue the picture structure, which will send the frame surface to HW Encode API.
+    int (*issue)(AVCodecContext *avctx, const FFHWBaseEncodePicture *base_pic);
+    // Get the output AVPacket.
+    int (*output)(AVCodecContext *avctx, const FFHWBaseEncodePicture *base_pic, AVPacket *pkt);
+    // Free the picture structure.
+    int (*free)(AVCodecContext *avctx, FFHWBaseEncodePicture *base_pic);
+}  FFHWEncodePictureOperation;
+
 typedef struct FFHWBaseEncodeContext {
     const AVClass *class;
 
+    // Hardware-specific hooks.
+    const struct FFHWEncodePictureOperation *op;
+
+    // Current encoding window, in display (input) order.
+    FFHWBaseEncodePicture *pic_start, *pic_end;
+    // The next picture to use as the previous reference picture in
+    // encoding order. Order from small to large in encoding order.
+    FFHWBaseEncodePicture *next_prev[MAX_PICTURE_REFERENCES];
+    int                  nb_next_prev;
+
+    // Next input order index (display order).
+    int64_t         input_order;
+    // Number of frames that output is behind input.
+    int64_t         output_delay;
+    // Next encode order index.
+    int64_t         encode_order;
+    // Number of frames decode output will need to be delayed.
+    int64_t         decode_delay;
+    // Next output order index (in encode order).
+    int64_t         output_order;
+
+    // Timestamp handling.
+    int64_t         first_pts;
+    int64_t         dts_pts_diff;
+    int64_t         ts_ring[MAX_REORDER_DELAY * 3 +
+                            MAX_ASYNC_DEPTH];
+
+    // Frame type decision.
+    int gop_size;
+    int closed_gop;
+    int gop_per_idr;
+    int p_per_i;
+    int max_b_depth;
+    int b_per_p;
+    int force_idr;
+    int idr_counter;
+    int gop_counter;
+    int end_of_stream;
+    int p_to_gpb;
+
+    // Whether the driver supports ROI at all.
+    int             roi_allowed;
+
+    // The encoder does not support cropping information, so warn about
+    // it the first time we encounter any nonzero crop fields.
+    int             crop_warned;
+    // If the driver does not support ROI then warn the first time we
+    // encounter a frame with ROI side data.
+    int             roi_warned;
+
+    // The frame to be filled with data.
+    AVFrame         *frame;
+
+    // Whether the HW supports sync buffer function.
+    // If supported, encode_fifo/async_depth will be used together.
+    // Used for output buffer synchronization.
+    int             async_encode;
+
+    // Store buffered pic.
+    AVFifo          *encode_fifo;
     // Max number of frame buffered in encoder.
     int             async_depth;
+
+    /** Tail data of a pic, now only used for av1 repeat frame header. */
+    AVPacket        *tail_pkt;
 } FFHWBaseEncodeContext;
 
+int ff_hw_base_encode_receive_packet(AVCodecContext *avctx, AVPacket *pkt);
+
 #define HW_BASE_ENCODE_COMMON_OPTIONS \
     { "async_depth", "Maximum processing parallelism. " \
       "Increase this to improve single channel performance.", \
diff --git a/libavcodec/vaapi_encode.c b/libavcodec/vaapi_encode.c
index 194422b36d..1055fca0b1 100644
--- a/libavcodec/vaapi_encode.c
+++ b/libavcodec/vaapi_encode.c
@@ -138,22 +138,26 @@ static int vaapi_encode_make_misc_param_buffer(AVCodecContext *avctx,
 static int vaapi_encode_wait(AVCodecContext *avctx,
                              VAAPIEncodePicture *pic)
 {
+#if VA_CHECK_VERSION(1, 9, 0)
+    FFHWBaseEncodeContext *base_ctx = avctx->priv_data;
+#endif
     VAAPIEncodeContext *ctx = avctx->priv_data;
+    FFHWBaseEncodePicture *base_pic = &pic->base;
     VAStatus vas;
 
-    av_assert0(pic->encode_issued);
+    av_assert0(base_pic->encode_issued);
 
-    if (pic->encode_complete) {
+    if (base_pic->encode_complete) {
         // Already waited for this picture.
         return 0;
     }
 
     av_log(avctx, AV_LOG_DEBUG, "Sync to pic %"PRId64"/%"PRId64" "
-           "(input surface %#x).\n", pic->display_order,
-           pic->encode_order, pic->input_surface);
+           "(input surface %#x).\n", base_pic->display_order,
+           base_pic->encode_order, pic->input_surface);
 
 #if VA_CHECK_VERSION(1, 9, 0)
-    if (ctx->has_sync_buffer_func) {
+    if (base_ctx->async_encode) {
         vas = vaSyncBuffer(ctx->hwctx->display,
                            pic->output_buffer,
                            VA_TIMEOUT_INFINITE);
@@ -174,9 +178,9 @@ static int vaapi_encode_wait(AVCodecContext *avctx,
     }
 
     // Input is definitely finished with now.
-    av_frame_free(&pic->input_image);
+    av_frame_free(&base_pic->input_image);
 
-    pic->encode_complete = 1;
+    base_pic->encode_complete = 1;
     return 0;
 }
 
@@ -263,9 +267,11 @@ static int vaapi_encode_make_tile_slice(AVCodecContext *avctx,
 }
 
 static int vaapi_encode_issue(AVCodecContext *avctx,
-                              VAAPIEncodePicture *pic)
+                              const FFHWBaseEncodePicture *base_pic)
 {
-    VAAPIEncodeContext *ctx = avctx->priv_data;
+    FFHWBaseEncodeContext *base_ctx = avctx->priv_data;
+    VAAPIEncodeContext         *ctx = avctx->priv_data;
+    VAAPIEncodePicture *pic = (VAAPIEncodePicture*)base_pic;
     VAAPIEncodeSlice *slice;
     VAStatus vas;
     int err, i;
@@ -274,52 +280,46 @@ static int vaapi_encode_issue(AVCodecContext *avctx,
     av_unused AVFrameSideData *sd;
 
     av_log(avctx, AV_LOG_DEBUG, "Issuing encode for pic %"PRId64"/%"PRId64" "
-           "as type %s.\n", pic->display_order, pic->encode_order,
-           ff_hw_base_encode_get_pictype_name(pic->type));
-    if (pic->nb_refs[0] == 0 && pic->nb_refs[1] == 0) {
+           "as type %s.\n", base_pic->display_order, base_pic->encode_order,
+           ff_hw_base_encode_get_pictype_name(base_pic->type));
+    if (base_pic->nb_refs[0] == 0 && base_pic->nb_refs[1] == 0) {
         av_log(avctx, AV_LOG_DEBUG, "No reference pictures.\n");
     } else {
         av_log(avctx, AV_LOG_DEBUG, "L0 refers to");
-        for (i = 0; i < pic->nb_refs[0]; i++) {
+        for (i = 0; i < base_pic->nb_refs[0]; i++) {
             av_log(avctx, AV_LOG_DEBUG, " %"PRId64"/%"PRId64,
-                   pic->refs[0][i]->display_order, pic->refs[0][i]->encode_order);
+                   base_pic->refs[0][i]->display_order, base_pic->refs[0][i]->encode_order);
         }
         av_log(avctx, AV_LOG_DEBUG, ".\n");
 
-        if (pic->nb_refs[1]) {
+        if (base_pic->nb_refs[1]) {
             av_log(avctx, AV_LOG_DEBUG, "L1 refers to");
-            for (i = 0; i < pic->nb_refs[1]; i++) {
+            for (i = 0; i < base_pic->nb_refs[1]; i++) {
                 av_log(avctx, AV_LOG_DEBUG, " %"PRId64"/%"PRId64,
-                       pic->refs[1][i]->display_order, pic->refs[1][i]->encode_order);
+                       base_pic->refs[1][i]->display_order, base_pic->refs[1][i]->encode_order);
             }
             av_log(avctx, AV_LOG_DEBUG, ".\n");
         }
     }
 
-    av_assert0(!pic->encode_issued);
-    for (i = 0; i < pic->nb_refs[0]; i++) {
-        av_assert0(pic->refs[0][i]);
-        av_assert0(pic->refs[0][i]->encode_issued);
+    av_assert0(!base_pic->encode_issued);
+    for (i = 0; i < base_pic->nb_refs[0]; i++) {
+        av_assert0(base_pic->refs[0][i]);
+        av_assert0(base_pic->refs[0][i]->encode_issued);
     }
-    for (i = 0; i < pic->nb_refs[1]; i++) {
-        av_assert0(pic->refs[1][i]);
-        av_assert0(pic->refs[1][i]->encode_issued);
+    for (i = 0; i < base_pic->nb_refs[1]; i++) {
+        av_assert0(base_pic->refs[1][i]);
+        av_assert0(base_pic->refs[1][i]->encode_issued);
     }
 
     av_log(avctx, AV_LOG_DEBUG, "Input surface is %#x.\n", pic->input_surface);
 
-    pic->recon_image = av_frame_alloc();
-    if (!pic->recon_image) {
-        err = AVERROR(ENOMEM);
-        goto fail;
-    }
-
-    err = av_hwframe_get_buffer(ctx->recon_frames_ref, pic->recon_image, 0);
+    err = av_hwframe_get_buffer(ctx->recon_frames_ref, base_pic->recon_image, 0);
     if (err < 0) {
         err = AVERROR(ENOMEM);
         goto fail;
     }
-    pic->recon_surface = (VASurfaceID)(uintptr_t)pic->recon_image->data[3];
+    pic->recon_surface = (VASurfaceID)(uintptr_t)base_pic->recon_image->data[3];
     av_log(avctx, AV_LOG_DEBUG, "Recon surface is %#x.\n", pic->recon_surface);
 
     pic->output_buffer_ref = ff_refstruct_pool_get(ctx->output_buffer_pool);
@@ -343,7 +343,7 @@ static int vaapi_encode_issue(AVCodecContext *avctx,
 
     pic->nb_param_buffers = 0;
 
-    if (pic->type == FF_HW_PICTURE_TYPE_IDR && ctx->codec->init_sequence_params) {
+    if (base_pic->type == FF_HW_PICTURE_TYPE_IDR && ctx->codec->init_sequence_params) {
         err = vaapi_encode_make_param_buffer(avctx, pic,
                                              VAEncSequenceParameterBufferType,
                                              ctx->codec_sequence_params,
@@ -352,7 +352,7 @@ static int vaapi_encode_issue(AVCodecContext *avctx,
             goto fail;
     }
 
-    if (pic->type == FF_HW_PICTURE_TYPE_IDR) {
+    if (base_pic->type == FF_HW_PICTURE_TYPE_IDR) {
         for (i = 0; i < ctx->nb_global_params; i++) {
             err = vaapi_encode_make_misc_param_buffer(avctx, pic,
                                                       ctx->global_params_type[i],
@@ -389,7 +389,7 @@ static int vaapi_encode_issue(AVCodecContext *avctx,
     }
 #endif
 
-    if (pic->type == FF_HW_PICTURE_TYPE_IDR) {
+    if (base_pic->type == FF_HW_PICTURE_TYPE_IDR) {
         if (ctx->va_packed_headers & VA_ENC_PACKED_HEADER_SEQUENCE &&
             ctx->codec->write_sequence_header) {
             bit_len = 8 * sizeof(data);
@@ -529,9 +529,9 @@ static int vaapi_encode_issue(AVCodecContext *avctx,
     }
 
 #if VA_CHECK_VERSION(1, 0, 0)
-    sd = av_frame_get_side_data(pic->input_image,
+    sd = av_frame_get_side_data(base_pic->input_image,
                                 AV_FRAME_DATA_REGIONS_OF_INTEREST);
-    if (sd && ctx->roi_allowed) {
+    if (sd && base_ctx->roi_allowed) {
         const AVRegionOfInterest *roi;
         uint32_t roi_size;
         VAEncMiscParameterBufferROI param_roi;
@@ -542,11 +542,11 @@ static int vaapi_encode_issue(AVCodecContext *avctx,
         av_assert0(roi_size && sd->size % roi_size == 0);
         nb_roi = sd->size / roi_size;
         if (nb_roi > ctx->roi_max_regions) {
-            if (!ctx->roi_warned) {
+            if (!base_ctx->roi_warned) {
                 av_log(avctx, AV_LOG_WARNING, "More ROIs set than "
                        "supported by driver (%d > %d).\n",
                        nb_roi, ctx->roi_max_regions);
-                ctx->roi_warned = 1;
+                base_ctx->roi_warned = 1;
             }
             nb_roi = ctx->roi_max_regions;
         }
@@ -639,8 +639,6 @@ static int vaapi_encode_issue(AVCodecContext *avctx,
         }
     }
 
-    pic->encode_issued = 1;
-
     return 0;
 
 fail_with_picture:
@@ -657,14 +655,13 @@ fail_at_end:
     av_freep(&pic->param_buffers);
     av_freep(&pic->slices);
     av_freep(&pic->roi);
-    av_frame_free(&pic->recon_image);
     ff_refstruct_unref(&pic->output_buffer_ref);
     pic->output_buffer = VA_INVALID_ID;
     return err;
 }
 
 static int vaapi_encode_set_output_property(AVCodecContext *avctx,
-                                            VAAPIEncodePicture *pic,
+                                            FFHWBaseEncodePicture *pic,
                                             AVPacket *pkt)
 {
     FFHWBaseEncodeContext *base_ctx = avctx->priv_data;
@@ -689,16 +686,16 @@ static int vaapi_encode_set_output_property(AVCodecContext *avctx,
         return 0;
     }
 
-    if (ctx->output_delay == 0) {
+    if (base_ctx->output_delay == 0) {
         pkt->dts = pkt->pts;
-    } else if (pic->encode_order < ctx->decode_delay) {
-        if (ctx->ts_ring[pic->encode_order] < INT64_MIN + ctx->dts_pts_diff)
+    } else if (pic->encode_order < base_ctx->decode_delay) {
+        if (base_ctx->ts_ring[pic->encode_order] < INT64_MIN + base_ctx->dts_pts_diff)
             pkt->dts = INT64_MIN;
         else
-            pkt->dts = ctx->ts_ring[pic->encode_order] - ctx->dts_pts_diff;
+            pkt->dts = base_ctx->ts_ring[pic->encode_order] - base_ctx->dts_pts_diff;
     } else {
-        pkt->dts = ctx->ts_ring[(pic->encode_order - ctx->decode_delay) %
-                                (3 * ctx->output_delay + base_ctx->async_depth)];
+        pkt->dts = base_ctx->ts_ring[(pic->encode_order - base_ctx->decode_delay) %
+                                     (3 * base_ctx->output_delay + base_ctx->async_depth)];
     }
 
     return 0;
@@ -817,9 +814,11 @@ end:
 }
 
 static int vaapi_encode_output(AVCodecContext *avctx,
-                               VAAPIEncodePicture *pic, AVPacket *pkt)
+                               const FFHWBaseEncodePicture *base_pic, AVPacket *pkt)
 {
-    VAAPIEncodeContext *ctx = avctx->priv_data;
+    FFHWBaseEncodeContext *base_ctx = avctx->priv_data;
+    VAAPIEncodeContext         *ctx = avctx->priv_data;
+    VAAPIEncodePicture         *pic = (VAAPIEncodePicture*)base_pic;
     AVPacket *pkt_ptr = pkt;
     int err;
 
@@ -832,17 +831,17 @@ static int vaapi_encode_output(AVCodecContext *avctx,
         ctx->coded_buffer_ref = ff_refstruct_ref(pic->output_buffer_ref);
 
         if (pic->tail_size) {
-            if (ctx->tail_pkt->size) {
+            if (base_ctx->tail_pkt->size) {
                 err = AVERROR_BUG;
                 goto end;
             }
 
-            err = ff_get_encode_buffer(avctx, ctx->tail_pkt, pic->tail_size, 0);
+            err = ff_get_encode_buffer(avctx, base_ctx->tail_pkt, pic->tail_size, 0);
             if (err < 0)
                 goto end;
 
-            memcpy(ctx->tail_pkt->data, pic->tail_data, pic->tail_size);
-            pkt_ptr = ctx->tail_pkt;
+            memcpy(base_ctx->tail_pkt->data, pic->tail_data, pic->tail_size);
+            pkt_ptr = base_ctx->tail_pkt;
         }
     } else {
         err = vaapi_encode_get_coded_data(avctx, pic, pkt);
@@ -851,9 +850,9 @@ static int vaapi_encode_output(AVCodecContext *avctx,
     }
 
     av_log(avctx, AV_LOG_DEBUG, "Output read for pic %"PRId64"/%"PRId64".\n",
-           pic->display_order, pic->encode_order);
+           base_pic->display_order, base_pic->encode_order);
 
-    vaapi_encode_set_output_property(avctx, pic, pkt_ptr);
+    vaapi_encode_set_output_property(avctx, (FFHWBaseEncodePicture*)pic, pkt_ptr);
 
 end:
     ff_refstruct_unref(&pic->output_buffer_ref);
@@ -864,12 +863,14 @@ end:
 static int vaapi_encode_discard(AVCodecContext *avctx,
                                 VAAPIEncodePicture *pic)
 {
+    FFHWBaseEncodePicture *base_pic = &pic->base;
+
     vaapi_encode_wait(avctx, pic);
 
     if (pic->output_buffer_ref) {
         av_log(avctx, AV_LOG_DEBUG, "Discard output for pic "
                "%"PRId64"/%"PRId64".\n",
-               pic->display_order, pic->encode_order);
+               base_pic->display_order, base_pic->encode_order);
 
         ff_refstruct_unref(&pic->output_buffer_ref);
         pic->output_buffer = VA_INVALID_ID;
@@ -878,8 +879,8 @@ static int vaapi_encode_discard(AVCodecContext *avctx,
     return 0;
 }
 
-static VAAPIEncodePicture *vaapi_encode_alloc(AVCodecContext *avctx,
-                                              const AVFrame *frame)
+static FFHWBaseEncodePicture *vaapi_encode_alloc(AVCodecContext *avctx,
+                                                 const AVFrame *frame)
 {
     VAAPIEncodeContext *ctx = avctx->priv_data;
     VAAPIEncodePicture *pic;
@@ -889,8 +890,8 @@ static VAAPIEncodePicture *vaapi_encode_alloc(AVCodecContext *avctx,
         return NULL;
 
     if (ctx->codec->picture_priv_data_size > 0) {
-        pic->priv_data = av_mallocz(ctx->codec->picture_priv_data_size);
-        if (!pic->priv_data) {
+        pic->base.priv_data = av_mallocz(ctx->codec->picture_priv_data_size);
+        if (!pic->base.priv_data) {
             av_freep(&pic);
             return NULL;
         }
@@ -900,15 +901,16 @@ static VAAPIEncodePicture *vaapi_encode_alloc(AVCodecContext *avctx,
     pic->recon_surface = VA_INVALID_ID;
     pic->output_buffer = VA_INVALID_ID;
 
-    return pic;
+    return &pic->base;
 }
 
 static int vaapi_encode_free(AVCodecContext *avctx,
-                             VAAPIEncodePicture *pic)
+                             FFHWBaseEncodePicture *base_pic)
 {
+    VAAPIEncodePicture *pic = (VAAPIEncodePicture*)base_pic;
     int i;
 
-    if (pic->encode_issued)
+    if (base_pic->encode_issued)
         vaapi_encode_discard(avctx, pic);
 
     if (pic->slices) {
@@ -916,17 +918,17 @@ static int vaapi_encode_free(AVCodecContext *avctx,
             av_freep(&pic->slices[i].codec_slice_params);
     }
 
-    av_frame_free(&pic->input_image);
-    av_frame_free(&pic->recon_image);
+    av_frame_free(&base_pic->input_image);
+    av_frame_free(&base_pic->recon_image);
 
-    av_buffer_unref(&pic->opaque_ref);
+    av_buffer_unref(&base_pic->opaque_ref);
 
     av_freep(&pic->param_buffers);
     av_freep(&pic->slices);
     // Output buffer should already be destroyed.
     av_assert0(pic->output_buffer == VA_INVALID_ID);
 
-    av_freep(&pic->priv_data);
+    av_freep(&base_pic->priv_data);
     av_freep(&pic->codec_picture_params);
     av_freep(&pic->roi);
 
@@ -935,564 +937,6 @@ static int vaapi_encode_free(AVCodecContext *avctx,
     return 0;
 }
 
-static void vaapi_encode_add_ref(AVCodecContext *avctx,
-                                 VAAPIEncodePicture *pic,
-                                 VAAPIEncodePicture *target,
-                                 int is_ref, int in_dpb, int prev)
-{
-    int refs = 0;
-
-    if (is_ref) {
-        av_assert0(pic != target);
-        av_assert0(pic->nb_refs[0] < MAX_PICTURE_REFERENCES &&
-                   pic->nb_refs[1] < MAX_PICTURE_REFERENCES);
-        if (target->display_order < pic->display_order)
-            pic->refs[0][pic->nb_refs[0]++] = target;
-        else
-            pic->refs[1][pic->nb_refs[1]++] = target;
-        ++refs;
-    }
-
-    if (in_dpb) {
-        av_assert0(pic->nb_dpb_pics < MAX_DPB_SIZE);
-        pic->dpb[pic->nb_dpb_pics++] = target;
-        ++refs;
-    }
-
-    if (prev) {
-        av_assert0(!pic->prev);
-        pic->prev = target;
-        ++refs;
-    }
-
-    target->ref_count[0] += refs;
-    target->ref_count[1] += refs;
-}
-
-static void vaapi_encode_remove_refs(AVCodecContext *avctx,
-                                     VAAPIEncodePicture *pic,
-                                     int level)
-{
-    int i;
-
-    if (pic->ref_removed[level])
-        return;
-
-    for (i = 0; i < pic->nb_refs[0]; i++) {
-        av_assert0(pic->refs[0][i]);
-        --pic->refs[0][i]->ref_count[level];
-        av_assert0(pic->refs[0][i]->ref_count[level] >= 0);
-    }
-
-    for (i = 0; i < pic->nb_refs[1]; i++) {
-        av_assert0(pic->refs[1][i]);
-        --pic->refs[1][i]->ref_count[level];
-        av_assert0(pic->refs[1][i]->ref_count[level] >= 0);
-    }
-
-    for (i = 0; i < pic->nb_dpb_pics; i++) {
-        av_assert0(pic->dpb[i]);
-        --pic->dpb[i]->ref_count[level];
-        av_assert0(pic->dpb[i]->ref_count[level] >= 0);
-    }
-
-    av_assert0(pic->prev || pic->type == FF_HW_PICTURE_TYPE_IDR);
-    if (pic->prev) {
-        --pic->prev->ref_count[level];
-        av_assert0(pic->prev->ref_count[level] >= 0);
-    }
-
-    pic->ref_removed[level] = 1;
-}
-
-static void vaapi_encode_set_b_pictures(AVCodecContext *avctx,
-                                        VAAPIEncodePicture *start,
-                                        VAAPIEncodePicture *end,
-                                        VAAPIEncodePicture *prev,
-                                        int current_depth,
-                                        VAAPIEncodePicture **last)
-{
-    VAAPIEncodeContext *ctx = avctx->priv_data;
-    VAAPIEncodePicture *pic, *next, *ref;
-    int i, len;
-
-    av_assert0(start && end && start != end && start->next != end);
-
-    // If we are at the maximum depth then encode all pictures as
-    // non-referenced B-pictures.  Also do this if there is exactly one
-    // picture left, since there will be nothing to reference it.
-    if (current_depth == ctx->max_b_depth || start->next->next == end) {
-        for (pic = start->next; pic; pic = pic->next) {
-            if (pic == end)
-                break;
-            pic->type    = FF_HW_PICTURE_TYPE_B;
-            pic->b_depth = current_depth;
-
-            vaapi_encode_add_ref(avctx, pic, start, 1, 1, 0);
-            vaapi_encode_add_ref(avctx, pic, end,   1, 1, 0);
-            vaapi_encode_add_ref(avctx, pic, prev,  0, 0, 1);
-
-            for (ref = end->refs[1][0]; ref; ref = ref->refs[1][0])
-                vaapi_encode_add_ref(avctx, pic, ref, 0, 1, 0);
-        }
-        *last = prev;
-
-    } else {
-        // Split the current list at the midpoint with a referenced
-        // B-picture, then descend into each side separately.
-        len = 0;
-        for (pic = start->next; pic != end; pic = pic->next)
-            ++len;
-        for (pic = start->next, i = 1; 2 * i < len; pic = pic->next, i++);
-
-        pic->type    = FF_HW_PICTURE_TYPE_B;
-        pic->b_depth = current_depth;
-
-        pic->is_reference = 1;
-
-        vaapi_encode_add_ref(avctx, pic, pic,   0, 1, 0);
-        vaapi_encode_add_ref(avctx, pic, start, 1, 1, 0);
-        vaapi_encode_add_ref(avctx, pic, end,   1, 1, 0);
-        vaapi_encode_add_ref(avctx, pic, prev,  0, 0, 1);
-
-        for (ref = end->refs[1][0]; ref; ref = ref->refs[1][0])
-            vaapi_encode_add_ref(avctx, pic, ref, 0, 1, 0);
-
-        if (i > 1)
-            vaapi_encode_set_b_pictures(avctx, start, pic, pic,
-                                        current_depth + 1, &next);
-        else
-            next = pic;
-
-        vaapi_encode_set_b_pictures(avctx, pic, end, next,
-                                    current_depth + 1, last);
-    }
-}
-
-static void vaapi_encode_add_next_prev(AVCodecContext *avctx,
-                                       VAAPIEncodePicture *pic)
-{
-    VAAPIEncodeContext *ctx = avctx->priv_data;
-    int i;
-
-    if (!pic)
-        return;
-
-    if (pic->type == FF_HW_PICTURE_TYPE_IDR) {
-        for (i = 0; i < ctx->nb_next_prev; i++) {
-            --ctx->next_prev[i]->ref_count[0];
-            ctx->next_prev[i] = NULL;
-        }
-        ctx->next_prev[0] = pic;
-        ++pic->ref_count[0];
-        ctx->nb_next_prev = 1;
-
-        return;
-    }
-
-    if (ctx->nb_next_prev < MAX_PICTURE_REFERENCES) {
-        ctx->next_prev[ctx->nb_next_prev++] = pic;
-        ++pic->ref_count[0];
-    } else {
-        --ctx->next_prev[0]->ref_count[0];
-        for (i = 0; i < MAX_PICTURE_REFERENCES - 1; i++)
-            ctx->next_prev[i] = ctx->next_prev[i + 1];
-        ctx->next_prev[i] = pic;
-        ++pic->ref_count[0];
-    }
-}
-
-static int vaapi_encode_pick_next(AVCodecContext *avctx,
-                                  VAAPIEncodePicture **pic_out)
-{
-    VAAPIEncodeContext *ctx = avctx->priv_data;
-    VAAPIEncodePicture *pic = NULL, *prev = NULL, *next, *start;
-    int i, b_counter, closed_gop_end;
-
-    // If there are any B-frames already queued, the next one to encode
-    // is the earliest not-yet-issued frame for which all references are
-    // available.
-    for (pic = ctx->pic_start; pic; pic = pic->next) {
-        if (pic->encode_issued)
-            continue;
-        if (pic->type != FF_HW_PICTURE_TYPE_B)
-            continue;
-        for (i = 0; i < pic->nb_refs[0]; i++) {
-            if (!pic->refs[0][i]->encode_issued)
-                break;
-        }
-        if (i != pic->nb_refs[0])
-            continue;
-
-        for (i = 0; i < pic->nb_refs[1]; i++) {
-            if (!pic->refs[1][i]->encode_issued)
-                break;
-        }
-        if (i == pic->nb_refs[1])
-            break;
-    }
-
-    if (pic) {
-        av_log(avctx, AV_LOG_DEBUG, "Pick B-picture at depth %d to "
-               "encode next.\n", pic->b_depth);
-        *pic_out = pic;
-        return 0;
-    }
-
-    // Find the B-per-Pth available picture to become the next picture
-    // on the top layer.
-    start = NULL;
-    b_counter = 0;
-    closed_gop_end = ctx->closed_gop ||
-                     ctx->idr_counter == ctx->gop_per_idr;
-    for (pic = ctx->pic_start; pic; pic = next) {
-        next = pic->next;
-        if (pic->encode_issued) {
-            start = pic;
-            continue;
-        }
-        // If the next available picture is force-IDR, encode it to start
-        // a new GOP immediately.
-        if (pic->force_idr)
-            break;
-        if (b_counter == ctx->b_per_p)
-            break;
-        // If this picture ends a closed GOP or starts a new GOP then it
-        // needs to be in the top layer.
-        if (ctx->gop_counter + b_counter + closed_gop_end >= ctx->gop_size)
-            break;
-        // If the picture after this one is force-IDR, we need to encode
-        // this one in the top layer.
-        if (next && next->force_idr)
-            break;
-        ++b_counter;
-    }
-
-    // At the end of the stream the last picture must be in the top layer.
-    if (!pic && ctx->end_of_stream) {
-        --b_counter;
-        pic = ctx->pic_end;
-        if (pic->encode_complete)
-            return AVERROR_EOF;
-        else if (pic->encode_issued)
-            return AVERROR(EAGAIN);
-    }
-
-    if (!pic) {
-        av_log(avctx, AV_LOG_DEBUG, "Pick nothing to encode next - "
-               "need more input for reference pictures.\n");
-        return AVERROR(EAGAIN);
-    }
-    if (ctx->input_order <= ctx->decode_delay && !ctx->end_of_stream) {
-        av_log(avctx, AV_LOG_DEBUG, "Pick nothing to encode next - "
-               "need more input for timestamps.\n");
-        return AVERROR(EAGAIN);
-    }
-
-    if (pic->force_idr) {
-        av_log(avctx, AV_LOG_DEBUG, "Pick forced IDR-picture to "
-               "encode next.\n");
-        pic->type = FF_HW_PICTURE_TYPE_IDR;
-        ctx->idr_counter = 1;
-        ctx->gop_counter = 1;
-
-    } else if (ctx->gop_counter + b_counter >= ctx->gop_size) {
-        if (ctx->idr_counter == ctx->gop_per_idr) {
-            av_log(avctx, AV_LOG_DEBUG, "Pick new-GOP IDR-picture to "
-                   "encode next.\n");
-            pic->type = FF_HW_PICTURE_TYPE_IDR;
-            ctx->idr_counter = 1;
-        } else {
-            av_log(avctx, AV_LOG_DEBUG, "Pick new-GOP I-picture to "
-                   "encode next.\n");
-            pic->type = FF_HW_PICTURE_TYPE_I;
-            ++ctx->idr_counter;
-        }
-        ctx->gop_counter = 1;
-
-    } else {
-        if (ctx->gop_counter + b_counter + closed_gop_end == ctx->gop_size) {
-            av_log(avctx, AV_LOG_DEBUG, "Pick group-end P-picture to "
-                   "encode next.\n");
-        } else {
-            av_log(avctx, AV_LOG_DEBUG, "Pick normal P-picture to "
-                   "encode next.\n");
-        }
-        pic->type = FF_HW_PICTURE_TYPE_P;
-        av_assert0(start);
-        ctx->gop_counter += 1 + b_counter;
-    }
-    pic->is_reference = 1;
-    *pic_out = pic;
-
-    vaapi_encode_add_ref(avctx, pic, pic, 0, 1, 0);
-    if (pic->type != FF_HW_PICTURE_TYPE_IDR) {
-        // TODO: apply both previous and forward multi reference for all vaapi encoders.
-        // And L0/L1 reference frame number can be set dynamically through query
-        // VAConfigAttribEncMaxRefFrames attribute.
-        if (avctx->codec_id == AV_CODEC_ID_AV1) {
-            for (i = 0; i < ctx->nb_next_prev; i++)
-                vaapi_encode_add_ref(avctx, pic, ctx->next_prev[i],
-                                     pic->type == FF_HW_PICTURE_TYPE_P,
-                                     b_counter > 0, 0);
-        } else
-            vaapi_encode_add_ref(avctx, pic, start,
-                                 pic->type == FF_HW_PICTURE_TYPE_P,
-                                 b_counter > 0, 0);
-
-        vaapi_encode_add_ref(avctx, pic, ctx->next_prev[ctx->nb_next_prev - 1], 0, 0, 1);
-    }
-
-    if (b_counter > 0) {
-        vaapi_encode_set_b_pictures(avctx, start, pic, pic, 1,
-                                    &prev);
-    } else {
-        prev = pic;
-    }
-    vaapi_encode_add_next_prev(avctx, prev);
-
-    return 0;
-}
-
-static int vaapi_encode_clear_old(AVCodecContext *avctx)
-{
-    VAAPIEncodeContext *ctx = avctx->priv_data;
-    VAAPIEncodePicture *pic, *prev, *next;
-
-    av_assert0(ctx->pic_start);
-
-    // Remove direct references once each picture is complete.
-    for (pic = ctx->pic_start; pic; pic = pic->next) {
-        if (pic->encode_complete && pic->next)
-            vaapi_encode_remove_refs(avctx, pic, 0);
-    }
-
-    // Remove indirect references once a picture has no direct references.
-    for (pic = ctx->pic_start; pic; pic = pic->next) {
-        if (pic->encode_complete && pic->ref_count[0] == 0)
-            vaapi_encode_remove_refs(avctx, pic, 1);
-    }
-
-    // Clear out all complete pictures with no remaining references.
-    prev = NULL;
-    for (pic = ctx->pic_start; pic; pic = next) {
-        next = pic->next;
-        if (pic->encode_complete && pic->ref_count[1] == 0) {
-            av_assert0(pic->ref_removed[0] && pic->ref_removed[1]);
-            if (prev)
-                prev->next = next;
-            else
-                ctx->pic_start = next;
-            vaapi_encode_free(avctx, pic);
-        } else {
-            prev = pic;
-        }
-    }
-
-    return 0;
-}
-
-static int vaapi_encode_check_frame(AVCodecContext *avctx,
-                                    const AVFrame *frame)
-{
-    VAAPIEncodeContext *ctx = avctx->priv_data;
-
-    if ((frame->crop_top  || frame->crop_bottom ||
-         frame->crop_left || frame->crop_right) && !ctx->crop_warned) {
-        av_log(avctx, AV_LOG_WARNING, "Cropping information on input "
-               "frames ignored due to lack of API support.\n");
-        ctx->crop_warned = 1;
-    }
-
-    if (!ctx->roi_allowed) {
-        AVFrameSideData *sd =
-            av_frame_get_side_data(frame, AV_FRAME_DATA_REGIONS_OF_INTEREST);
-
-        if (sd && !ctx->roi_warned) {
-            av_log(avctx, AV_LOG_WARNING, "ROI side data on input "
-                   "frames ignored due to lack of driver support.\n");
-            ctx->roi_warned = 1;
-        }
-    }
-
-    return 0;
-}
-
-static int vaapi_encode_send_frame(AVCodecContext *avctx, AVFrame *frame)
-{
-    FFHWBaseEncodeContext *base_ctx = avctx->priv_data;
-    VAAPIEncodeContext *ctx = avctx->priv_data;
-    VAAPIEncodePicture *pic;
-    int err;
-
-    if (frame) {
-        av_log(avctx, AV_LOG_DEBUG, "Input frame: %ux%u (%"PRId64").\n",
-               frame->width, frame->height, frame->pts);
-
-        err = vaapi_encode_check_frame(avctx, frame);
-        if (err < 0)
-            return err;
-
-        pic = vaapi_encode_alloc(avctx, frame);
-        if (!pic)
-            return AVERROR(ENOMEM);
-
-        pic->input_image = av_frame_alloc();
-        if (!pic->input_image) {
-            err = AVERROR(ENOMEM);
-            goto fail;
-        }
-
-        if (ctx->input_order == 0 || frame->pict_type == AV_PICTURE_TYPE_I)
-            pic->force_idr = 1;
-
-        pic->pts = frame->pts;
-        pic->duration = frame->duration;
-
-        if (avctx->flags & AV_CODEC_FLAG_COPY_OPAQUE) {
-            err = av_buffer_replace(&pic->opaque_ref, frame->opaque_ref);
-            if (err < 0)
-                goto fail;
-
-            pic->opaque = frame->opaque;
-        }
-
-        av_frame_move_ref(pic->input_image, frame);
-
-        if (ctx->input_order == 0)
-            ctx->first_pts = pic->pts;
-        if (ctx->input_order == ctx->decode_delay)
-            ctx->dts_pts_diff = pic->pts - ctx->first_pts;
-        if (ctx->output_delay > 0)
-            ctx->ts_ring[ctx->input_order %
-                        (3 * ctx->output_delay + base_ctx->async_depth)] = pic->pts;
-
-        pic->display_order = ctx->input_order;
-        ++ctx->input_order;
-
-        if (ctx->pic_start) {
-            ctx->pic_end->next = pic;
-            ctx->pic_end       = pic;
-        } else {
-            ctx->pic_start     = pic;
-            ctx->pic_end       = pic;
-        }
-
-    } else {
-        ctx->end_of_stream = 1;
-
-        // Fix timestamps if we hit end-of-stream before the initial decode
-        // delay has elapsed.
-        if (ctx->input_order < ctx->decode_delay)
-            ctx->dts_pts_diff = ctx->pic_end->pts - ctx->first_pts;
-    }
-
-    return 0;
-
-fail:
-    vaapi_encode_free(avctx, pic);
-    return err;
-}
-
-int ff_vaapi_encode_receive_packet(AVCodecContext *avctx, AVPacket *pkt)
-{
-    VAAPIEncodeContext *ctx = avctx->priv_data;
-    VAAPIEncodePicture *pic = NULL;
-    AVFrame *frame = ctx->frame;
-    int err;
-
-start:
-    /** if no B frame before repeat P frame, sent repeat P frame out. */
-    if (ctx->tail_pkt->size) {
-        for (VAAPIEncodePicture *tmp = ctx->pic_start; tmp; tmp = tmp->next) {
-            if (tmp->type == FF_HW_PICTURE_TYPE_B && tmp->pts < ctx->tail_pkt->pts)
-                break;
-            else if (!tmp->next) {
-                av_packet_move_ref(pkt, ctx->tail_pkt);
-                goto end;
-            }
-        }
-    }
-
-    err = ff_encode_get_frame(avctx, frame);
-    if (err < 0 && err != AVERROR_EOF)
-        return err;
-
-    if (err == AVERROR_EOF)
-        frame = NULL;
-
-    err = vaapi_encode_send_frame(avctx, frame);
-    if (err < 0)
-        return err;
-
-    if (!ctx->pic_start) {
-        if (ctx->end_of_stream)
-            return AVERROR_EOF;
-        else
-            return AVERROR(EAGAIN);
-    }
-
-    if (ctx->has_sync_buffer_func) {
-        if (av_fifo_can_write(ctx->encode_fifo)) {
-            err = vaapi_encode_pick_next(avctx, &pic);
-            if (!err) {
-                av_assert0(pic);
-                pic->encode_order = ctx->encode_order +
-                    av_fifo_can_read(ctx->encode_fifo);
-                err = vaapi_encode_issue(avctx, pic);
-                if (err < 0) {
-                    av_log(avctx, AV_LOG_ERROR, "Encode failed: %d.\n", err);
-                    return err;
-                }
-                av_fifo_write(ctx->encode_fifo, &pic, 1);
-            }
-        }
-
-        if (!av_fifo_can_read(ctx->encode_fifo))
-            return err;
-
-        // More frames can be buffered
-        if (av_fifo_can_write(ctx->encode_fifo) && !ctx->end_of_stream)
-            return AVERROR(EAGAIN);
-
-        av_fifo_read(ctx->encode_fifo, &pic, 1);
-        ctx->encode_order = pic->encode_order + 1;
-    } else {
-        err = vaapi_encode_pick_next(avctx, &pic);
-        if (err < 0)
-            return err;
-        av_assert0(pic);
-
-        pic->encode_order = ctx->encode_order++;
-
-        err = vaapi_encode_issue(avctx, pic);
-        if (err < 0) {
-            av_log(avctx, AV_LOG_ERROR, "Encode failed: %d.\n", err);
-            return err;
-        }
-    }
-
-    err = vaapi_encode_output(avctx, pic, pkt);
-    if (err < 0) {
-        av_log(avctx, AV_LOG_ERROR, "Output failed: %d.\n", err);
-        return err;
-    }
-
-    ctx->output_order = pic->encode_order;
-    vaapi_encode_clear_old(avctx);
-
-    /** loop to get an available pkt in encoder flushing. */
-    if (ctx->end_of_stream && !pkt->size)
-        goto start;
-
-end:
-    if (pkt->size)
-        av_log(avctx, AV_LOG_DEBUG, "Output packet: pts %"PRId64", dts %"PRId64", "
-               "size %d bytes.\n", pkt->pts, pkt->dts, pkt->size);
-
-    return 0;
-}
-
 static av_cold void vaapi_encode_add_global_param(AVCodecContext *avctx, int type,
                                                   void *buffer, size_t size)
 {
@@ -2188,7 +1632,8 @@ static av_cold int vaapi_encode_init_max_frame_size(AVCodecContext *avctx)
 
 static av_cold int vaapi_encode_init_gop_structure(AVCodecContext *avctx)
 {
-    VAAPIEncodeContext *ctx = avctx->priv_data;
+    FFHWBaseEncodeContext *base_ctx = avctx->priv_data;
+    VAAPIEncodeContext         *ctx = avctx->priv_data;
     VAStatus vas;
     VAConfigAttrib attr = { VAConfigAttribEncMaxRefFrames };
     uint32_t ref_l0, ref_l1;
@@ -2211,7 +1656,7 @@ static av_cold int vaapi_encode_init_gop_structure(AVCodecContext *avctx)
         ref_l1 = attr.value >> 16 & 0xffff;
     }
 
-    ctx->p_to_gpb = 0;
+    base_ctx->p_to_gpb = 0;
     prediction_pre_only = 0;
 
 #if VA_CHECK_VERSION(1, 9, 0)
@@ -2247,7 +1692,7 @@ static av_cold int vaapi_encode_init_gop_structure(AVCodecContext *avctx)
 
             if (attr.value & VA_PREDICTION_DIRECTION_BI_NOT_EMPTY) {
                 if (ref_l0 > 0 && ref_l1 > 0) {
-                    ctx->p_to_gpb = 1;
+                    base_ctx->p_to_gpb = 1;
                     av_log(avctx, AV_LOG_VERBOSE, "Driver does not support P-frames, "
                            "replacing them with B-frames.\n");
                 }
@@ -2259,7 +1704,7 @@ static av_cold int vaapi_encode_init_gop_structure(AVCodecContext *avctx)
     if (ctx->codec->flags & FF_HW_FLAG_INTRA_ONLY ||
         avctx->gop_size <= 1) {
         av_log(avctx, AV_LOG_VERBOSE, "Using intra frames only.\n");
-        ctx->gop_size = 1;
+        base_ctx->gop_size = 1;
     } else if (ref_l0 < 1) {
         av_log(avctx, AV_LOG_ERROR, "Driver does not support any "
                "reference frames.\n");
@@ -2267,41 +1712,41 @@ static av_cold int vaapi_encode_init_gop_structure(AVCodecContext *avctx)
     } else if (!(ctx->codec->flags & FF_HW_FLAG_B_PICTURES) ||
                ref_l1 < 1 || avctx->max_b_frames < 1 ||
                prediction_pre_only) {
-        if (ctx->p_to_gpb)
+        if (base_ctx->p_to_gpb)
            av_log(avctx, AV_LOG_VERBOSE, "Using intra and B-frames "
                   "(supported references: %d / %d).\n",
                   ref_l0, ref_l1);
         else
             av_log(avctx, AV_LOG_VERBOSE, "Using intra and P-frames "
                    "(supported references: %d / %d).\n", ref_l0, ref_l1);
-        ctx->gop_size = avctx->gop_size;
-        ctx->p_per_i  = INT_MAX;
-        ctx->b_per_p  = 0;
+        base_ctx->gop_size = avctx->gop_size;
+        base_ctx->p_per_i  = INT_MAX;
+        base_ctx->b_per_p  = 0;
     } else {
-       if (ctx->p_to_gpb)
+       if (base_ctx->p_to_gpb)
            av_log(avctx, AV_LOG_VERBOSE, "Using intra and B-frames "
                   "(supported references: %d / %d).\n",
                   ref_l0, ref_l1);
        else
            av_log(avctx, AV_LOG_VERBOSE, "Using intra, P- and B-frames "
                   "(supported references: %d / %d).\n", ref_l0, ref_l1);
-        ctx->gop_size = avctx->gop_size;
-        ctx->p_per_i  = INT_MAX;
-        ctx->b_per_p  = avctx->max_b_frames;
+        base_ctx->gop_size = avctx->gop_size;
+        base_ctx->p_per_i  = INT_MAX;
+        base_ctx->b_per_p  = avctx->max_b_frames;
         if (ctx->codec->flags & FF_HW_FLAG_B_PICTURE_REFERENCES) {
-            ctx->max_b_depth = FFMIN(ctx->desired_b_depth,
-                                     av_log2(ctx->b_per_p) + 1);
+            base_ctx->max_b_depth = FFMIN(ctx->desired_b_depth,
+                                          av_log2(base_ctx->b_per_p) + 1);
         } else {
-            ctx->max_b_depth = 1;
+            base_ctx->max_b_depth = 1;
         }
     }
 
     if (ctx->codec->flags & FF_HW_FLAG_NON_IDR_KEY_PICTURES) {
-        ctx->closed_gop  = !!(avctx->flags & AV_CODEC_FLAG_CLOSED_GOP);
-        ctx->gop_per_idr = ctx->idr_interval + 1;
+        base_ctx->closed_gop  = !!(avctx->flags & AV_CODEC_FLAG_CLOSED_GOP);
+        base_ctx->gop_per_idr = ctx->idr_interval + 1;
     } else {
-        ctx->closed_gop  = 1;
-        ctx->gop_per_idr = 1;
+        base_ctx->closed_gop  = 1;
+        base_ctx->gop_per_idr = 1;
     }
 
     return 0;
@@ -2614,7 +2059,8 @@ static av_cold int vaapi_encode_init_quality(AVCodecContext *avctx)
 static av_cold int vaapi_encode_init_roi(AVCodecContext *avctx)
 {
 #if VA_CHECK_VERSION(1, 0, 0)
-    VAAPIEncodeContext *ctx = avctx->priv_data;
+    FFHWBaseEncodeContext *base_ctx = avctx->priv_data;
+    VAAPIEncodeContext         *ctx = avctx->priv_data;
     VAStatus vas;
     VAConfigAttrib attr = { VAConfigAttribEncROI };
 
@@ -2629,14 +2075,14 @@ static av_cold int vaapi_encode_init_roi(AVCodecContext *avctx)
     }
 
     if (attr.value == VA_ATTRIB_NOT_SUPPORTED) {
-        ctx->roi_allowed = 0;
+        base_ctx->roi_allowed = 0;
     } else {
         VAConfigAttribValEncROI roi = {
             .value = attr.value,
         };
 
         ctx->roi_max_regions = roi.bits.num_roi_regions;
-        ctx->roi_allowed = ctx->roi_max_regions > 0 &&
+        base_ctx->roi_allowed = ctx->roi_max_regions > 0 &&
             (ctx->va_rc_mode == VA_RC_CQP ||
              roi.bits.roi_rc_qp_delta_support);
     }
@@ -2771,6 +2217,16 @@ static av_cold int vaapi_encode_create_recon_frames(AVCodecContext *avctx)
     return err;
 }
 
+static const FFHWEncodePictureOperation vaapi_op = {
+    .alloc  = &vaapi_encode_alloc,
+
+    .issue  = &vaapi_encode_issue,
+
+    .output = &vaapi_encode_output,
+
+    .free   = &vaapi_encode_free,
+};
+
 av_cold int ff_vaapi_encode_init(AVCodecContext *avctx)
 {
     FFHWBaseEncodeContext *base_ctx = avctx->priv_data;
@@ -2782,10 +2238,12 @@ av_cold int ff_vaapi_encode_init(AVCodecContext *avctx)
     ctx->va_config  = VA_INVALID_ID;
     ctx->va_context = VA_INVALID_ID;
 
+    base_ctx->op = &vaapi_op;
+
     /* If you add something that can fail above this av_frame_alloc(),
      * modify ff_vaapi_encode_close() accordingly. */
-    ctx->frame = av_frame_alloc();
-    if (!ctx->frame) {
+    base_ctx->frame = av_frame_alloc();
+    if (!base_ctx->frame) {
         return AVERROR(ENOMEM);
     }
 
@@ -2810,8 +2268,8 @@ av_cold int ff_vaapi_encode_init(AVCodecContext *avctx)
     ctx->device = (AVHWDeviceContext*)ctx->device_ref->data;
     ctx->hwctx = ctx->device->hwctx;
 
-    ctx->tail_pkt = av_packet_alloc();
-    if (!ctx->tail_pkt) {
+    base_ctx->tail_pkt = av_packet_alloc();
+    if (!base_ctx->tail_pkt) {
         err = AVERROR(ENOMEM);
         goto fail;
     }
@@ -2910,8 +2368,8 @@ av_cold int ff_vaapi_encode_init(AVCodecContext *avctx)
             goto fail;
     }
 
-    ctx->output_delay = ctx->b_per_p;
-    ctx->decode_delay = ctx->max_b_depth;
+    base_ctx->output_delay = base_ctx->b_per_p;
+    base_ctx->decode_delay = base_ctx->max_b_depth;
 
     if (ctx->codec->sequence_params_size > 0) {
         ctx->codec_sequence_params =
@@ -2966,11 +2424,11 @@ av_cold int ff_vaapi_encode_init(AVCodecContext *avctx)
     // check vaSyncBuffer function
     vas = vaSyncBuffer(ctx->hwctx->display, VA_INVALID_ID, 0);
     if (vas != VA_STATUS_ERROR_UNIMPLEMENTED) {
-        ctx->has_sync_buffer_func = 1;
-        ctx->encode_fifo = av_fifo_alloc2(base_ctx->async_depth,
-                                          sizeof(VAAPIEncodePicture *),
-                                          0);
-        if (!ctx->encode_fifo)
+        base_ctx->async_encode = 1;
+        base_ctx->encode_fifo = av_fifo_alloc2(base_ctx->async_depth,
+                                               sizeof(VAAPIEncodePicture*),
+                                               0);
+        if (!base_ctx->encode_fifo)
             return AVERROR(ENOMEM);
     }
 #endif
@@ -2983,15 +2441,16 @@ fail:
 
 av_cold int ff_vaapi_encode_close(AVCodecContext *avctx)
 {
-    VAAPIEncodeContext *ctx = avctx->priv_data;
-    VAAPIEncodePicture *pic, *next;
+    FFHWBaseEncodeContext *base_ctx = avctx->priv_data;
+    VAAPIEncodeContext         *ctx = avctx->priv_data;
+    FFHWBaseEncodePicture *pic, *next;
 
     /* We check ctx->frame to know whether ff_vaapi_encode_init()
      * has been called and va_config/va_context initialized. */
-    if (!ctx->frame)
+    if (!base_ctx->frame)
         return 0;
 
-    for (pic = ctx->pic_start; pic; pic = next) {
+    for (pic = base_ctx->pic_start; pic; pic = next) {
         next = pic->next;
         vaapi_encode_free(avctx, pic);
     }
@@ -3008,12 +2467,12 @@ av_cold int ff_vaapi_encode_close(AVCodecContext *avctx)
         ctx->va_config = VA_INVALID_ID;
     }
 
-    av_frame_free(&ctx->frame);
-    av_packet_free(&ctx->tail_pkt);
+    av_frame_free(&base_ctx->frame);
+    av_packet_free(&base_ctx->tail_pkt);
 
     av_freep(&ctx->codec_sequence_params);
     av_freep(&ctx->codec_picture_params);
-    av_fifo_freep2(&ctx->encode_fifo);
+    av_fifo_freep2(&base_ctx->encode_fifo);
 
     av_buffer_unref(&ctx->recon_frames_ref);
     av_buffer_unref(&ctx->input_frames_ref);
diff --git a/libavcodec/vaapi_encode.h b/libavcodec/vaapi_encode.h
index 9fdb945b18..5e442d9c23 100644
--- a/libavcodec/vaapi_encode.h
+++ b/libavcodec/vaapi_encode.h
@@ -29,7 +29,6 @@
 
 #include "libavutil/hwcontext.h"
 #include "libavutil/hwcontext_vaapi.h"
-#include "libavutil/fifo.h"
 
 #include "avcodec.h"
 #include "hwconfig.h"
@@ -64,16 +63,7 @@ typedef struct VAAPIEncodeSlice {
 } VAAPIEncodeSlice;
 
 typedef struct VAAPIEncodePicture {
-    struct VAAPIEncodePicture *next;
-
-    int64_t         display_order;
-    int64_t         encode_order;
-    int64_t         pts;
-    int64_t         duration;
-    int             force_idr;
-
-    void           *opaque;
-    AVBufferRef    *opaque_ref;
+    FFHWBaseEncodePicture base;
 
 #if VA_CHECK_VERSION(1, 0, 0)
     // ROI regions.
@@ -82,15 +72,7 @@ typedef struct VAAPIEncodePicture {
     void           *roi;
 #endif
 
-    int             type;
-    int             b_depth;
-    int             encode_issued;
-    int             encode_complete;
-
-    AVFrame        *input_image;
     VASurfaceID     input_surface;
-
-    AVFrame        *recon_image;
     VASurfaceID     recon_surface;
 
     int          nb_param_buffers;
@@ -100,31 +82,8 @@ typedef struct VAAPIEncodePicture {
     VABufferID     *output_buffer_ref;
     VABufferID      output_buffer;
 
-    void           *priv_data;
     void           *codec_picture_params;
 
-    // Whether this picture is a reference picture.
-    int             is_reference;
-
-    // The contents of the DPB after this picture has been decoded.
-    // This will contain the picture itself if it is a reference picture,
-    // but not if it isn't.
-    int                     nb_dpb_pics;
-    struct VAAPIEncodePicture *dpb[MAX_DPB_SIZE];
-    // The reference pictures used in decoding this picture. If they are
-    // used by later pictures they will also appear in the DPB. ref[0][] for
-    // previous reference frames. ref[1][] for future reference frames.
-    int                     nb_refs[MAX_REFERENCE_LIST_NUM];
-    struct VAAPIEncodePicture *refs[MAX_REFERENCE_LIST_NUM][MAX_PICTURE_REFERENCES];
-    // The previous reference picture in encode order.  Must be in at least
-    // one of the reference list and DPB list.
-    struct VAAPIEncodePicture *prev;
-    // Reference count for other pictures referring to this one through
-    // the above pointers, directly from incomplete pictures and indirectly
-    // through completed pictures.
-    int             ref_count[2];
-    int             ref_removed[2];
-
     int          nb_slices;
     VAAPIEncodeSlice *slices;
 
@@ -298,30 +257,6 @@ typedef struct VAAPIEncodeContext {
     // structure (VAEncPictureParameterBuffer*).
     void           *codec_picture_params;
 
-    // Current encoding window, in display (input) order.
-    VAAPIEncodePicture *pic_start, *pic_end;
-    // The next picture to use as the previous reference picture in
-    // encoding order. Order from small to large in encoding order.
-    VAAPIEncodePicture *next_prev[MAX_PICTURE_REFERENCES];
-    int                 nb_next_prev;
-
-    // Next input order index (display order).
-    int64_t         input_order;
-    // Number of frames that output is behind input.
-    int64_t         output_delay;
-    // Next encode order index.
-    int64_t         encode_order;
-    // Number of frames decode output will need to be delayed.
-    int64_t         decode_delay;
-    // Next output order index (in encode order).
-    int64_t         output_order;
-
-    // Timestamp handling.
-    int64_t         first_pts;
-    int64_t         dts_pts_diff;
-    int64_t         ts_ring[MAX_REORDER_DELAY * 3 +
-                            MAX_ASYNC_DEPTH];
-
     // Slice structure.
     int slice_block_rows;
     int slice_block_cols;
@@ -340,41 +275,12 @@ typedef struct VAAPIEncodeContext {
     // Location of the i-th tile row boundary.
     int row_bd[MAX_TILE_ROWS + 1];
 
-    // Frame type decision.
-    int gop_size;
-    int closed_gop;
-    int gop_per_idr;
-    int p_per_i;
-    int max_b_depth;
-    int b_per_p;
-    int force_idr;
-    int idr_counter;
-    int gop_counter;
-    int end_of_stream;
-    int p_to_gpb;
-
-    // Whether the driver supports ROI at all.
-    int             roi_allowed;
     // Maximum number of regions supported by the driver.
     int             roi_max_regions;
     // Quantisation range for offset calculations.  Set by codec-specific
     // code, as it may change based on parameters.
     int             roi_quant_range;
 
-    // The encoder does not support cropping information, so warn about
-    // it the first time we encounter any nonzero crop fields.
-    int             crop_warned;
-    // If the driver does not support ROI then warn the first time we
-    // encounter a frame with ROI side data.
-    int             roi_warned;
-
-    AVFrame         *frame;
-
-    // Whether the driver support vaSyncBuffer
-    int             has_sync_buffer_func;
-    // Store buffered pic
-    AVFifo          *encode_fifo;
-
     /** Head data for current output pkt, used only for AV1. */
     //void  *header_data;
     //size_t header_data_size;
@@ -384,9 +290,6 @@ typedef struct VAAPIEncodeContext {
      * This is a RefStruct reference.
      */
     VABufferID     *coded_buffer_ref;
-
-    /** Tail data of a pic, now only used for av1 repeat frame header. */
-    AVPacket        *tail_pkt;
 } VAAPIEncodeContext;
 
 typedef struct VAAPIEncodeType {
@@ -468,9 +371,6 @@ typedef struct VAAPIEncodeType {
                                  char *data, size_t *data_len);
 } VAAPIEncodeType;
 
-
-int ff_vaapi_encode_receive_packet(AVCodecContext *avctx, AVPacket *pkt);
-
 int ff_vaapi_encode_init(AVCodecContext *avctx);
 int ff_vaapi_encode_close(AVCodecContext *avctx);
 
diff --git a/libavcodec/vaapi_encode_av1.c b/libavcodec/vaapi_encode_av1.c
index 5bd12f9f92..1c9235acd9 100644
--- a/libavcodec/vaapi_encode_av1.c
+++ b/libavcodec/vaapi_encode_av1.c
@@ -360,6 +360,7 @@ static int vaapi_encode_av1_write_sequence_header(AVCodecContext *avctx,
 
 static int vaapi_encode_av1_init_sequence_params(AVCodecContext *avctx)
 {
+    FFHWBaseEncodeContext       *base_ctx = avctx->priv_data;
     VAAPIEncodeContext               *ctx = avctx->priv_data;
     VAAPIEncodeAV1Context           *priv = avctx->priv_data;
     AV1RawOBU                     *sh_obu = &priv->sh;
@@ -441,8 +442,8 @@ static int vaapi_encode_av1_init_sequence_params(AVCodecContext *avctx)
     vseq->seq_level_idx           = sh->seq_level_idx[0];
     vseq->seq_tier                = sh->seq_tier[0];
     vseq->order_hint_bits_minus_1 = sh->order_hint_bits_minus_1;
-    vseq->intra_period            = ctx->gop_size;
-    vseq->ip_period               = ctx->b_per_p + 1;
+    vseq->intra_period            = base_ctx->gop_size;
+    vseq->ip_period               = base_ctx->b_per_p + 1;
 
     vseq->seq_fields.bits.enable_order_hint = sh->enable_order_hint;
 
@@ -465,16 +466,17 @@ end:
 }
 
 static int vaapi_encode_av1_init_picture_params(AVCodecContext *avctx,
-                                                VAAPIEncodePicture *pic)
+                                                VAAPIEncodePicture *vaapi_pic)
 {
     VAAPIEncodeContext              *ctx = avctx->priv_data;
     VAAPIEncodeAV1Context          *priv = avctx->priv_data;
+    const FFHWBaseEncodePicture     *pic = &vaapi_pic->base;
     VAAPIEncodeAV1Picture          *hpic = pic->priv_data;
     AV1RawOBU                    *fh_obu = &priv->fh;
     AV1RawFrameHeader                *fh = &fh_obu->obu.frame.header;
-    VAEncPictureParameterBufferAV1 *vpic = pic->codec_picture_params;
+    VAEncPictureParameterBufferAV1 *vpic = vaapi_pic->codec_picture_params;
     CodedBitstreamFragment          *obu = &priv->current_obu;
-    VAAPIEncodePicture    *ref;
+    FFHWBaseEncodePicture *ref;
     VAAPIEncodeAV1Picture *href;
     int slot, i;
     int ret;
@@ -482,8 +484,8 @@ static int vaapi_encode_av1_init_picture_params(AVCodecContext *avctx,
         { 1, 0, 0, 0, -1, 0, -1, -1 };
 
     memset(fh_obu, 0, sizeof(*fh_obu));
-    pic->nb_slices = priv->tile_groups;
-    pic->non_independent_frame = pic->encode_order < pic->display_order;
+    vaapi_pic->nb_slices = priv->tile_groups;
+    vaapi_pic->non_independent_frame = pic->encode_order < pic->display_order;
     fh_obu->header.obu_type = AV1_OBU_FRAME_HEADER;
     fh_obu->header.obu_has_size_field = 1;
 
@@ -601,8 +603,8 @@ static int vaapi_encode_av1_init_picture_params(AVCodecContext *avctx,
     vpic->frame_width_minus_1  = fh->frame_width_minus_1;
     vpic->frame_height_minus_1 = fh->frame_height_minus_1;
     vpic->primary_ref_frame    = fh->primary_ref_frame;
-    vpic->reconstructed_frame  = pic->recon_surface;
-    vpic->coded_buf            = pic->output_buffer;
+    vpic->reconstructed_frame  = vaapi_pic->recon_surface;
+    vpic->coded_buf            = vaapi_pic->output_buffer;
     vpic->tile_cols            = fh->tile_cols;
     vpic->tile_rows            = fh->tile_rows;
     vpic->order_hint           = fh->order_hint;
@@ -630,12 +632,12 @@ static int vaapi_encode_av1_init_picture_params(AVCodecContext *avctx,
 
     for (i = 0; i < MAX_REFERENCE_LIST_NUM; i++) {
         for (int j = 0; j < pic->nb_refs[i]; j++) {
-            VAAPIEncodePicture *ref_pic = pic->refs[i][j];
+            FFHWBaseEncodePicture *ref_pic = pic->refs[i][j];
 
             slot = ((VAAPIEncodeAV1Picture*)ref_pic->priv_data)->slot;
             av_assert0(vpic->reference_frames[slot] == VA_INVALID_SURFACE);
 
-            vpic->reference_frames[slot] = ref_pic->recon_surface;
+            vpic->reference_frames[slot] = ((VAAPIEncodePicture *)ref_pic)->recon_surface;
         }
     }
 
@@ -752,7 +754,7 @@ static int vaapi_encode_av1_init_slice_params(AVCodecContext *avctx,
 }
 
 static int vaapi_encode_av1_write_picture_header(AVCodecContext *avctx,
-                                                 VAAPIEncodePicture *pic,
+                                                 VAAPIEncodePicture *vaapi_pic,
                                                  char *data, size_t *data_len)
 {
     VAAPIEncodeAV1Context     *priv = avctx->priv_data;
@@ -760,10 +762,11 @@ static int vaapi_encode_av1_write_picture_header(AVCodecContext *avctx,
     CodedBitstreamAV1Context *cbctx = priv->cbc->priv_data;
     AV1RawOBU               *fh_obu = &priv->fh;
     AV1RawFrameHeader       *rep_fh = &fh_obu->obu.frame_header;
+    const FFHWBaseEncodePicture *pic = &vaapi_pic->base;
     VAAPIEncodeAV1Picture *href;
     int ret = 0;
 
-    pic->tail_size = 0;
+    vaapi_pic->tail_size = 0;
     /** Pack repeat frame header. */
     if (pic->display_order > pic->encode_order) {
         memset(fh_obu, 0, sizeof(*fh_obu));
@@ -785,11 +788,11 @@ static int vaapi_encode_av1_write_picture_header(AVCodecContext *avctx,
         if (ret < 0)
             goto end;
 
-        ret = vaapi_encode_av1_write_obu(avctx, pic->tail_data, &pic->tail_size, obu);
+        ret = vaapi_encode_av1_write_obu(avctx, vaapi_pic->tail_data, &vaapi_pic->tail_size, obu);
         if (ret < 0)
             goto end;
 
-        pic->tail_size /= 8;
+        vaapi_pic->tail_size /= 8;
     }
 
     memcpy(data, &priv->fh_data, MAX_PARAM_BUFFER_SIZE * sizeof(char));
@@ -1038,7 +1041,7 @@ const FFCodec ff_av1_vaapi_encoder = {
     .p.id           = AV_CODEC_ID_AV1,
     .priv_data_size = sizeof(VAAPIEncodeAV1Context),
     .init           = &vaapi_encode_av1_init,
-    FF_CODEC_RECEIVE_PACKET_CB(&ff_vaapi_encode_receive_packet),
+    FF_CODEC_RECEIVE_PACKET_CB(&ff_hw_base_encode_receive_packet),
     .close          = &vaapi_encode_av1_close,
     .p.priv_class   = &vaapi_encode_av1_class,
     .p.capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE |
diff --git a/libavcodec/vaapi_encode_h264.c b/libavcodec/vaapi_encode_h264.c
index aa5b499d9f..2f91655d18 100644
--- a/libavcodec/vaapi_encode_h264.c
+++ b/libavcodec/vaapi_encode_h264.c
@@ -233,7 +233,7 @@ static int vaapi_encode_h264_write_extra_header(AVCodecContext *avctx,
                 goto fail;
         }
         if (priv->sei_needed & SEI_TIMING) {
-            if (pic->type == FF_HW_PICTURE_TYPE_IDR) {
+            if (pic->base.type == FF_HW_PICTURE_TYPE_IDR) {
                 err = ff_cbs_sei_add_message(priv->cbc, au, 1,
                                              SEI_TYPE_BUFFERING_PERIOD,
                                              &priv->sei_buffering_period, NULL);
@@ -295,6 +295,7 @@ fail:
 
 static int vaapi_encode_h264_init_sequence_params(AVCodecContext *avctx)
 {
+    FFHWBaseEncodeContext        *base_ctx = avctx->priv_data;
     VAAPIEncodeContext                *ctx = avctx->priv_data;
     VAAPIEncodeH264Context           *priv = avctx->priv_data;
     H264RawSPS                        *sps = &priv->raw_sps;
@@ -326,18 +327,18 @@ static int vaapi_encode_h264_init_sequence_params(AVCodecContext *avctx)
         sps->constraint_set1_flag = 1;
 
     if (avctx->profile == AV_PROFILE_H264_HIGH || avctx->profile == AV_PROFILE_H264_HIGH_10)
-        sps->constraint_set3_flag = ctx->gop_size == 1;
+        sps->constraint_set3_flag = base_ctx->gop_size == 1;
 
     if (avctx->profile == AV_PROFILE_H264_MAIN ||
         avctx->profile == AV_PROFILE_H264_HIGH || avctx->profile == AV_PROFILE_H264_HIGH_10) {
         sps->constraint_set4_flag = 1;
-        sps->constraint_set5_flag = ctx->b_per_p == 0;
+        sps->constraint_set5_flag = base_ctx->b_per_p == 0;
     }
 
-    if (ctx->gop_size == 1)
+    if (base_ctx->gop_size == 1)
         priv->dpb_frames = 0;
     else
-        priv->dpb_frames = 1 + ctx->max_b_depth;
+        priv->dpb_frames = 1 + base_ctx->max_b_depth;
 
     if (avctx->level != AV_LEVEL_UNKNOWN) {
         sps->level_idc = avctx->level;
@@ -374,7 +375,7 @@ static int vaapi_encode_h264_init_sequence_params(AVCodecContext *avctx)
     sps->bit_depth_chroma_minus8 = bit_depth - 8;
 
     sps->log2_max_frame_num_minus4 = 4;
-    sps->pic_order_cnt_type        = ctx->max_b_depth ? 0 : 2;
+    sps->pic_order_cnt_type        = base_ctx->max_b_depth ? 0 : 2;
     if (sps->pic_order_cnt_type == 0) {
         sps->log2_max_pic_order_cnt_lsb_minus4 = 4;
     }
@@ -501,8 +502,8 @@ static int vaapi_encode_h264_init_sequence_params(AVCodecContext *avctx)
     sps->vui.motion_vectors_over_pic_boundaries_flag = 1;
     sps->vui.log2_max_mv_length_horizontal = 15;
     sps->vui.log2_max_mv_length_vertical   = 15;
-    sps->vui.max_num_reorder_frames        = ctx->max_b_depth;
-    sps->vui.max_dec_frame_buffering       = ctx->max_b_depth + 1;
+    sps->vui.max_num_reorder_frames        = base_ctx->max_b_depth;
+    sps->vui.max_dec_frame_buffering       = base_ctx->max_b_depth + 1;
 
     pps->nal_unit_header.nal_ref_idc = 3;
     pps->nal_unit_header.nal_unit_type = H264_NAL_PPS;
@@ -535,9 +536,9 @@ static int vaapi_encode_h264_init_sequence_params(AVCodecContext *avctx)
     *vseq = (VAEncSequenceParameterBufferH264) {
         .seq_parameter_set_id = sps->seq_parameter_set_id,
         .level_idc        = sps->level_idc,
-        .intra_period     = ctx->gop_size,
-        .intra_idr_period = ctx->gop_size,
-        .ip_period        = ctx->b_per_p + 1,
+        .intra_period     = base_ctx->gop_size,
+        .intra_idr_period = base_ctx->gop_size,
+        .ip_period        = base_ctx->b_per_p + 1,
 
         .bits_per_second       = ctx->va_bit_rate,
         .max_num_ref_frames    = sps->max_num_ref_frames,
@@ -619,14 +620,15 @@ static int vaapi_encode_h264_init_sequence_params(AVCodecContext *avctx)
 }
 
 static int vaapi_encode_h264_init_picture_params(AVCodecContext *avctx,
-                                                 VAAPIEncodePicture *pic)
+                                                 VAAPIEncodePicture *vaapi_pic)
 {
-    VAAPIEncodeContext               *ctx = avctx->priv_data;
+    FFHWBaseEncodeContext       *base_ctx = avctx->priv_data;
     VAAPIEncodeH264Context          *priv = avctx->priv_data;
+    const FFHWBaseEncodePicture      *pic = &vaapi_pic->base;
     VAAPIEncodeH264Picture          *hpic = pic->priv_data;
-    VAAPIEncodePicture              *prev = pic->prev;
+    FFHWBaseEncodePicture           *prev = pic->prev;
     VAAPIEncodeH264Picture         *hprev = prev ? prev->priv_data : NULL;
-    VAEncPictureParameterBufferH264 *vpic = pic->codec_picture_params;
+    VAEncPictureParameterBufferH264 *vpic = vaapi_pic->codec_picture_params;
     int i, j = 0;
 
     if (pic->type == FF_HW_PICTURE_TYPE_IDR) {
@@ -662,7 +664,7 @@ static int vaapi_encode_h264_init_picture_params(AVCodecContext *avctx,
         hpic->pic_order_cnt *= 2;
     }
 
-    hpic->dpb_delay     = pic->display_order - pic->encode_order + ctx->max_b_depth;
+    hpic->dpb_delay     = pic->display_order - pic->encode_order + base_ctx->max_b_depth;
     hpic->cpb_delay     = pic->encode_order - hpic->last_idr_frame;
 
     if (priv->aud) {
@@ -699,7 +701,7 @@ static int vaapi_encode_h264_init_picture_params(AVCodecContext *avctx,
         priv->sei_recovery_point = (H264RawSEIRecoveryPoint) {
             .recovery_frame_cnt = 0,
             .exact_match_flag   = 1,
-            .broken_link_flag   = ctx->b_per_p > 0,
+            .broken_link_flag   = base_ctx->b_per_p > 0,
         };
 
         priv->sei_needed |= SEI_RECOVERY_POINT;
@@ -722,7 +724,7 @@ static int vaapi_encode_h264_init_picture_params(AVCodecContext *avctx,
     }
 
     vpic->CurrPic = (VAPictureH264) {
-        .picture_id          = pic->recon_surface,
+        .picture_id          = vaapi_pic->recon_surface,
         .frame_idx           = hpic->frame_num,
         .flags               = 0,
         .TopFieldOrderCnt    = hpic->pic_order_cnt,
@@ -730,14 +732,14 @@ static int vaapi_encode_h264_init_picture_params(AVCodecContext *avctx,
     };
     for (int k = 0; k < MAX_REFERENCE_LIST_NUM; k++) {
         for (i = 0; i < pic->nb_refs[k]; i++) {
-            VAAPIEncodePicture      *ref = pic->refs[k][i];
+            FFHWBaseEncodePicture *ref = pic->refs[k][i];
             VAAPIEncodeH264Picture *href;
 
             av_assert0(ref && ref->encode_order < pic->encode_order);
             href = ref->priv_data;
 
             vpic->ReferenceFrames[j++] = (VAPictureH264) {
-                .picture_id          = ref->recon_surface,
+                .picture_id          = ((VAAPIEncodePicture *)ref)->recon_surface,
                 .frame_idx           = href->frame_num,
                 .flags               = VA_PICTURE_H264_SHORT_TERM_REFERENCE,
                 .TopFieldOrderCnt    = href->pic_order_cnt,
@@ -753,7 +755,7 @@ static int vaapi_encode_h264_init_picture_params(AVCodecContext *avctx,
         };
     }
 
-    vpic->coded_buf = pic->output_buffer;
+    vpic->coded_buf = vaapi_pic->output_buffer;
 
     vpic->frame_num = hpic->frame_num;
 
@@ -764,12 +766,13 @@ static int vaapi_encode_h264_init_picture_params(AVCodecContext *avctx,
 }
 
 static void vaapi_encode_h264_default_ref_pic_list(AVCodecContext *avctx,
-                                                   VAAPIEncodePicture *pic,
+                                                   VAAPIEncodePicture *vaapi_pic,
                                                    VAAPIEncodePicture **rpl0,
                                                    VAAPIEncodePicture **rpl1,
                                                    int *rpl_size)
 {
-    VAAPIEncodePicture *prev;
+    FFHWBaseEncodePicture *pic = &vaapi_pic->base;
+    FFHWBaseEncodePicture *prev;
     VAAPIEncodeH264Picture *hp, *hn, *hc;
     int i, j, n = 0;
 
@@ -783,17 +786,17 @@ static void vaapi_encode_h264_default_ref_pic_list(AVCodecContext *avctx,
 
         if (pic->type == FF_HW_PICTURE_TYPE_P) {
             for (j = n; j > 0; j--) {
-                hc = rpl0[j - 1]->priv_data;
+                hc = rpl0[j - 1]->base.priv_data;
                 av_assert0(hc->frame_num != hn->frame_num);
                 if (hc->frame_num > hn->frame_num)
                     break;
                 rpl0[j] = rpl0[j - 1];
             }
-            rpl0[j] = prev->dpb[i];
+            rpl0[j] = (VAAPIEncodePicture *)prev->dpb[i];
 
         } else if (pic->type == FF_HW_PICTURE_TYPE_B) {
             for (j = n; j > 0; j--) {
-                hc = rpl0[j - 1]->priv_data;
+                hc = rpl0[j - 1]->base.priv_data;
                 av_assert0(hc->pic_order_cnt != hp->pic_order_cnt);
                 if (hc->pic_order_cnt < hp->pic_order_cnt) {
                     if (hn->pic_order_cnt > hp->pic_order_cnt ||
@@ -805,10 +808,10 @@ static void vaapi_encode_h264_default_ref_pic_list(AVCodecContext *avctx,
                 }
                 rpl0[j] = rpl0[j - 1];
             }
-            rpl0[j] = prev->dpb[i];
+            rpl0[j] = (VAAPIEncodePicture *)prev->dpb[i];
 
             for (j = n; j > 0; j--) {
-                hc = rpl1[j - 1]->priv_data;
+                hc = rpl1[j - 1]->base.priv_data;
                 av_assert0(hc->pic_order_cnt != hp->pic_order_cnt);
                 if (hc->pic_order_cnt > hp->pic_order_cnt) {
                     if (hn->pic_order_cnt < hp->pic_order_cnt ||
@@ -820,7 +823,7 @@ static void vaapi_encode_h264_default_ref_pic_list(AVCodecContext *avctx,
                 }
                 rpl1[j] = rpl1[j - 1];
             }
-            rpl1[j] = prev->dpb[i];
+            rpl1[j] = (VAAPIEncodePicture *)prev->dpb[i];
         }
 
         ++n;
@@ -840,7 +843,7 @@ static void vaapi_encode_h264_default_ref_pic_list(AVCodecContext *avctx,
         av_log(avctx, AV_LOG_DEBUG, "Default RefPicList0 for fn=%d/poc=%d:",
                hp->frame_num, hp->pic_order_cnt);
         for (i = 0; i < n; i++) {
-            hn = rpl0[i]->priv_data;
+            hn = rpl0[i]->base.priv_data;
             av_log(avctx, AV_LOG_DEBUG, "  fn=%d/poc=%d",
                    hn->frame_num, hn->pic_order_cnt);
         }
@@ -850,7 +853,7 @@ static void vaapi_encode_h264_default_ref_pic_list(AVCodecContext *avctx,
         av_log(avctx, AV_LOG_DEBUG, "Default RefPicList1 for fn=%d/poc=%d:",
                hp->frame_num, hp->pic_order_cnt);
         for (i = 0; i < n; i++) {
-            hn = rpl1[i]->priv_data;
+            hn = rpl1[i]->base.priv_data;
             av_log(avctx, AV_LOG_DEBUG, "  fn=%d/poc=%d",
                    hn->frame_num, hn->pic_order_cnt);
         }
@@ -861,16 +864,17 @@ static void vaapi_encode_h264_default_ref_pic_list(AVCodecContext *avctx,
 }
 
 static int vaapi_encode_h264_init_slice_params(AVCodecContext *avctx,
-                                               VAAPIEncodePicture *pic,
+                                               VAAPIEncodePicture *vaapi_pic,
                                                VAAPIEncodeSlice *slice)
 {
     VAAPIEncodeH264Context          *priv = avctx->priv_data;
+    const FFHWBaseEncodePicture      *pic = &vaapi_pic->base;
     VAAPIEncodeH264Picture          *hpic = pic->priv_data;
-    VAAPIEncodePicture              *prev = pic->prev;
+    FFHWBaseEncodePicture           *prev = pic->prev;
     H264RawSPS                       *sps = &priv->raw_sps;
     H264RawPPS                       *pps = &priv->raw_pps;
     H264RawSliceHeader                *sh = &priv->raw_slice.header;
-    VAEncPictureParameterBufferH264 *vpic = pic->codec_picture_params;
+    VAEncPictureParameterBufferH264 *vpic = vaapi_pic->codec_picture_params;
     VAEncSliceParameterBufferH264 *vslice = slice->codec_slice_params;
     int i, j;
 
@@ -903,7 +907,7 @@ static int vaapi_encode_h264_init_slice_params(AVCodecContext *avctx,
         sh->slice_qp_delta = priv->fixed_qp_idr - (pps->pic_init_qp_minus26 + 26);
 
     if (pic->is_reference && pic->type != FF_HW_PICTURE_TYPE_IDR) {
-        VAAPIEncodePicture *discard_list[MAX_DPB_SIZE];
+        FFHWBaseEncodePicture *discard_list[MAX_DPB_SIZE];
         int discard = 0, keep = 0;
 
         // Discard everything which is in the DPB of the previous frame but
@@ -944,14 +948,14 @@ static int vaapi_encode_h264_init_slice_params(AVCodecContext *avctx,
         VAAPIEncodeH264Picture *href;
         int n;
 
-        vaapi_encode_h264_default_ref_pic_list(avctx, pic,
+        vaapi_encode_h264_default_ref_pic_list(avctx, vaapi_pic,
                                                def_l0, def_l1, &n);
 
         if (pic->type == FF_HW_PICTURE_TYPE_P) {
             int need_rplm = 0;
             for (i = 0; i < pic->nb_refs[0]; i++) {
                 av_assert0(pic->refs[0][i]);
-                if (pic->refs[0][i] != def_l0[i])
+                if (pic->refs[0][i] != (FFHWBaseEncodePicture *)def_l0[i])
                     need_rplm = 1;
             }
 
@@ -982,7 +986,7 @@ static int vaapi_encode_h264_init_slice_params(AVCodecContext *avctx,
                 av_assert0(pic->refs[0][i]);
                 href = pic->refs[0][i]->priv_data;
                 av_assert0(href->pic_order_cnt < hpic->pic_order_cnt);
-                if (pic->refs[0][i] != def_l0[n0])
+                if (pic->refs[0][i] != (FFHWBaseEncodePicture *)def_l0[n0])
                     need_rplm_l0 = 1;
                 ++n0;
             }
@@ -991,7 +995,7 @@ static int vaapi_encode_h264_init_slice_params(AVCodecContext *avctx,
                 av_assert0(pic->refs[1][i]);
                 href = pic->refs[1][i]->priv_data;
                 av_assert0(href->pic_order_cnt > hpic->pic_order_cnt);
-                if (pic->refs[1][i] != def_l1[n1])
+                if (pic->refs[1][i] != (FFHWBaseEncodePicture *)def_l1[n1])
                     need_rplm_l1 = 1;
                 ++n1;
             }
@@ -1380,7 +1384,7 @@ const FFCodec ff_h264_vaapi_encoder = {
     .p.id           = AV_CODEC_ID_H264,
     .priv_data_size = sizeof(VAAPIEncodeH264Context),
     .init           = &vaapi_encode_h264_init,
-    FF_CODEC_RECEIVE_PACKET_CB(&ff_vaapi_encode_receive_packet),
+    FF_CODEC_RECEIVE_PACKET_CB(&ff_hw_base_encode_receive_packet),
     .close          = &vaapi_encode_h264_close,
     .p.priv_class   = &vaapi_encode_h264_class,
     .p.capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE |
diff --git a/libavcodec/vaapi_encode_h265.c b/libavcodec/vaapi_encode_h265.c
index e65c91304a..6befe3426e 100644
--- a/libavcodec/vaapi_encode_h265.c
+++ b/libavcodec/vaapi_encode_h265.c
@@ -259,6 +259,7 @@ fail:
 
 static int vaapi_encode_h265_init_sequence_params(AVCodecContext *avctx)
 {
+    FFHWBaseEncodeContext        *base_ctx = avctx->priv_data;
     VAAPIEncodeContext                *ctx = avctx->priv_data;
     VAAPIEncodeH265Context           *priv = avctx->priv_data;
     H265RawVPS                        *vps = &priv->raw_vps;
@@ -340,7 +341,7 @@ static int vaapi_encode_h265_init_sequence_params(AVCodecContext *avctx)
     ptl->general_max_420chroma_constraint_flag  = chroma_format <= 1;
     ptl->general_max_monochrome_constraint_flag = chroma_format == 0;
 
-    ptl->general_intra_constraint_flag = ctx->gop_size == 1;
+    ptl->general_intra_constraint_flag = base_ctx->gop_size == 1;
     ptl->general_one_picture_only_constraint_flag = 0;
 
     ptl->general_lower_bit_rate_constraint_flag = 1;
@@ -353,7 +354,7 @@ static int vaapi_encode_h265_init_sequence_params(AVCodecContext *avctx)
         level = ff_h265_guess_level(ptl, avctx->bit_rate,
                                     ctx->surface_width, ctx->surface_height,
                                     ctx->nb_slices, ctx->tile_rows, ctx->tile_cols,
-                                    (ctx->b_per_p > 0) + 1);
+                                    (base_ctx->b_per_p > 0) + 1);
         if (level) {
             av_log(avctx, AV_LOG_VERBOSE, "Using level %s.\n", level->name);
             ptl->general_level_idc = level->level_idc;
@@ -367,8 +368,8 @@ static int vaapi_encode_h265_init_sequence_params(AVCodecContext *avctx)
     }
 
     vps->vps_sub_layer_ordering_info_present_flag = 0;
-    vps->vps_max_dec_pic_buffering_minus1[0]      = ctx->max_b_depth + 1;
-    vps->vps_max_num_reorder_pics[0]              = ctx->max_b_depth;
+    vps->vps_max_dec_pic_buffering_minus1[0]      = base_ctx->max_b_depth + 1;
+    vps->vps_max_num_reorder_pics[0]              = base_ctx->max_b_depth;
     vps->vps_max_latency_increase_plus1[0]        = 0;
 
     vps->vps_max_layer_id             = 0;
@@ -642,9 +643,9 @@ static int vaapi_encode_h265_init_sequence_params(AVCodecContext *avctx)
         .general_level_idc   = vps->profile_tier_level.general_level_idc,
         .general_tier_flag   = vps->profile_tier_level.general_tier_flag,
 
-        .intra_period     = ctx->gop_size,
-        .intra_idr_period = ctx->gop_size,
-        .ip_period        = ctx->b_per_p + 1,
+        .intra_period     = base_ctx->gop_size,
+        .intra_idr_period = base_ctx->gop_size,
+        .ip_period        = base_ctx->b_per_p + 1,
         .bits_per_second  = ctx->va_bit_rate,
 
         .pic_width_in_luma_samples  = sps->pic_width_in_luma_samples,
@@ -755,14 +756,15 @@ static int vaapi_encode_h265_init_sequence_params(AVCodecContext *avctx)
 }
 
 static int vaapi_encode_h265_init_picture_params(AVCodecContext *avctx,
-                                                 VAAPIEncodePicture *pic)
+                                                 VAAPIEncodePicture *vaapi_pic)
 {
-    VAAPIEncodeContext               *ctx = avctx->priv_data;
+    FFHWBaseEncodeContext       *base_ctx = avctx->priv_data;
     VAAPIEncodeH265Context          *priv = avctx->priv_data;
+    FFHWBaseEncodePicture            *pic = &vaapi_pic->base;
     VAAPIEncodeH265Picture          *hpic = pic->priv_data;
-    VAAPIEncodePicture              *prev = pic->prev;
+    FFHWBaseEncodePicture           *prev = pic->prev;
     VAAPIEncodeH265Picture         *hprev = prev ? prev->priv_data : NULL;
-    VAEncPictureParameterBufferHEVC *vpic = pic->codec_picture_params;
+    VAEncPictureParameterBufferHEVC *vpic = vaapi_pic->codec_picture_params;
     int i, j = 0;
 
     if (pic->type == FF_HW_PICTURE_TYPE_IDR) {
@@ -787,13 +789,13 @@ static int vaapi_encode_h265_init_picture_params(AVCodecContext *avctx,
             hpic->slice_type     = HEVC_SLICE_P;
             hpic->pic_type       = 1;
         } else {
-            VAAPIEncodePicture *irap_ref;
+            FFHWBaseEncodePicture *irap_ref;
             av_assert0(pic->refs[0][0] && pic->refs[1][0]);
             for (irap_ref = pic; irap_ref; irap_ref = irap_ref->refs[1][0]) {
                 if (irap_ref->type == FF_HW_PICTURE_TYPE_I)
                     break;
             }
-            if (pic->b_depth == ctx->max_b_depth) {
+            if (pic->b_depth == base_ctx->max_b_depth) {
                 hpic->slice_nal_unit = irap_ref ? HEVC_NAL_RASL_N
                                                 : HEVC_NAL_TRAIL_N;
             } else {
@@ -909,21 +911,21 @@ static int vaapi_encode_h265_init_picture_params(AVCodecContext *avctx,
     }
 
     vpic->decoded_curr_pic = (VAPictureHEVC) {
-        .picture_id    = pic->recon_surface,
+        .picture_id    = vaapi_pic->recon_surface,
         .pic_order_cnt = hpic->pic_order_cnt,
         .flags         = 0,
     };
 
     for (int k = 0; k < MAX_REFERENCE_LIST_NUM; k++) {
         for (i = 0; i < pic->nb_refs[k]; i++) {
-            VAAPIEncodePicture      *ref = pic->refs[k][i];
+            FFHWBaseEncodePicture *ref = pic->refs[k][i];
             VAAPIEncodeH265Picture *href;
 
             av_assert0(ref && ref->encode_order < pic->encode_order);
             href = ref->priv_data;
 
             vpic->reference_frames[j++] = (VAPictureHEVC) {
-                .picture_id    = ref->recon_surface,
+                .picture_id    = ((VAAPIEncodePicture *)ref)->recon_surface,
                 .pic_order_cnt = href->pic_order_cnt,
                 .flags = (ref->display_order < pic->display_order ?
                           VA_PICTURE_HEVC_RPS_ST_CURR_BEFORE : 0) |
@@ -940,7 +942,7 @@ static int vaapi_encode_h265_init_picture_params(AVCodecContext *avctx,
         };
     }
 
-    vpic->coded_buf = pic->output_buffer;
+    vpic->coded_buf = vaapi_pic->output_buffer;
 
     vpic->nal_unit_type = hpic->slice_nal_unit;
 
@@ -970,16 +972,17 @@ static int vaapi_encode_h265_init_picture_params(AVCodecContext *avctx,
 }
 
 static int vaapi_encode_h265_init_slice_params(AVCodecContext *avctx,
-                                               VAAPIEncodePicture *pic,
+                                               VAAPIEncodePicture *vaapi_pic,
                                                VAAPIEncodeSlice *slice)
 {
-    VAAPIEncodeContext                *ctx = avctx->priv_data;
+    FFHWBaseEncodeContext        *base_ctx = avctx->priv_data;
     VAAPIEncodeH265Context           *priv = avctx->priv_data;
+    const FFHWBaseEncodePicture       *pic = &vaapi_pic->base;
     VAAPIEncodeH265Picture           *hpic = pic->priv_data;
     const H265RawSPS                  *sps = &priv->raw_sps;
     const H265RawPPS                  *pps = &priv->raw_pps;
     H265RawSliceHeader                 *sh = &priv->raw_slice.header;
-    VAEncPictureParameterBufferHEVC  *vpic = pic->codec_picture_params;
+    VAEncPictureParameterBufferHEVC  *vpic = vaapi_pic->codec_picture_params;
     VAEncSliceParameterBufferHEVC  *vslice = slice->codec_slice_params;
     int i;
 
@@ -996,7 +999,7 @@ static int vaapi_encode_h265_init_slice_params(AVCodecContext *avctx,
 
     sh->slice_type = hpic->slice_type;
 
-    if (sh->slice_type == HEVC_SLICE_P && ctx->p_to_gpb)
+    if (sh->slice_type == HEVC_SLICE_P && base_ctx->p_to_gpb)
         sh->slice_type = HEVC_SLICE_B;
 
     sh->slice_pic_order_cnt_lsb = hpic->pic_order_cnt &
@@ -1140,7 +1143,7 @@ static int vaapi_encode_h265_init_slice_params(AVCodecContext *avctx,
         .slice_tc_offset_div2   = sh->slice_tc_offset_div2,
 
         .slice_fields.bits = {
-            .last_slice_of_pic_flag       = slice->index == pic->nb_slices - 1,
+            .last_slice_of_pic_flag       = slice->index == vaapi_pic->nb_slices - 1,
             .dependent_slice_segment_flag = sh->dependent_slice_segment_flag,
             .colour_plane_id              = sh->colour_plane_id,
             .slice_temporal_mvp_enabled_flag =
@@ -1171,7 +1174,7 @@ static int vaapi_encode_h265_init_slice_params(AVCodecContext *avctx,
         av_assert0(pic->type == FF_HW_PICTURE_TYPE_P ||
                    pic->type == FF_HW_PICTURE_TYPE_B);
         vslice->ref_pic_list0[0] = vpic->reference_frames[0];
-        if (ctx->p_to_gpb && pic->type == FF_HW_PICTURE_TYPE_P)
+        if (base_ctx->p_to_gpb && pic->type == FF_HW_PICTURE_TYPE_P)
             // Reference for GPB B-frame, L0 == L1
             vslice->ref_pic_list1[0] = vpic->reference_frames[0];
     }
@@ -1181,7 +1184,7 @@ static int vaapi_encode_h265_init_slice_params(AVCodecContext *avctx,
         vslice->ref_pic_list1[0] = vpic->reference_frames[1];
     }
 
-    if (pic->type == FF_HW_PICTURE_TYPE_P && ctx->p_to_gpb) {
+    if (pic->type == FF_HW_PICTURE_TYPE_P && base_ctx->p_to_gpb) {
         vslice->slice_type = HEVC_SLICE_B;
         for (i = 0; i < FF_ARRAY_ELEMS(vslice->ref_pic_list0); i++) {
             vslice->ref_pic_list1[i].picture_id = vslice->ref_pic_list0[i].picture_id;
@@ -1494,7 +1497,7 @@ const FFCodec ff_hevc_vaapi_encoder = {
     .p.id           = AV_CODEC_ID_HEVC,
     .priv_data_size = sizeof(VAAPIEncodeH265Context),
     .init           = &vaapi_encode_h265_init,
-    FF_CODEC_RECEIVE_PACKET_CB(&ff_vaapi_encode_receive_packet),
+    FF_CODEC_RECEIVE_PACKET_CB(&ff_hw_base_encode_receive_packet),
     .close          = &vaapi_encode_h265_close,
     .p.priv_class   = &vaapi_encode_h265_class,
     .p.capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE |
diff --git a/libavcodec/vaapi_encode_mjpeg.c b/libavcodec/vaapi_encode_mjpeg.c
index 9e533e9fbe..4c31881e6a 100644
--- a/libavcodec/vaapi_encode_mjpeg.c
+++ b/libavcodec/vaapi_encode_mjpeg.c
@@ -220,12 +220,13 @@ static int vaapi_encode_mjpeg_write_extra_buffer(AVCodecContext *avctx,
 }
 
 static int vaapi_encode_mjpeg_init_picture_params(AVCodecContext *avctx,
-                                                  VAAPIEncodePicture *pic)
+                                                  VAAPIEncodePicture *vaapi_pic)
 {
     VAAPIEncodeMJPEGContext         *priv = avctx->priv_data;
+    const FFHWBaseEncodePicture      *pic = &vaapi_pic->base;
     JPEGRawFrameHeader                *fh = &priv->frame_header;
     JPEGRawScanHeader                 *sh = &priv->scan.header;
-    VAEncPictureParameterBufferJPEG *vpic = pic->codec_picture_params;
+    VAEncPictureParameterBufferJPEG *vpic = vaapi_pic->codec_picture_params;
     const AVPixFmtDescriptor *desc;
     const uint8_t components_rgb[3] = { 'R', 'G', 'B' };
     const uint8_t components_yuv[3] = {  1,   2,   3  };
@@ -377,8 +378,8 @@ static int vaapi_encode_mjpeg_init_picture_params(AVCodecContext *avctx,
 
 
     *vpic = (VAEncPictureParameterBufferJPEG) {
-        .reconstructed_picture = pic->recon_surface,
-        .coded_buf             = pic->output_buffer,
+        .reconstructed_picture = vaapi_pic->recon_surface,
+        .coded_buf             = vaapi_pic->output_buffer,
 
         .picture_width  = fh->X,
         .picture_height = fh->Y,
@@ -406,7 +407,7 @@ static int vaapi_encode_mjpeg_init_picture_params(AVCodecContext *avctx,
         vpic->quantiser_table_selector[i] = fh->Tq[i];
     }
 
-    pic->nb_slices = 1;
+    vaapi_pic->nb_slices = 1;
 
     return 0;
 }
@@ -572,7 +573,7 @@ const FFCodec ff_mjpeg_vaapi_encoder = {
     .p.id           = AV_CODEC_ID_MJPEG,
     .priv_data_size = sizeof(VAAPIEncodeMJPEGContext),
     .init           = &vaapi_encode_mjpeg_init,
-    FF_CODEC_RECEIVE_PACKET_CB(&ff_vaapi_encode_receive_packet),
+    FF_CODEC_RECEIVE_PACKET_CB(&ff_hw_base_encode_receive_packet),
     .close          = &vaapi_encode_mjpeg_close,
     .p.priv_class   = &vaapi_encode_mjpeg_class,
     .p.capabilities = AV_CODEC_CAP_HARDWARE | AV_CODEC_CAP_DR1 |
diff --git a/libavcodec/vaapi_encode_mpeg2.c b/libavcodec/vaapi_encode_mpeg2.c
index 69784f9ba6..c20196fb48 100644
--- a/libavcodec/vaapi_encode_mpeg2.c
+++ b/libavcodec/vaapi_encode_mpeg2.c
@@ -166,6 +166,7 @@ fail:
 
 static int vaapi_encode_mpeg2_init_sequence_params(AVCodecContext *avctx)
 {
+    FFHWBaseEncodeContext         *base_ctx = avctx->priv_data;
     VAAPIEncodeContext                 *ctx = avctx->priv_data;
     VAAPIEncodeMPEG2Context           *priv = avctx->priv_data;
     MPEG2RawSequenceHeader              *sh = &priv->sequence_header;
@@ -281,7 +282,7 @@ static int vaapi_encode_mpeg2_init_sequence_params(AVCodecContext *avctx)
 
     se->bit_rate_extension        = priv->bit_rate >> 18;
     se->vbv_buffer_size_extension = priv->vbv_buffer_size >> 10;
-    se->low_delay                 = ctx->b_per_p == 0;
+    se->low_delay                 = base_ctx->b_per_p == 0;
 
     se->frame_rate_extension_n = ext_n;
     se->frame_rate_extension_d = ext_d;
@@ -353,8 +354,8 @@ static int vaapi_encode_mpeg2_init_sequence_params(AVCodecContext *avctx)
 
 
     *vseq = (VAEncSequenceParameterBufferMPEG2) {
-        .intra_period = ctx->gop_size,
-        .ip_period    = ctx->b_per_p + 1,
+        .intra_period = base_ctx->gop_size,
+        .ip_period    = base_ctx->b_per_p + 1,
 
         .picture_width  = avctx->width,
         .picture_height = avctx->height,
@@ -417,12 +418,13 @@ static int vaapi_encode_mpeg2_init_sequence_params(AVCodecContext *avctx)
 }
 
 static int vaapi_encode_mpeg2_init_picture_params(AVCodecContext *avctx,
-                                                 VAAPIEncodePicture *pic)
+                                                  VAAPIEncodePicture *vaapi_pic)
 {
     VAAPIEncodeMPEG2Context          *priv = avctx->priv_data;
+    const FFHWBaseEncodePicture       *pic = &vaapi_pic->base;
     MPEG2RawPictureHeader              *ph = &priv->picture_header;
     MPEG2RawPictureCodingExtension    *pce = &priv->picture_coding_extension.data.picture_coding;
-    VAEncPictureParameterBufferMPEG2 *vpic = pic->codec_picture_params;
+    VAEncPictureParameterBufferMPEG2 *vpic = vaapi_pic->codec_picture_params;
 
     if (pic->type == FF_HW_PICTURE_TYPE_IDR || pic->type == FF_HW_PICTURE_TYPE_I) {
         ph->temporal_reference  = 0;
@@ -448,8 +450,8 @@ static int vaapi_encode_mpeg2_init_picture_params(AVCodecContext *avctx,
         pce->f_code[1][1] = 15;
     }
 
-    vpic->reconstructed_picture = pic->recon_surface;
-    vpic->coded_buf             = pic->output_buffer;
+    vpic->reconstructed_picture = vaapi_pic->recon_surface;
+    vpic->coded_buf             = vaapi_pic->output_buffer;
 
     switch (pic->type) {
     case FF_HW_PICTURE_TYPE_IDR:
@@ -458,12 +460,12 @@ static int vaapi_encode_mpeg2_init_picture_params(AVCodecContext *avctx,
         break;
     case FF_HW_PICTURE_TYPE_P:
         vpic->picture_type = VAEncPictureTypePredictive;
-        vpic->forward_reference_picture = pic->refs[0][0]->recon_surface;
+        vpic->forward_reference_picture = ((VAAPIEncodePicture *)pic->refs[0][0])->recon_surface;
         break;
     case FF_HW_PICTURE_TYPE_B:
         vpic->picture_type = VAEncPictureTypeBidirectional;
-        vpic->forward_reference_picture  = pic->refs[0][0]->recon_surface;
-        vpic->backward_reference_picture = pic->refs[1][0]->recon_surface;
+        vpic->forward_reference_picture  = ((VAAPIEncodePicture *)pic->refs[0][0])->recon_surface;
+        vpic->backward_reference_picture = ((VAAPIEncodePicture *)pic->refs[1][0])->recon_surface;
         break;
     default:
         av_assert0(0 && "invalid picture type");
@@ -479,11 +481,12 @@ static int vaapi_encode_mpeg2_init_picture_params(AVCodecContext *avctx,
 }
 
 static int vaapi_encode_mpeg2_init_slice_params(AVCodecContext *avctx,
-                                               VAAPIEncodePicture *pic,
-                                               VAAPIEncodeSlice *slice)
+                                                VAAPIEncodePicture *vaapi_pic,
+                                                VAAPIEncodeSlice *slice)
 {
-    VAAPIEncodeMPEG2Context            *priv = avctx->priv_data;
-    VAEncSliceParameterBufferMPEG2   *vslice = slice->codec_slice_params;
+    const FFHWBaseEncodePicture       *pic = &vaapi_pic->base;
+    VAAPIEncodeMPEG2Context          *priv = avctx->priv_data;
+    VAEncSliceParameterBufferMPEG2 *vslice = slice->codec_slice_params;
     int qp;
 
     vslice->macroblock_address = slice->block_start;
@@ -695,7 +698,7 @@ const FFCodec ff_mpeg2_vaapi_encoder = {
     .p.id           = AV_CODEC_ID_MPEG2VIDEO,
     .priv_data_size = sizeof(VAAPIEncodeMPEG2Context),
     .init           = &vaapi_encode_mpeg2_init,
-    FF_CODEC_RECEIVE_PACKET_CB(&ff_vaapi_encode_receive_packet),
+    FF_CODEC_RECEIVE_PACKET_CB(&ff_hw_base_encode_receive_packet),
     .close          = &vaapi_encode_mpeg2_close,
     .p.priv_class   = &vaapi_encode_mpeg2_class,
     .p.capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE |
diff --git a/libavcodec/vaapi_encode_vp8.c b/libavcodec/vaapi_encode_vp8.c
index 1f2de050d2..a95f9c5447 100644
--- a/libavcodec/vaapi_encode_vp8.c
+++ b/libavcodec/vaapi_encode_vp8.c
@@ -52,6 +52,7 @@ typedef struct VAAPIEncodeVP8Context {
 
 static int vaapi_encode_vp8_init_sequence_params(AVCodecContext *avctx)
 {
+    FFHWBaseEncodeContext       *base_ctx = avctx->priv_data;
     VAAPIEncodeContext               *ctx = avctx->priv_data;
     VAEncSequenceParameterBufferVP8 *vseq = ctx->codec_sequence_params;
 
@@ -66,22 +67,23 @@ static int vaapi_encode_vp8_init_sequence_params(AVCodecContext *avctx)
 
     if (!(ctx->va_rc_mode & VA_RC_CQP)) {
         vseq->bits_per_second = ctx->va_bit_rate;
-        vseq->intra_period    = ctx->gop_size;
+        vseq->intra_period    = base_ctx->gop_size;
     }
 
     return 0;
 }
 
 static int vaapi_encode_vp8_init_picture_params(AVCodecContext *avctx,
-                                                VAAPIEncodePicture *pic)
+                                                VAAPIEncodePicture *vaapi_pic)
 {
+    const FFHWBaseEncodePicture     *pic = &vaapi_pic->base;
     VAAPIEncodeVP8Context          *priv = avctx->priv_data;
-    VAEncPictureParameterBufferVP8 *vpic = pic->codec_picture_params;
+    VAEncPictureParameterBufferVP8 *vpic = vaapi_pic->codec_picture_params;
     int i;
 
-    vpic->reconstructed_frame = pic->recon_surface;
+    vpic->reconstructed_frame = vaapi_pic->recon_surface;
 
-    vpic->coded_buf = pic->output_buffer;
+    vpic->coded_buf = vaapi_pic->output_buffer;
 
     switch (pic->type) {
     case FF_HW_PICTURE_TYPE_IDR:
@@ -101,7 +103,7 @@ static int vaapi_encode_vp8_init_picture_params(AVCodecContext *avctx,
         vpic->ref_last_frame =
         vpic->ref_gf_frame   =
         vpic->ref_arf_frame  =
-            pic->refs[0][0]->recon_surface;
+            ((VAAPIEncodePicture *)pic->refs[0][0])->recon_surface;
         break;
     default:
         av_assert0(0 && "invalid picture type");
@@ -145,7 +147,7 @@ static int vaapi_encode_vp8_write_quant_table(AVCodecContext *avctx,
 
     memset(&quant, 0, sizeof(quant));
 
-    if (pic->type == FF_HW_PICTURE_TYPE_P)
+    if (pic->base.type == FF_HW_PICTURE_TYPE_P)
         q = priv->q_index_p;
     else
         q = priv->q_index_i;
@@ -250,7 +252,7 @@ const FFCodec ff_vp8_vaapi_encoder = {
     .p.id           = AV_CODEC_ID_VP8,
     .priv_data_size = sizeof(VAAPIEncodeVP8Context),
     .init           = &vaapi_encode_vp8_init,
-    FF_CODEC_RECEIVE_PACKET_CB(&ff_vaapi_encode_receive_packet),
+    FF_CODEC_RECEIVE_PACKET_CB(&ff_hw_base_encode_receive_packet),
     .close          = &ff_vaapi_encode_close,
     .p.priv_class   = &vaapi_encode_vp8_class,
     .p.capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE |
diff --git a/libavcodec/vaapi_encode_vp9.c b/libavcodec/vaapi_encode_vp9.c
index 690e4a6bd1..a737a0cdd7 100644
--- a/libavcodec/vaapi_encode_vp9.c
+++ b/libavcodec/vaapi_encode_vp9.c
@@ -53,6 +53,7 @@ typedef struct VAAPIEncodeVP9Context {
 
 static int vaapi_encode_vp9_init_sequence_params(AVCodecContext *avctx)
 {
+    FFHWBaseEncodeContext       *base_ctx = avctx->priv_data;
     VAAPIEncodeContext               *ctx = avctx->priv_data;
     VAEncSequenceParameterBufferVP9 *vseq = ctx->codec_sequence_params;
     VAEncPictureParameterBufferVP9  *vpic = ctx->codec_picture_params;
@@ -64,7 +65,7 @@ static int vaapi_encode_vp9_init_sequence_params(AVCodecContext *avctx)
 
     if (!(ctx->va_rc_mode & VA_RC_CQP)) {
         vseq->bits_per_second = ctx->va_bit_rate;
-        vseq->intra_period    = ctx->gop_size;
+        vseq->intra_period    = base_ctx->gop_size;
     }
 
     vpic->frame_width_src  = avctx->width;
@@ -76,17 +77,18 @@ static int vaapi_encode_vp9_init_sequence_params(AVCodecContext *avctx)
 }
 
 static int vaapi_encode_vp9_init_picture_params(AVCodecContext *avctx,
-                                                VAAPIEncodePicture *pic)
+                                                VAAPIEncodePicture *vaapi_pic)
 {
-    VAAPIEncodeContext              *ctx = avctx->priv_data;
+    FFHWBaseEncodeContext      *base_ctx = avctx->priv_data;
     VAAPIEncodeVP9Context          *priv = avctx->priv_data;
+    const FFHWBaseEncodePicture     *pic = &vaapi_pic->base;
     VAAPIEncodeVP9Picture          *hpic = pic->priv_data;
-    VAEncPictureParameterBufferVP9 *vpic = pic->codec_picture_params;
+    VAEncPictureParameterBufferVP9 *vpic = vaapi_pic->codec_picture_params;
     int i;
     int num_tile_columns;
 
-    vpic->reconstructed_frame = pic->recon_surface;
-    vpic->coded_buf = pic->output_buffer;
+    vpic->reconstructed_frame = vaapi_pic->recon_surface;
+    vpic->coded_buf = vaapi_pic->output_buffer;
 
     // Maximum width of a tile in units of superblocks is MAX_TILE_WIDTH_B64(64)
     // So the number of tile columns is related to the width of the picture.
@@ -107,7 +109,7 @@ static int vaapi_encode_vp9_init_picture_params(AVCodecContext *avctx,
             VAAPIEncodeVP9Picture *href = pic->refs[0][0]->priv_data;
             av_assert0(href->slot == 0 || href->slot == 1);
 
-            if (ctx->max_b_depth > 0) {
+            if (base_ctx->max_b_depth > 0) {
                 hpic->slot = !href->slot;
                 vpic->refresh_frame_flags = 1 << hpic->slot | 0xfc;
             } else {
@@ -127,7 +129,7 @@ static int vaapi_encode_vp9_init_picture_params(AVCodecContext *avctx,
             av_assert0(href0->slot < pic->b_depth + 1 &&
                        href1->slot < pic->b_depth + 1);
 
-            if (pic->b_depth == ctx->max_b_depth) {
+            if (pic->b_depth == base_ctx->max_b_depth) {
                 // Unreferenced frame.
                 vpic->refresh_frame_flags = 0x00;
                 hpic->slot = 8;
@@ -159,11 +161,11 @@ static int vaapi_encode_vp9_init_picture_params(AVCodecContext *avctx,
 
     for (i = 0; i < MAX_REFERENCE_LIST_NUM; i++) {
         for (int j = 0; j < pic->nb_refs[i]; j++) {
-            VAAPIEncodePicture *ref_pic = pic->refs[i][j];
+            FFHWBaseEncodePicture *ref_pic = pic->refs[i][j];
             int slot;
             slot = ((VAAPIEncodeVP9Picture*)ref_pic->priv_data)->slot;
             av_assert0(vpic->reference_frames[slot] == VA_INVALID_SURFACE);
-            vpic->reference_frames[slot] = ref_pic->recon_surface;
+            vpic->reference_frames[slot] = ((VAAPIEncodePicture *)ref_pic)->recon_surface;
         }
     }
 
@@ -307,7 +309,7 @@ const FFCodec ff_vp9_vaapi_encoder = {
     .p.id           = AV_CODEC_ID_VP9,
     .priv_data_size = sizeof(VAAPIEncodeVP9Context),
     .init           = &vaapi_encode_vp9_init,
-    FF_CODEC_RECEIVE_PACKET_CB(&ff_vaapi_encode_receive_packet),
+    FF_CODEC_RECEIVE_PACKET_CB(&ff_hw_base_encode_receive_packet),
     .close          = &ff_vaapi_encode_close,
     .p.priv_class   = &vaapi_encode_vp9_class,
     .p.capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE |
