diff mbox series

[FFmpeg-devel,v2,1/8] avutil/hwcontext: Add VPE implementation

Message ID 8847CE4FAA78C048A83C2AE8AA751D2770A5370D@SHASXM03.verisilicon.com
State New
Headers show
Series [FFmpeg-devel,v2,1/8] avutil/hwcontext: Add VPE implementation
Related show

Checks

Context Check Description
andriy/default pending
andriy/make success Make finished
andriy/make_fate success Make fate finished

Commit Message

Zhang, Guiyong May 31, 2020, 6:24 a.m. UTC
VPE(VeriSilicon Paltform Engine) is VeriSilicon's hardware engine
for multi formats video encoding and decoding.

It is used with the VPE hwaccel codec API and library to initialize
and use a VPE device which is within the hwcontext libavutil framework.

Signed-off-by: Qin.Wang <Qin.Wang@verisilicon.com>
---
 configure                      |   3 +
 libavutil/Makefile             |   3 +
 libavutil/hwcontext.c          |   4 +
 libavutil/hwcontext.h          |   1 +
 libavutil/hwcontext_internal.h |   1 +
 libavutil/hwcontext_vpe.c      | 402 +++++++++++++++++++++++++++++++++++++++++
 libavutil/hwcontext_vpe.h      |  52 ++++++
 libavutil/pixdesc.c            |   4 +
 libavutil/pixfmt.h             |   7 +
 libavutil/version.h            |   2 +-
 10 files changed, 478 insertions(+), 1 deletion(-)
 create mode 100644 libavutil/hwcontext_vpe.c
 create mode 100644 libavutil/hwcontext_vpe.h

Comments

Carl Eugen Hoyos May 31, 2020, 11:24 a.m. UTC | #1
> Am 31.05.2020 um 08:24 schrieb Zhang, Guiyong <Guiyong.Zhang@verisilicon.com>:
> 
> @@ -1822,6 +1823,7 @@ EXTERNAL_LIBRARY_LIST="
>     opengl
>     pocketsphinx
>     vapoursynth
> +    vpe

If libvpe is a free library, please point to the repository, if not please move this to the non-free section.

Carl Eugen
diff mbox series

Patch

diff --git a/configure b/configure
index 8569a60..7f0f843 100755
--- a/configure
+++ b/configure
@@ -322,6 +322,7 @@  External library support:
   --enable-vulkan          enable Vulkan code [no]
   --disable-xlib           disable xlib [autodetect]
   --disable-zlib           disable zlib [autodetect]
+  --enable-vpe             enable vpe codec [no]
 
   The following libraries provide various hardware acceleration features:
   --disable-amf            disable AMF video encoding code [autodetect]
@@ -1822,6 +1823,7 @@  EXTERNAL_LIBRARY_LIST="
     opengl
     pocketsphinx
     vapoursynth
+    vpe
 "
 
 HWACCEL_AUTODETECT_LIBRARY_LIST="
@@ -6476,6 +6478,7 @@  enabled rkmpp             && { require_pkg_config rkmpp rockchip_mpp  rockchip/r
                                  die "ERROR: rkmpp requires --enable-libdrm"; }
                              }
 enabled vapoursynth       && require_pkg_config vapoursynth "vapoursynth-script >= 42" VSScript.h vsscript_init
+enabled vpe               && require_pkg_config libvpi libvpi "vpe/vpi_types.h vpe/vpi_api.h" vpi_create
 
 
 if enabled gcrypt; then
diff --git a/libavutil/Makefile b/libavutil/Makefile
index 9b08372..5809b38 100644
--- a/libavutil/Makefile
+++ b/libavutil/Makefile
@@ -46,6 +46,7 @@  HEADERS = adler32.h                                                     \
           hwcontext_videotoolbox.h                                      \
           hwcontext_vdpau.h                                             \
           hwcontext_vulkan.h                                            \
+          hwcontext_vpe.h                                               \
           imgutils.h                                                    \
           intfloat.h                                                    \
           intreadwrite.h                                                \
@@ -184,6 +185,7 @@  OBJS-$(CONFIG_VAAPI)                    += hwcontext_vaapi.o
 OBJS-$(CONFIG_VIDEOTOOLBOX)             += hwcontext_videotoolbox.o
 OBJS-$(CONFIG_VDPAU)                    += hwcontext_vdpau.o
 OBJS-$(CONFIG_VULKAN)                   += hwcontext_vulkan.o
+OBJS-$(CONFIG_VPE)                      += hwcontext_vpe.o
 
 OBJS += $(COMPAT_OBJS:%=../compat/%)
 
@@ -201,6 +203,7 @@  SKIPHEADERS-$(CONFIG_VAAPI)            += hwcontext_vaapi.h
 SKIPHEADERS-$(CONFIG_VIDEOTOOLBOX)     += hwcontext_videotoolbox.h
 SKIPHEADERS-$(CONFIG_VDPAU)            += hwcontext_vdpau.h
 SKIPHEADERS-$(CONFIG_VULKAN)           += hwcontext_vulkan.h
+SKIPHEADERS-$(CONFIG_VPE)              += hwcontext_vpe.h
 
 TESTPROGS = adler32                                                     \
             aes                                                         \
diff --git a/libavutil/hwcontext.c b/libavutil/hwcontext.c
index d13d0f7..5b12013 100644
--- a/libavutil/hwcontext.c
+++ b/libavutil/hwcontext.c
@@ -62,6 +62,9 @@  static const HWContextType * const hw_table[] = {
 #if CONFIG_VULKAN
     &ff_hwcontext_type_vulkan,
 #endif
+#if CONFIG_VPE
+    &ff_hwcontext_type_vpe,
+#endif
     NULL,
 };
 
@@ -77,6 +80,7 @@  static const char *const hw_type_names[] = {
     [AV_HWDEVICE_TYPE_VIDEOTOOLBOX] = "videotoolbox",
     [AV_HWDEVICE_TYPE_MEDIACODEC] = "mediacodec",
     [AV_HWDEVICE_TYPE_VULKAN] = "vulkan",
+    [AV_HWDEVICE_TYPE_VPE] = "vpe",
 };
 
 enum AVHWDeviceType av_hwdevice_find_type_by_name(const char *name)
diff --git a/libavutil/hwcontext.h b/libavutil/hwcontext.h
index 04d19d8..62d948e 100644
--- a/libavutil/hwcontext.h
+++ b/libavutil/hwcontext.h
@@ -37,6 +37,7 @@  enum AVHWDeviceType {
     AV_HWDEVICE_TYPE_OPENCL,
     AV_HWDEVICE_TYPE_MEDIACODEC,
     AV_HWDEVICE_TYPE_VULKAN,
+    AV_HWDEVICE_TYPE_VPE,
 };
 
 typedef struct AVHWDeviceInternal AVHWDeviceInternal;
diff --git a/libavutil/hwcontext_internal.h b/libavutil/hwcontext_internal.h
index e626649..3190a3a 100644
--- a/libavutil/hwcontext_internal.h
+++ b/libavutil/hwcontext_internal.h
@@ -174,5 +174,6 @@  extern const HWContextType ff_hwcontext_type_vdpau;
 extern const HWContextType ff_hwcontext_type_videotoolbox;
 extern const HWContextType ff_hwcontext_type_mediacodec;
 extern const HWContextType ff_hwcontext_type_vulkan;
+extern const HWContextType ff_hwcontext_type_vpe;
 
 #endif /* AVUTIL_HWCONTEXT_INTERNAL_H */
diff --git a/libavutil/hwcontext_vpe.c b/libavutil/hwcontext_vpe.c
new file mode 100644
index 0000000..7b218e3
--- /dev/null
+++ b/libavutil/hwcontext_vpe.c
@@ -0,0 +1,402 @@ 
+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <sys/ioctl.h>
+
+#include "buffer.h"
+#include "pixfmt.h"
+#include "pixdesc.h"
+#include "hwcontext.h"
+#include "hwcontext_internal.h"
+#include "hwcontext_vpe.h"
+#include "libavutil/opt.h"
+
+typedef struct VpeDevicePriv {
+    VpiSysInfo *sys_info;
+} VpeDevicePriv;
+
+typedef struct VpeDeviceContext {
+    VpiMediaProc *media_proc;
+} VpeDeviceContext;
+
+static const enum AVPixelFormat supported_sw_formats[] = {
+    AV_PIX_FMT_NV12,
+    AV_PIX_FMT_P010LE,
+    AV_PIX_FMT_YUV420P,
+    AV_PIX_FMT_YUV422P,
+    AV_PIX_FMT_NV21,
+    AV_PIX_FMT_YUV420P10LE,
+    AV_PIX_FMT_YUV420P10BE,
+    AV_PIX_FMT_YUV422P10LE,
+    AV_PIX_FMT_YUV422P10BE,
+    AV_PIX_FMT_P010BE,
+    AV_PIX_FMT_YUV444P,
+    AV_PIX_FMT_RGB24,
+    AV_PIX_FMT_BGR24,
+    AV_PIX_FMT_ARGB,
+    AV_PIX_FMT_RGBA,
+    AV_PIX_FMT_ABGR,
+    AV_PIX_FMT_BGRA,
+};
+
+static const enum AVPixelFormat supported_hw_formats[] = {
+    AV_PIX_FMT_VPE,
+};
+
+static int vpe_frames_get_constraints(AVHWDeviceContext *ctx,
+                                      const void *hwconfig,
+                                      AVHWFramesConstraints *constraints)
+{
+    int i;
+
+    constraints->valid_sw_formats =
+        av_malloc_array(FF_ARRAY_ELEMS(supported_sw_formats) + 1,
+                        sizeof(*constraints->valid_sw_formats));
+    if (!constraints->valid_sw_formats)
+        return AVERROR(ENOMEM);
+
+    for (i = 0; i < FF_ARRAY_ELEMS(supported_sw_formats); i++)
+        constraints->valid_sw_formats[i] = supported_sw_formats[i];
+    constraints->valid_sw_formats[FF_ARRAY_ELEMS(supported_sw_formats)] =
+        AV_PIX_FMT_NONE;
+
+    constraints->valid_hw_formats =
+        av_malloc_array(2, sizeof(*constraints->valid_hw_formats));
+    if (!constraints->valid_hw_formats) {
+        return AVERROR(ENOMEM);
+    }
+
+    constraints->valid_hw_formats[0] = AV_PIX_FMT_VPE;
+    constraints->valid_hw_formats[1] = AV_PIX_FMT_NONE;
+
+    constraints->min_width  = 144;
+    constraints->min_height = 144;
+    constraints->max_width  = 4096;
+    constraints->max_height = 4096;
+
+    return 0;
+}
+
+static int vpe_transfer_get_formats(AVHWFramesContext *ctx,
+                                    enum AVHWFrameTransferDirection dir,
+                                    enum AVPixelFormat **formats)
+{
+    enum AVPixelFormat *fmts;
+
+    fmts = av_malloc_array(2, sizeof(*fmts));
+    if (!fmts)
+        return AVERROR(ENOMEM);
+
+    fmts[0] = ctx->sw_format;
+    fmts[1] = AV_PIX_FMT_NONE;
+
+    *formats = fmts;
+
+    return 0;
+}
+
+static int vpe_transfer_data_from(AVHWFramesContext *ctx, AVFrame *dst,
+                                  const AVFrame *src)
+{
+    VpeDeviceContext *priv = ctx->device_ctx->internal->priv;
+    VpiFrame *in_frame;
+    VpiFrame out_frame;
+    VpiCtrlCmdParam cmd_param;
+
+    VpiPicInfo *pic_info;
+    int pp_index;
+    int i, ret;
+
+    in_frame = (VpiFrame *)src->data[0];
+    for (i = 1; i < PIC_INDEX_MAX_NUMBER; i++) {
+        pic_info = (VpiPicInfo *)src->buf[i]->data;
+        if (pic_info->enabled == 1) {
+            pp_index = i;
+            break;
+        }
+    }
+    if (i == PIC_INDEX_MAX_NUMBER) {
+        return AVERROR_EXTERNAL;
+    }
+
+    cmd_param.cmd  = VPI_CMD_HWDW_SET_INDEX;
+    cmd_param.data = (void *)&pp_index;
+    priv->media_proc->vpi->control(priv->media_proc->ctx,
+                                  (void *)&cmd_param, NULL);
+
+    out_frame.data[0]     = dst->data[0];
+    out_frame.data[1]     = dst->data[1];
+    out_frame.data[2]     = dst->data[2];
+    out_frame.linesize[0] = dst->linesize[0];
+    out_frame.linesize[1] = dst->linesize[1];
+    out_frame.linesize[2] = dst->linesize[2];
+    ret = priv->media_proc->vpi->process(priv->media_proc->ctx,
+                                         in_frame, &out_frame);
+
+    return ret;
+}
+
+static void vpe_buffer_free(void *opaque, uint8_t *data)
+{
+    AVHWFramesContext *ctx           = opaque;
+    AVVpeDeviceContext *device_hwctx = ctx->device_ctx->hwctx;
+    VpiCtrlCmdParam cmd_param;
+
+    cmd_param.cmd  = VPI_CMD_FREE_FRAME_BUFFER;
+    cmd_param.data = (void *)data;
+    device_hwctx->func->control(&device_hwctx->device, &cmd_param, NULL);
+}
+
+static AVBufferRef *vpe_pool_alloc(void *opaque, int size)
+{
+    AVHWFramesContext *ctx           = opaque;
+    AVVpeDeviceContext *device_hwctx = ctx->device_ctx->hwctx;
+    VpiFrame *v_frame                = NULL;
+    VpiCtrlCmdParam cmd_param;
+    AVBufferRef *ret;
+
+    cmd_param.cmd = VPI_CMD_GET_FRAME_BUFFER;
+    device_hwctx->func->control(&device_hwctx->device, &cmd_param, (void *)&v_frame);
+    if (!v_frame) {
+        return NULL;
+    }
+    ret = av_buffer_create((uint8_t*)v_frame, size,
+                            vpe_buffer_free, ctx, 0);
+
+    return ret;
+}
+
+static int vpe_frames_init(AVHWFramesContext *hwfc)
+{
+    AVVpeDeviceContext *device_hwctx = hwfc->device_ctx->hwctx;
+    AVVpeFramesContext *frame_hwctx  = hwfc->hwctx;
+    VpiCtrlCmdParam cmd_param;
+    int size         = 0;
+    int picinfo_size = 0;
+    int i;
+
+    for (i = 0; i < FF_ARRAY_ELEMS(supported_sw_formats); i++) {
+        if (hwfc->sw_format == supported_sw_formats[i]) {
+            break;
+        }
+    }
+    if (i == FF_ARRAY_ELEMS(supported_sw_formats)) {
+        av_log(hwfc, AV_LOG_ERROR, "Pixel format '%s' is not supported\n",
+               av_get_pix_fmt_name(hwfc->sw_format));
+        return AVERROR(ENOSYS);
+    }
+
+    if (!hwfc->pool) {
+        cmd_param.cmd = VPI_CMD_GET_VPEFRAME_SIZE;
+        device_hwctx->func->control(&device_hwctx->device, &cmd_param, (void *)&size);
+        if (size == 0) {
+            return AVERROR_EXTERNAL;
+        }
+        frame_hwctx->frame = (VpiFrame *)av_mallocz(size);
+        if (!frame_hwctx->frame) {
+            return AVERROR(ENOMEM);
+        }
+        frame_hwctx->frame_size = size;
+
+        cmd_param.cmd = VPI_CMD_GET_PICINFO_SIZE;
+        device_hwctx->func->control(&device_hwctx->device, &cmd_param, (void *)&picinfo_size);
+        if (picinfo_size == 0) {
+            return AVERROR_EXTERNAL;
+        }
+        frame_hwctx->pic_info_size = picinfo_size;
+
+        hwfc->internal->pool_internal =
+            av_buffer_pool_init2(size, hwfc, vpe_pool_alloc, NULL);
+        if (!hwfc->internal->pool_internal)
+            return AVERROR(ENOMEM);
+    }
+    return 0;
+}
+
+static int vpe_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
+{
+    AVVpeFramesContext *vpeframe_ctx = hwfc->hwctx;
+    VpiPicInfo *pic_info;
+    int i;
+
+    frame->buf[0] = av_buffer_pool_get(hwfc->pool);
+    if (!frame->buf[0])
+        return AVERROR(ENOMEM);
+
+    frame->data[0] = frame->buf[0]->data;
+    memset(frame->data[0], 0, vpeframe_ctx->frame_size);
+
+    /* suppor max 4 channel low res buffer */
+    for (i = 1; i < PIC_INDEX_MAX_NUMBER; i++) {
+        frame->buf[i] = av_buffer_alloc(vpeframe_ctx->pic_info_size);
+        if (frame->buf[i] == 0) {
+            return AVERROR(ENOMEM);
+        }
+    }
+    /* always enable the first channel */
+    pic_info = (VpiPicInfo *)frame->buf[1]->data;
+    pic_info->enabled = 1;
+
+    frame->format  = AV_PIX_FMT_VPE;
+    frame->width   = hwfc->width;
+    frame->height  = hwfc->height;
+    return 0;
+}
+
+static void vpe_device_uninit(AVHWDeviceContext *device_ctx)
+{
+    VpeDeviceContext *priv = device_ctx->internal->priv;
+
+    if (priv->media_proc) {
+        if (priv->media_proc->ctx) {
+            priv->media_proc->vpi->close(priv->media_proc->ctx);
+            vpi_destroy(priv->media_proc->ctx);
+            priv->media_proc->ctx = NULL;
+        }
+        vpi_freep(&priv->media_proc);
+    }
+}
+
+static int vpe_device_init(AVHWDeviceContext *device_ctx)
+{
+    VpeDeviceContext *priv = device_ctx->internal->priv;
+    int ret = 0;
+
+    ret = vpi_get_media_proc_struct(&priv->media_proc);
+    if (ret || !priv->media_proc) {
+        return AVERROR(ENOMEM);
+    }
+
+    ret = vpi_create(&priv->media_proc->ctx, &priv->media_proc->vpi,
+                    HWDOWNLOAD_VPE);
+    if (ret != 0)
+        return AVERROR_EXTERNAL;
+
+    ret = priv->media_proc->vpi->init(priv->media_proc->ctx, NULL);
+    if (ret != 0)
+       return AVERROR_EXTERNAL;
+
+    return 0;
+}
+
+static void vpe_device_free(AVHWDeviceContext *device_ctx)
+{
+    AVVpeDeviceContext *hwctx = device_ctx->hwctx;
+    VpeDevicePriv *priv = (VpeDevicePriv *)device_ctx->user_opaque;
+
+    vpi_destroy(priv->sys_info);
+
+    vpi_freep(&priv->sys_info);
+    av_freep(&priv);
+
+    if (hwctx->device >= 0) {
+        close(hwctx->device);
+    }
+}
+
+static int vpe_device_create(AVHWDeviceContext *device_ctx, const char *device,
+                             AVDictionary *opts, int flags)
+{
+    AVVpeDeviceContext *hwctx = device_ctx->hwctx;
+    AVDictionaryEntry *opt;
+    VpeDevicePriv *priv;
+    const char *path;
+    int ret;
+
+    priv = av_mallocz(sizeof(*priv));
+    if (!priv)
+        return AVERROR(ENOMEM);
+    ret = vpi_get_sys_info_struct(&priv->sys_info);
+    if (ret || !priv->sys_info) {
+        av_freep(&priv);
+        return AVERROR(ENOMEM);
+    }
+
+    device_ctx->user_opaque = priv;
+    device_ctx->free = vpe_device_free;
+
+    // Try to open the device as a transcode path.
+    // Default to using the first node if the user did not
+    // supply a path.
+    path = device ? device : "/dev/transcoder0";
+    hwctx->device = open(path, O_RDWR);
+    if (hwctx->device == -1) {
+        av_log(device_ctx, AV_LOG_ERROR, "failed to open hw device\n");
+        return AVERROR(errno);
+    }
+
+    priv->sys_info->device        = hwctx->device;
+    priv->sys_info->device_name   = av_strdup(path);
+    if (!priv->sys_info->device_name) {
+        return AVERROR(ENOMEM);
+    }
+    priv->sys_info->priority      = VPE_TASK_VOD;
+    priv->sys_info->sys_log_level = 0;
+
+    if (opts) {
+        opt = av_dict_get(opts, "priority", NULL, 0);
+        if (opt) {
+            if (!strcmp(opt->value, "live")) {
+                priv->sys_info->priority = VPE_TASK_LIVE;
+            } else if (!strcmp(opt->value, "vod")) {
+                priv->sys_info->priority = VPE_TASK_VOD;
+            } else {
+                av_log(device_ctx, AV_LOG_ERROR, "Unknow priority : %s\n",
+                       opt->value);
+                return AVERROR_INVALIDDATA;
+            }
+        }
+
+        opt = av_dict_get(opts, "fbloglevel", NULL, 0);
+        if (opt) {
+            priv->sys_info->sys_log_level = atoi(opt->value);
+        }
+    }
+
+    if (vpi_create(&priv->sys_info, &hwctx->func, HWCONTEXT_VPE) != 0) {
+        return AVERROR_EXTERNAL;
+    }
+
+    return 0;
+}
+
+const HWContextType ff_hwcontext_type_vpe = {
+    .type = AV_HWDEVICE_TYPE_VPE,
+    .name = "VPE",
+
+    .device_hwctx_size = sizeof(AVVpeDeviceContext),
+    .device_priv_size  = sizeof(VpeDeviceContext),
+    .frames_hwctx_size = sizeof(AVVpeFramesContext),
+
+    .device_create          = vpe_device_create,
+    .device_init            = vpe_device_init,
+    .device_uninit          = vpe_device_uninit,
+    .frames_get_constraints = vpe_frames_get_constraints,
+    .frames_init            = vpe_frames_init,
+    .frames_get_buffer      = vpe_get_buffer,
+    .transfer_get_formats   = vpe_transfer_get_formats,
+    .transfer_data_from     = vpe_transfer_data_from,
+
+    .pix_fmts = (const enum AVPixelFormat[]){ AV_PIX_FMT_VPE, AV_PIX_FMT_NONE },
+};
diff --git a/libavutil/hwcontext_vpe.h b/libavutil/hwcontext_vpe.h
new file mode 100644
index 0000000..6fd5291
--- /dev/null
+++ b/libavutil/hwcontext_vpe.h
@@ -0,0 +1,52 @@ 
+/*
+ * Verisilicon VPI Video codec
+ * Copyright (c) 2019 VeriSilicon, Inc.
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVUTIL_HWCONTEXT_VPE_H
+#define AVUTIL_HWCONTEXT_VPE_H
+
+#include <vpe/vpi_api.h>
+#include <vpe/vpi_types.h>
+
+/**
+ * @file
+ * An API-specific header for AV_HWDEVICE_TYPE_VPE.
+ */
+
+/**
+ * This struct is allocated as AVHWDeviceContext.hwctx
+ * It will save some device level info
+ */
+
+typedef struct AVVpeDeviceContext {
+    int device;
+    VpiApi *func;
+} AVVpeDeviceContext;
+
+/**
+ * This struct is allocated as AVHWFramesContext.hwctx
+ * It will save some frame level info
+ */
+typedef struct AVVpeFramesContext {
+    VpiFrame *frame;
+    int frame_size;
+    int pic_info_size;
+} AVVpeFramesContext;
+#endif /* AVUTIL_HWCONTEXT_VPE_H */
diff --git a/libavutil/pixdesc.c b/libavutil/pixdesc.c
index 9d61c52..f2dc9fa 100644
--- a/libavutil/pixdesc.c
+++ b/libavutil/pixdesc.c
@@ -2371,6 +2371,10 @@  static const AVPixFmtDescriptor av_pix_fmt_descriptors[AV_PIX_FMT_NB] = {
         .name = "vulkan",
         .flags = AV_PIX_FMT_FLAG_HWACCEL,
     },
+    [AV_PIX_FMT_VPE] = {
+        .name = "vpe",
+        .flags = AV_PIX_FMT_FLAG_HWACCEL,
+    },
 };
 #if FF_API_PLUS1_MINUS1
 FF_ENABLE_DEPRECATION_WARNINGS
diff --git a/libavutil/pixfmt.h b/libavutil/pixfmt.h
index 1c625cf..252261e 100644
--- a/libavutil/pixfmt.h
+++ b/libavutil/pixfmt.h
@@ -358,6 +358,13 @@  enum AVPixelFormat {
     AV_PIX_FMT_Y210BE,    ///< packed YUV 4:2:2 like YUYV422, 20bpp, data in the high bits, big-endian
     AV_PIX_FMT_Y210LE,    ///< packed YUV 4:2:2 like YUYV422, 20bpp, data in the high bits, little-endian
 
+    /**
+     * VPE hardware images.
+     *
+     * data[0] points to a VpiFrame
+     */
+    AV_PIX_FMT_VPE,
+
     AV_PIX_FMT_NB         ///< number of pixel formats, DO NOT USE THIS if you want to link with shared libav* because the number of formats might differ between versions
 };
 
diff --git a/libavutil/version.h b/libavutil/version.h
index 7acecf5..5d5cec6 100644
--- a/libavutil/version.h
+++ b/libavutil/version.h
@@ -79,7 +79,7 @@ 
  */
 
 #define LIBAVUTIL_VERSION_MAJOR  56
-#define LIBAVUTIL_VERSION_MINOR  49
+#define LIBAVUTIL_VERSION_MINOR  50
 #define LIBAVUTIL_VERSION_MICRO 100
 
 #define LIBAVUTIL_VERSION_INT   AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \