From patchwork Sun May 31 06:30:17 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Zhang, Guiyong" X-Patchwork-Id: 20039 Return-Path: X-Original-To: patchwork@ffaux-bg.ffmpeg.org Delivered-To: patchwork@ffaux-bg.ffmpeg.org Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org [79.124.17.100]) by ffaux.localdomain (Postfix) with ESMTP id 849EF44A961 for ; Sun, 31 May 2020 09:30:28 +0300 (EEST) Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 696EC689259; Sun, 31 May 2020 09:30:28 +0300 (EEST) X-Original-To: ffmpeg-devel@ffmpeg.org Delivered-To: ffmpeg-devel@ffmpeg.org Received: from shasxm03.verisilicon.com (unknown [101.89.135.45]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id A4C77680637 for ; Sun, 31 May 2020 09:30:20 +0300 (EEST) Content-Language: en-US DKIM-Signature: v=1; a=rsa-sha256; d=Verisilicon.com; s=default; c=simple/simple; t=1590906618; h=from:subject:to:date:message-id; bh=mkvjqd4qGa2l0kE6gyusk+ACftOVk5wuiNEanC67ewQ=; b=QdGj2aKa20z93JFkUODFTu7Eka1lF8cH8BzEb1sFJ77xCTPIZCRzfy2sJerXWkZXk6yClqHObPL vaNika+EYz6uIF4Xccwuq3muNBAtMvxrGn0zOHRdKFZkpOKZhSU0gfSZSwWkuSGiRMYiBFdU8o/pk 03RpKgLMP0GLZWMRQBE= Received: from SHASXM03.verisilicon.com ([fe80::938:4dda:a2f9:38aa]) by SHASXM06.verisilicon.com ([fe80::59a8:ce34:dc14:ddda%16]) with mapi id 14.03.0123.003; Sun, 31 May 2020 14:30:17 +0800 From: "Zhang, Guiyong" To: "ffmpeg-devel@ffmpeg.org" Thread-Topic: [PATCH v2 8/8] vfilter/pp: Add VPE post processing filter Thread-Index: AdY3FPMEqhWeJ/o+Tdezx5RhIaD1nA== Date: Sun, 31 May 2020 06:30:17 +0000 Message-ID: <8847CE4FAA78C048A83C2AE8AA751D2770A5376D@SHASXM03.verisilicon.com> Accept-Language: en-US, zh-CN X-MS-Has-Attach: X-MS-TNEF-Correlator: x-originating-ip: [10.10.25.45] x-tm-as-product-ver: SMEX-11.0.0.4179-8.100.1062-25452.001 x-tm-as-result: No--12.891600-0.000000-31 x-tm-as-user-approved-sender: Yes x-tm-as-user-blocked-sender: No MIME-Version: 1.0 X-Content-Filtered-By: Mailman/MimeDel 2.1.20 Subject: [FFmpeg-devel] [PATCH v2 8/8] vfilter/pp: Add VPE post processing filter X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" The input of this filter is raw video data, it supports most of the popular raw data formats like NV12, YUV420P, YUV420P10BE etc. Signed-off-by: Guiyong.zhang --- configure | 1 + libavfilter/Makefile | 1 + libavfilter/allfilters.c | 1 + libavfilter/vf_pp_vpe.c | 391 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 394 insertions(+) create mode 100755 libavfilter/vf_pp_vpe.c -- 1.8.3.1 diff --git a/configure b/configure index 6146954..78a3203 100755 --- a/configure +++ b/configure @@ -3643,6 +3643,7 @@ xfade_opencl_filter_deps="opencl" yadif_cuda_filter_deps="ffnvcodec" yadif_cuda_filter_deps_any="cuda_nvcc cuda_llvm" spliter_vpe_filter_deps="vpe" +pp_vpe_filter_deps="vpe" # examples avio_list_dir_deps="avformat avutil" diff --git a/libavfilter/Makefile b/libavfilter/Makefile index 104ec0c..ba4fad8 100644 --- a/libavfilter/Makefile +++ b/libavfilter/Makefile @@ -467,6 +467,7 @@ OBJS-$(CONFIG_ZMQ_FILTER) += f_zmq.o OBJS-$(CONFIG_ZOOMPAN_FILTER) += vf_zoompan.o OBJS-$(CONFIG_ZSCALE_FILTER) += vf_zscale.o OBJS-$(CONFIG_SPLITER_VPE_FILTER) += vf_spliter_vpe.o +OBJS-$(CONFIG_PP_VPE_FILTER) += vf_pp_vpe.o OBJS-$(CONFIG_ALLRGB_FILTER) += vsrc_testsrc.o OBJS-$(CONFIG_ALLYUV_FILTER) += vsrc_testsrc.o diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c index 353a3ca..40c78f0 100644 --- a/libavfilter/allfilters.c +++ b/libavfilter/allfilters.c @@ -445,6 +445,7 @@ extern AVFilter ff_vf_zmq; extern AVFilter ff_vf_zoompan; extern AVFilter ff_vf_zscale; extern AVFilter ff_vf_spliter_vpe; +extern AVFilter ff_vf_pp_vpe; extern AVFilter ff_vsrc_allrgb; extern AVFilter ff_vsrc_allyuv; diff --git a/libavfilter/vf_pp_vpe.c b/libavfilter/vf_pp_vpe.c new file mode 100755 index 0000000..ea718cc --- /dev/null +++ b/libavfilter/vf_pp_vpe.c @@ -0,0 +1,391 @@ +/* + * Verisilicon VPE Post Processing Filter + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERC`ABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include + +#include +#include + +#include "avfilter.h" +#include "formats.h" +#include "internal.h" +#include "libavutil/pixfmt.h" +#include "libavutil/buffer.h" +#include "libavutil/hwcontext.h" +#include "libavutil/opt.h" +#include "libavutil/frame.h" +#include "libavfilter/filters.h" +#include "libavutil/hwcontext_vpe.h" + +typedef struct VpePPFilter { + const AVClass *av_class; + AVBufferRef *hw_device; + AVBufferRef *hw_frame; + + VpiCtx ctx; + VpiApi *vpi; + + int nb_outputs; + int force_10bit; + char *low_res; + VpiPPOpition cfg; +} VpePPFilter; + +static const enum AVPixelFormat input_pix_fmts[] = { + AV_PIX_FMT_NV12, AV_PIX_FMT_P010LE, AV_PIX_FMT_YUV420P, + AV_PIX_FMT_YUV422P, AV_PIX_FMT_NV21, AV_PIX_FMT_YUV420P10LE, + AV_PIX_FMT_YUV420P10BE, AV_PIX_FMT_YUV422P10LE, AV_PIX_FMT_YUV422P10BE, + AV_PIX_FMT_P010BE, AV_PIX_FMT_YUV444P, AV_PIX_FMT_RGB24, + AV_PIX_FMT_BGR24, AV_PIX_FMT_ARGB, AV_PIX_FMT_RGBA, + AV_PIX_FMT_ABGR, AV_PIX_FMT_BGRA, AV_PIX_FMT_NONE, +}; + +typedef struct PixelMapTable { + enum AVPixelFormat src; + VpiPixsFmt des; +} PixelMapTable; + +static PixelMapTable ptable[] = { + { AV_PIX_FMT_YUV420P, VPI_FMT_YUV420P }, + { AV_PIX_FMT_YUV422P, VPI_FMT_YUV422P }, + { AV_PIX_FMT_NV12, VPI_FMT_NV12 }, + { AV_PIX_FMT_NV21, VPI_FMT_NV21 }, + { AV_PIX_FMT_YUV420P10LE, VPI_FMT_YUV420P10LE }, + { AV_PIX_FMT_YUV420P10BE, VPI_FMT_YUV420P10BE }, + { AV_PIX_FMT_YUV422P10LE, VPI_FMT_YUV422P10LE }, + { AV_PIX_FMT_YUV422P10BE, VPI_FMT_YUV422P10BE }, + { AV_PIX_FMT_P010LE, VPI_FMT_P010LE }, + { AV_PIX_FMT_P010BE, VPI_FMT_P010BE }, + { AV_PIX_FMT_YUV444P, VPI_FMT_YUV444P }, + { AV_PIX_FMT_RGB24, VPI_FMT_RGB24 }, + { AV_PIX_FMT_BGR24, VPI_FMT_BGR24 }, + { AV_PIX_FMT_ARGB, VPI_FMT_ARGB }, + { AV_PIX_FMT_RGBA, VPI_FMT_RGBA }, + { AV_PIX_FMT_ABGR, VPI_FMT_ABGR }, + { AV_PIX_FMT_BGRA, VPI_FMT_BGRA }, +}; + +static const enum AVPixelFormat output_pix_fmts[] = { + AV_PIX_FMT_VPE, + AV_PIX_FMT_NONE, +}; + +static av_cold int vpe_pp_init(AVFilterContext *avf_ctx) +{ + VpePPFilter *ctx = avf_ctx->priv; + int ret = 0; + AVFilterPad pad = { 0 }; + + ret = vpi_create(&ctx->ctx, &ctx->vpi, PP_VPE); + if (ret) + return AVERROR_EXTERNAL; + + ret = ctx->vpi->init(ctx->ctx, NULL); + if (ret) + return AVERROR_EXTERNAL; + + pad.type = AVMEDIA_TYPE_VIDEO; + pad.name = "output0"; + if ((ret = ff_insert_outpad(avf_ctx, 0, &pad)) < 0) { + return ret; + } + + return 0; +} + +static av_cold void vpe_pp_uninit(AVFilterContext *avf_ctx) +{ + VpePPFilter *ctx = avf_ctx->priv; + + if (ctx->hw_device) { + ctx->vpi->close(ctx->ctx); + av_buffer_unref(&ctx->hw_device); + vpi_destroy(ctx->ctx); + avf_ctx->priv = NULL; + } +} + +static void vpe_pp_picture_consumed(void *opaque, uint8_t *data) +{ + VpePPFilter *ctx = opaque; + VpiCtrlCmdParam cmd; + + cmd.cmd = VPI_CMD_PP_CONSUME; + cmd.data = data; + ctx->vpi->control(ctx->ctx, (void *)&cmd, NULL); + free(data); +} + +static int vpe_pp_output_avframe(VpePPFilter *ctx, VpiFrame *input, + AVFrame *output) +{ + AVHWFramesContext *hwframe_ctx = (AVHWFramesContext *)ctx->hw_frame->data; + AVVpeFramesContext *vpeframe_ctx = (AVVpeFramesContext *)hwframe_ctx->hwctx; + VpiFrame *frame_hwctx = vpeframe_ctx->frame; + + if (input) { + output->width = input->width; + output->height = input->height; + output->linesize[0] = input->linesize[0]; + output->linesize[1] = input->linesize[1]; + output->linesize[2] = input->linesize[2]; + output->key_frame = input->key_frame; + output->format = AV_PIX_FMT_VPE; + output->data[0] = (void*)input; + output->buf[0] = + av_buffer_create((uint8_t *)input, sizeof(VpiFrame), + vpe_pp_picture_consumed, (void *)ctx, + AV_BUFFER_FLAG_READONLY); + if (output->buf[0] == NULL) + return AVERROR(ENOMEM); + + memcpy(frame_hwctx, input, sizeof(VpiFrame)); + output->hw_frames_ctx = av_buffer_ref(ctx->hw_frame); + if (output->hw_frames_ctx == NULL) + return AVERROR(ENOMEM); + + } else { + memset(output, 0, sizeof(AVFrame)); + } + + return 0; +} + +static int vpe_pp_output_vpeframe(AVFrame *input, VpiFrame *output, + int max_frame_delay) +{ + memset(output, 0, sizeof(VpiFrame)); + if (input) { + output->width = input->width; + output->height = input->height; + output->linesize[0] = input->linesize[0]; + output->linesize[1] = input->linesize[1]; + output->linesize[2] = input->linesize[2]; + output->key_frame = input->key_frame; + output->pts = input->pts; + output->pkt_dts = input->pkt_dts; + output->data[0] = input->data[0]; + output->data[1] = input->data[1]; + output->data[2] = input->data[2]; + output->max_frames_delay = max_frame_delay; + } + + return 0; +} + +static int vpe_pp_filter_frame(AVFilterLink *inlink, AVFrame *frame) +{ + AVFilterContext *avf_ctx = inlink->dst; + AVFilterLink *outlink = avf_ctx->outputs[0]; + AVHWFramesContext *hwframe_ctx = NULL; + AVFrame *buf_out = NULL; + VpePPFilter *ctx = avf_ctx->priv; + VpiFrame in_picture, *out_picture; + AVVpeFramesContext *vpeframe_ctx = NULL; + int ret = 0; + int max_frame_delay = 0; + + hwframe_ctx = (AVHWFramesContext *)ctx->hw_frame->data; + vpeframe_ctx = (AVVpeFramesContext *)hwframe_ctx->hwctx; + max_frame_delay = vpeframe_ctx->frame->max_frames_delay; + ret = vpe_pp_output_vpeframe(frame, &in_picture, max_frame_delay); + if (ret) + return ret; + + out_picture = (VpiFrame *)malloc(sizeof(VpiFrame)); + ret = ctx->vpi->process(ctx->ctx, &in_picture, out_picture); + if (ret) + return AVERROR_EXTERNAL; + + buf_out = av_frame_alloc(); + if (!buf_out) + return AVERROR(ENOMEM); + + ret = av_frame_copy_props(buf_out, frame); + if (ret) + return ret; + + ret = vpe_pp_output_avframe(ctx, out_picture, buf_out); + if (ret < 0) + return AVERROR_EXTERNAL; + + av_frame_free(&frame); + + ret = ff_outlink_get_status(outlink); + if (ret < 0) + return ret; + + ret = ff_filter_frame(outlink, buf_out); + if (ret < 0) + return ret; + + return 0; +} + +static int vpe_pp_init_hwctx(AVFilterContext *ctx, AVFilterLink *inlink) +{ + AVHWFramesContext *hwframe_ctx; + int ret = 0; + VpePPFilter *filter = ctx->priv; + + if (ctx->hw_device_ctx) { + filter->hw_device = av_buffer_ref(ctx->hw_device_ctx); + if (!filter->hw_device) + return AVERROR(ENOMEM); + } else { + return AVERROR(ENOMEM); + } + + filter->hw_frame = av_hwframe_ctx_alloc(filter->hw_device); + if (!filter->hw_frame) + return AVERROR(ENOMEM); + + hwframe_ctx = (AVHWFramesContext *)filter->hw_frame->data; + if (!hwframe_ctx->pool) { + hwframe_ctx->format = AV_PIX_FMT_VPE; + hwframe_ctx->sw_format = inlink->format; + hwframe_ctx->width = inlink->w; + hwframe_ctx->height = inlink->h; + + if ((ret = av_hwframe_ctx_init(filter->hw_frame)) < 0) { + return ret; + } + } + inlink->hw_frames_ctx = filter->hw_frame; + return 0; +} + +static int vpe_get_format(enum AVPixelFormat format) +{ + int i = 0; + + for (i = 0; i < sizeof(ptable) / sizeof(PixelMapTable); i++) { + if (format == ptable[i].src) + return ptable[i].des; + } + return AVERROR(EINVAL); +} + +static int vpe_pp_config_props(AVFilterLink *inlink) +{ + AVFilterContext *avf_ctx = inlink->dst; + AVHWFramesContext *hwframe_ctx; + AVVpeFramesContext *vpeframe_ctx; + VpePPFilter *ctx = avf_ctx->priv; + VpiPPOpition *cfg = &ctx->cfg; + VpiCtrlCmdParam cmd; + int ret = 0; + + ret = vpe_pp_init_hwctx(avf_ctx, inlink); + if (ret < 0){ + return AVERROR_EXTERNAL; + } + + hwframe_ctx = (AVHWFramesContext *)ctx->hw_frame->data; + vpeframe_ctx = (AVVpeFramesContext *)hwframe_ctx->hwctx; + /*Get config*/ + cfg->w = inlink->w; + cfg->h = inlink->h; + cfg->format = vpe_get_format(inlink->format); + cfg->nb_outputs = ctx->nb_outputs; + cfg->force_10bit = ctx->force_10bit; + cfg->low_res = ctx->low_res; + cfg->frame = vpeframe_ctx->frame; + + cmd.cmd = VPI_CMD_PP_CONFIG; + cmd.data = cfg; + ret = ctx->vpi->control(ctx->ctx, (void *)&cmd, NULL); + if (ret < 0){ + return AVERROR_EXTERNAL; + } + return 0; +} + +static int vpe_pp_query_formats(AVFilterContext *avf_ctx) +{ + int ret; + AVFilterFormats *in_fmts = ff_make_format_list(input_pix_fmts); + AVFilterFormats *out_fmts; + + ret = ff_formats_ref(in_fmts, &avf_ctx->inputs[0]->out_formats); + if (ret < 0){ + av_log(NULL, AV_LOG_ERROR, "ff_formats_ref error=%d\n", ret); + return ret; + } + + out_fmts = ff_make_format_list(output_pix_fmts); + ret = ff_formats_ref(out_fmts, &avf_ctx->outputs[0]->in_formats); + + return ret; +} + +#define OFFSET(x) offsetof(VpePPFilter, x) +#define FLAGS (AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_FILTERING_PARAM) + +static const AVOption vpe_pp_options[] = { + { "outputs", + "set number of outputs", + OFFSET(nb_outputs), + AV_OPT_TYPE_INT, + { .i64 = 1 }, + 1, + 4, + FLAGS }, + { "low_res", + "specific resize configuration.", + OFFSET(low_res), + AV_OPT_TYPE_STRING, + { .str = NULL }, + .flags = FLAGS }, + { "force10bit", + "upsampling 8bit to 10bit", + OFFSET(force_10bit), + AV_OPT_TYPE_INT, + { .i64 = 0 }, + 0, + 1, + FLAGS }, + { NULL }, +}; + +AVFILTER_DEFINE_CLASS(vpe_pp); + +static const AVFilterPad vpe_pp_inputs[] = { + { + .name = "default", + .type = AVMEDIA_TYPE_VIDEO, + .filter_frame = vpe_pp_filter_frame, + .config_props = vpe_pp_config_props, + }, + { NULL } +}; + +AVFilter ff_vf_pp_vpe = { + .name = "vpe_pp", + .description = NULL_IF_CONFIG_SMALL("Filter using vpe post processing."), + .priv_size = sizeof(VpePPFilter), + .priv_class = &vpe_pp_class, + .init = vpe_pp_init, + .uninit = vpe_pp_uninit, + .query_formats = vpe_pp_query_formats, + .inputs = vpe_pp_inputs, + .outputs = NULL, + .flags = 0, +};