From patchwork Mon Apr 8 08:53:33 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: Zachary Zhou X-Patchwork-Id: 12640 Return-Path: X-Original-To: patchwork@ffaux-bg.ffmpeg.org Delivered-To: patchwork@ffaux-bg.ffmpeg.org Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org [79.124.17.100]) by ffaux.localdomain (Postfix) with ESMTP id 0E1BE449305 for ; Mon, 8 Apr 2019 11:54:17 +0300 (EEST) Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id EBFD268AB0A; Mon, 8 Apr 2019 11:54:16 +0300 (EEST) X-Original-To: ffmpeg-devel@ffmpeg.org Delivered-To: ffmpeg-devel@ffmpeg.org Received: from mga14.intel.com (mga14.intel.com [192.55.52.115]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id 0E99C68A6A5 for ; Mon, 8 Apr 2019 11:54:09 +0300 (EEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga006.fm.intel.com ([10.253.24.20]) by fmsmga103.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 08 Apr 2019 01:54:08 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.60,324,1549958400"; d="scan'208";a="334782928" Received: from jiandon-desk.sh.intel.com ([10.239.193.48]) by fmsmga006.fm.intel.com with ESMTP; 08 Apr 2019 01:54:06 -0700 From: Zachary Zhou To: ffmpeg-devel@ffmpeg.org Date: Mon, 8 Apr 2019 16:53:33 +0800 Message-Id: <20190408085334.23947-2-zachary.zhou@intel.com> X-Mailer: git-send-email 2.17.1 In-Reply-To: <20190408085334.23947-1-zachary.zhou@intel.com> References: <20190408085334.23947-1-zachary.zhou@intel.com> MIME-Version: 1.0 Subject: [FFmpeg-devel] [PATCH RFC v2 2/3] libavcodec: Add thumbnail output to vaapi_h264 decoder X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Cc: zachary.zhou@intel.com Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" This is sample code for reference HW support for decode+scaling in a single HW command (VDBOX+SFC). The primary target usage is video analytics, but can be used playback, transcoding, etc. For VAAPI - https://github.com/intel/libva basically, it allows multiple outputs (in different resolutions) using the decode context in a single call (you can search for “additional_outputs” in va.h). VAAPI sample code - https://github.com/intel/libva-utils/commit/957a269f02b00760b7e807643c821ee26abc529b --- libavcodec/avcodec.h | 8 +++ libavcodec/decode.c | 16 +++++ libavcodec/options_table.h | 4 ++ libavcodec/vaapi_decode.c | 122 ++++++++++++++++++++++++++++++++++--- libavcodec/vaapi_decode.h | 30 +++++++++ libavcodec/vaapi_h264.c | 13 ++++ 6 files changed, 185 insertions(+), 8 deletions(-) diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h index 0ce22ec4fa..36db21c0a5 100644 --- a/libavcodec/avcodec.h +++ b/libavcodec/avcodec.h @@ -3357,6 +3357,14 @@ typedef struct AVCodecContext { * - encoding: unused */ int discard_damaged_percentage; + + /* + * Thumbnail options + */ + int thumbnail_flags; + int thumbnail_format; + int thumbnail_width; + int thumbnail_height; } AVCodecContext; #if FF_API_CODEC_GET_SET diff --git a/libavcodec/decode.c b/libavcodec/decode.c index a32ff2fcd3..2107751197 100644 --- a/libavcodec/decode.c +++ b/libavcodec/decode.c @@ -1795,6 +1795,22 @@ FF_ENABLE_DEPRECATION_WARNINGS frame->channels = avctx->channels; break; } + + //thumbnail side data + if (avctx->thumbnail_flags) { + AVFrameSideData *frame_thumbnail_sd = av_frame_new_side_data(frame, + AV_FRAME_DATA_THUMBNAIL_INFO, + sizeof(AVThumbnailInfo)); + if (frame_thumbnail_sd) { + av_dict_set_int(&frame_thumbnail_sd->metadata, "thumbnail_flags", avctx->thumbnail_flags, 0); + av_dict_set_int(&frame_thumbnail_sd->metadata, "thumbnail_width", avctx->thumbnail_width, 0); + av_dict_set_int(&frame_thumbnail_sd->metadata, "thumbnail_height", avctx->thumbnail_height, 0); + av_dict_set_int(&frame_thumbnail_sd->metadata, "thumbnail_format", avctx->thumbnail_format, 0); + + av_log(avctx, AV_LOG_DEBUG, "Thumbnail new side data\n"); + } + } + return 0; } diff --git a/libavcodec/options_table.h b/libavcodec/options_table.h index a3235bcd57..aa158f713c 100644 --- a/libavcodec/options_table.h +++ b/libavcodec/options_table.h @@ -480,6 +480,10 @@ static const AVOption avcodec_options[] = { {"allow_profile_mismatch", "attempt to decode anyway if HW accelerated decoder's supported profiles do not exactly match the stream", 0, AV_OPT_TYPE_CONST, {.i64 = AV_HWACCEL_FLAG_ALLOW_PROFILE_MISMATCH }, INT_MIN, INT_MAX, V | D, "hwaccel_flags"}, {"extra_hw_frames", "Number of extra hardware frames to allocate for the user", OFFSET(extra_hw_frames), AV_OPT_TYPE_INT, { .i64 = -1 }, -1, INT_MAX, V|D }, {"discard_damaged_percentage", "Percentage of damaged samples to discard a frame", OFFSET(discard_damaged_percentage), AV_OPT_TYPE_INT, {.i64 = 95 }, 0, 100, V|D }, +{"thumbnail_flags", "set thumbnail flags", OFFSET(thumbnail_flags), AV_OPT_TYPE_INT, { .i64 = 1 }, 0, 1, V|D, "thumbnail"}, +{"thumbnail_format", "set thumbnail format", OFFSET(thumbnail_format), AV_OPT_TYPE_PIXEL_FMT, {.i64=AV_PIX_FMT_ARGB}, -1, INT_MAX, V|D, "thumbnail"}, +{"thumbnail_width", "set thumbnail width", OFFSET(thumbnail_width), AV_OPT_TYPE_INT, {.i64 = 480}, 0, INT_MAX, V|D, "thumbnail"}, +{"thumbnail_height", "set thumbnail height", OFFSET(thumbnail_height), AV_OPT_TYPE_INT, {.i64 = 360}, 0, INT_MAX, V|D, "thumbnail"}, {NULL}, }; diff --git a/libavcodec/vaapi_decode.c b/libavcodec/vaapi_decode.c index 69512e1d45..2e9e762dc0 100644 --- a/libavcodec/vaapi_decode.c +++ b/libavcodec/vaapi_decode.c @@ -145,6 +145,17 @@ static void ff_vaapi_decode_destroy_buffers(AVCodecContext *avctx, pic->slice_buffers[i], vas, vaErrorStr(vas)); } } + + //destory thumbnail buffer + if (avctx->thumbnail_flags) { + vas = vaDestroyBuffer(ctx->hwctx->display, + pic->thumbnail_buffer); + if (vas != VA_STATUS_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Failed to destroy thumbnail " + "buffer %#x: %d (%s).\n", + pic->thumbnail_buffer, vas, vaErrorStr(vas)); + } + } } int ff_vaapi_decode_issue(AVCodecContext *avctx, @@ -184,6 +195,56 @@ int ff_vaapi_decode_issue(AVCodecContext *avctx, goto fail_with_picture; } + //add thumbnail buffer + if (avctx->thumbnail_flags) { + VARectangle rect_src; /**< @brief Rectangle for source input */ + VARectangle rect_thumbnail; /**< @brief Rectangle for thumbnail output */ + VAProcPipelineParameterBuffer buffer; + + memset(&rect_src, 0, sizeof(rect_src)); + memset(&rect_thumbnail, 0, sizeof(rect_thumbnail)); + memset(&buffer, 0, sizeof(buffer)); + + rect_src.x = rect_src.y = 0; + rect_src.width = (uint16_t)pic->thumbnail_src_width; + rect_src.height = (uint16_t)pic->thumbnail_src_height; + + rect_thumbnail.x = rect_src.y = 0; + rect_thumbnail.width = (uint16_t)avctx->thumbnail_width; + rect_thumbnail.height = (uint16_t)avctx->thumbnail_height; + + buffer.surface_region = &rect_src; + buffer.output_region = &rect_thumbnail; + buffer.additional_outputs = (VASurfaceID*)&(pic->thumbnail_output_surface); + buffer.num_additional_outputs = 1; + + vas = vaCreateBuffer(ctx->hwctx->display, ctx->va_context, + VAProcPipelineParameterBufferType, + sizeof(buffer), + 1, + (uint8_t*)&buffer, + &pic->thumbnail_buffer); + if (vas != VA_STATUS_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Failed to create thumbnail buffer: " + "%d (%s).\n", vas, vaErrorStr(vas)); + err = AVERROR(EIO); + goto fail_with_picture; + } + + av_log(avctx, AV_LOG_DEBUG, "H264 create thumbnail buffer.\n"); + + vas = vaRenderPicture(ctx->hwctx->display, ctx->va_context, + &pic->thumbnail_buffer, 1); + if (vas != VA_STATUS_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Failed to upload sfc: " + "%d (%s).\n", vas, vaErrorStr(vas)); + err = AVERROR(EIO); + goto fail_with_picture; + } + + av_log(avctx, AV_LOG_DEBUG, "H264 render thumbnail buffer.\n"); + } + vas = vaEndPicture(ctx->hwctx->display, ctx->va_context); if (vas != VA_STATUS_SUCCESS) { av_log(avctx, AV_LOG_ERROR, "Failed to end picture decode " @@ -493,14 +554,49 @@ static int vaapi_decode_make_config(AVCodecContext *avctx, } } - vas = vaCreateConfig(hwctx->display, matched_va_profile, - VAEntrypointVLD, NULL, 0, - va_config); - if (vas != VA_STATUS_SUCCESS) { - av_log(avctx, AV_LOG_ERROR, "Failed to create decode " - "configuration: %d (%s).\n", vas, vaErrorStr(vas)); - err = AVERROR(EIO); - goto fail; + if (avctx->thumbnail_flags) { + VAConfigAttrib attrib; + + memset(&attrib, 0, sizeof(attrib)); + attrib.type = VAConfigAttribDecProcessing; + attrib.value = 0; + + vaGetConfigAttributes( + hwctx->display, + matched_va_profile, + VAEntrypointVLD, + &attrib, + 1); + + if (attrib.value != VA_DEC_PROCESSING) { + err = AVERROR(EINVAL); + goto fail; + } + + av_log(avctx, AV_LOG_DEBUG, "vaapi driver supports VDSFC\n"); + + attrib.type = VAConfigAttribDecProcessing; + attrib.value = VA_DEC_PROCESSING; + + vas = vaCreateConfig(hwctx->display, matched_va_profile, + VAEntrypointVLD, &attrib, 1, + va_config); + if (vas != VA_STATUS_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Failed to create decode " + "configuration: %d (%s).\n", vas, vaErrorStr(vas)); + err = AVERROR(EIO); + goto fail; + } + } else { + vas = vaCreateConfig(hwctx->display, matched_va_profile, + VAEntrypointVLD, NULL, 0, + va_config); + if (vas != VA_STATUS_SUCCESS) { + av_log(avctx, AV_LOG_ERROR, "Failed to create decode " + "configuration: %d (%s).\n", vas, vaErrorStr(vas)); + err = AVERROR(EIO); + goto fail; + } } hwconfig = av_hwdevice_hwconfig_alloc(device_ref); @@ -587,6 +683,7 @@ int ff_vaapi_common_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx) { AVHWFramesContext *hw_frames = (AVHWFramesContext *)hw_frames_ctx->data; + AVVAAPIFramesContext *avfc = hw_frames->hwctx; AVHWDeviceContext *device_ctx = hw_frames->device_ctx; AVVAAPIDeviceContext *hwctx; VAConfigID va_config = VA_INVALID_ID; @@ -604,6 +701,15 @@ int ff_vaapi_common_frame_params(AVCodecContext *avctx, if (va_config != VA_INVALID_ID) vaDestroyConfig(hwctx->display, va_config); + if (avctx->thumbnail_flags) { + avfc->thumbnail_flags = avctx->thumbnail_flags; + avfc->thumbnail_format = avctx->thumbnail_format; + avfc->thumbnail_width = avctx->thumbnail_width; + avfc->thumbnail_height = avctx->thumbnail_height; + + av_log(avctx, AV_LOG_DEBUG, "VDSFC save thumbnail info avfc\n"); + } + return 0; } diff --git a/libavcodec/vaapi_decode.h b/libavcodec/vaapi_decode.h index 6b415dd1d3..f70658153a 100644 --- a/libavcodec/vaapi_decode.h +++ b/libavcodec/vaapi_decode.h @@ -37,6 +37,29 @@ static inline VASurfaceID ff_vaapi_get_surface_id(AVFrame *pic) return (uintptr_t)pic->data[3]; } +static inline VASurfaceID ff_vaapi_get_thumbnail_surface_id(AVFrame *pic) +{ + VASurfaceID thumbnail_surface_id = VA_INVALID_ID; + + AVFrameSideData *frame_thumbnail_sd = av_frame_get_side_data(pic, + AV_FRAME_DATA_THUMBNAIL_INFO); + if (frame_thumbnail_sd) { + thumbnail_surface_id = (uintptr_t)frame_thumbnail_sd->buf->data; + } + + return thumbnail_surface_id; +} + +static inline int ff_vaapi_get_thumbnail_src_width(AVFrame *pic) +{ + return pic->width; +} + +static inline int ff_vaapi_get_thumbnail_src_height(AVFrame *pic) +{ + return pic->height; +} + enum { MAX_PARAM_BUFFERS = 16, }; @@ -50,6 +73,13 @@ typedef struct VAAPIDecodePicture { int nb_slices; VABufferID *slice_buffers; int slices_allocated; + + //thumbnail info + VASurfaceID thumbnail_output_surface; + int thumbnail_src_width; + int thumbnail_src_height; + VABufferID thumbnail_buffer; + } VAAPIDecodePicture; typedef struct VAAPIDecodeContext { diff --git a/libavcodec/vaapi_h264.c b/libavcodec/vaapi_h264.c index 5854587a25..11919cafa4 100644 --- a/libavcodec/vaapi_h264.c +++ b/libavcodec/vaapi_h264.c @@ -302,6 +302,19 @@ static int vaapi_h264_start_frame(AVCodecContext *avctx, if (err < 0) goto fail; + //get thumbnail surface id + if (avctx->thumbnail_flags) { + pic->thumbnail_output_surface = ff_vaapi_get_thumbnail_surface_id(h->cur_pic_ptr->f); + + //get thumbnail src width and height + pic->thumbnail_src_width = ff_vaapi_get_thumbnail_src_width(h->cur_pic_ptr->f); + pic->thumbnail_src_height = ff_vaapi_get_thumbnail_src_height(h->cur_pic_ptr->f); + + av_log(avctx, AV_LOG_DEBUG, "H264 get thumbnail surface id: %d\n", pic->thumbnail_output_surface); + av_log(avctx, AV_LOG_DEBUG, "H264 get thumbnail width: %d\n", pic->thumbnail_src_width); + av_log(avctx, AV_LOG_DEBUG, "H264 get thumbnail height: %d\n", pic->thumbnail_src_height); + } + return 0; fail: