From patchwork Tue Dec 22 07:44:58 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Guo, Yejun" X-Patchwork-Id: 24615 Return-Path: X-Original-To: patchwork@ffaux-bg.ffmpeg.org Delivered-To: patchwork@ffaux-bg.ffmpeg.org Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org [79.124.17.100]) by ffaux.localdomain (Postfix) with ESMTP id 5F30B44A63F for ; Tue, 22 Dec 2020 09:46:11 +0200 (EET) Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 4129468A8A0; Tue, 22 Dec 2020 09:46:11 +0200 (EET) X-Original-To: ffmpeg-devel@ffmpeg.org Delivered-To: ffmpeg-devel@ffmpeg.org Received: from mga02.intel.com (mga02.intel.com [134.134.136.20]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id 6E3CF68A7D4 for ; Tue, 22 Dec 2020 09:46:04 +0200 (EET) IronPort-SDR: ZP8/fLBa6MzW/ggRbb+AUTGK0awXu1cGvPE7vIn/R1OoGGCa/GYhNTMN9qP/ORtsF1KanjKAgG 3VuBHy+tfdSw== X-IronPort-AV: E=McAfee;i="6000,8403,9842"; a="162888179" X-IronPort-AV: E=Sophos;i="5.78,438,1599548400"; d="scan'208";a="162888179" Received: from fmsmga005.fm.intel.com ([10.253.24.32]) by orsmga101.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 21 Dec 2020 23:45:58 -0800 IronPort-SDR: nL8rww2IJVYdeJxcko+ArOh8Jnn73gSiFNQM/bm7kbAQeWvO0Q/828vJ43Js9wLc7tOTkvdDrr /TW8UuvOrVTg== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.78,438,1599548400"; d="scan'208";a="564200330" Received: from yguo18-skl-u1604.sh.intel.com (HELO localhost.localdomain) ([10.239.159.53]) by fmsmga005.fm.intel.com with ESMTP; 21 Dec 2020 23:45:57 -0800 From: "Guo, Yejun" To: ffmpeg-devel@ffmpeg.org Date: Tue, 22 Dec 2020 15:44:58 +0800 Message-Id: <20201222074458.6251-1-yejun.guo@intel.com> X-Mailer: git-send-email 2.17.1 Subject: [FFmpeg-devel] [PATCH 3/8] dnn_backend_openvino.c: separate function execute_model_ov X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Cc: yejun.guo@intel.com MIME-Version: 1.0 Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" function fill_model_input_ov and infer_completion_callback are extracted, it will help the async execution for reuse. Signed-off-by: Xie, Lin Signed-off-by: Wu Zhiwen Signed-off-by: Guo, Yejun --- libavfilter/dnn/dnn_backend_openvino.c | 292 +++++++++++++++---------- 1 file changed, 175 insertions(+), 117 deletions(-) diff --git a/libavfilter/dnn/dnn_backend_openvino.c b/libavfilter/dnn/dnn_backend_openvino.c index d510e162c6..1196db0c90 100644 --- a/libavfilter/dnn/dnn_backend_openvino.c +++ b/libavfilter/dnn/dnn_backend_openvino.c @@ -50,6 +50,21 @@ typedef struct OVModel{ ie_infer_request_t *infer_request; } OVModel; +typedef struct TaskItem { + OVModel *ov_model; + const char *input_name; + AVFrame *in_frame; + const char *output_name; + AVFrame *out_frame; + int do_ioproc; + int done; +} TaskItem; + +typedef struct RequestItem { + ie_infer_request_t *infer_request; + TaskItem *task; +} RequestItem; + #define APPEND_STRING(generated_string, iterate_string) \ generated_string = generated_string ? av_asprintf("%s %s", generated_string, iterate_string) : \ av_asprintf("%s", iterate_string); @@ -63,10 +78,6 @@ static const AVOption dnn_openvino_options[] = { AVFILTER_DEFINE_CLASS(dnn_openvino); -static DNNReturnType execute_model_ov(const DNNModel *model, const char *input_name, AVFrame *in_frame, - const char **output_names, uint32_t nb_output, AVFrame *out_frame, - int do_ioproc); - static DNNDataType precision_to_datatype(precision_e precision) { switch (precision) @@ -79,6 +90,136 @@ static DNNDataType precision_to_datatype(precision_e precision) } } +static DNNReturnType fill_model_input_ov(OVModel *ov_model, TaskItem *task, RequestItem *request) +{ + dimensions_t dims; + precision_e precision; + ie_blob_buffer_t blob_buffer; + OVContext *ctx = &ov_model->ctx; + IEStatusCode status; + DNNData input; + ie_blob_t *input_blob = NULL; + + status = ie_infer_request_get_blob(request->infer_request, task->input_name, &input_blob); + if (status != OK) { + av_log(ctx, AV_LOG_ERROR, "Failed to get input blob with name %s\n", task->input_name); + return DNN_ERROR; + } + + status |= ie_blob_get_dims(input_blob, &dims); + status |= ie_blob_get_precision(input_blob, &precision); + if (status != OK) { + av_log(ctx, AV_LOG_ERROR, "Failed to get input blob dims/precision\n"); + return DNN_ERROR; + } + + status = ie_blob_get_buffer(input_blob, &blob_buffer); + if (status != OK) { + av_log(ctx, AV_LOG_ERROR, "Failed to get input blob buffer\n"); + return DNN_ERROR; + } + + input.height = dims.dims[2]; + input.width = dims.dims[3]; + input.channels = dims.dims[1]; + input.data = blob_buffer.buffer; + input.dt = precision_to_datatype(precision); + if (task->do_ioproc) { + if (ov_model->model->pre_proc != NULL) { + ov_model->model->pre_proc(task->in_frame, &input, ov_model->model->userdata); + } else { + proc_from_frame_to_dnn(task->in_frame, &input, ctx); + } + } + ie_blob_free(&input_blob); + + return DNN_SUCCESS; +} + +static void infer_completion_callback(void *args) +{ + dimensions_t dims; + precision_e precision; + IEStatusCode status; + RequestItem *request = args; + TaskItem *task = request->task; + ie_blob_t *output_blob = NULL; + ie_blob_buffer_t blob_buffer; + DNNData output; + OVContext *ctx = &task->ov_model->ctx; + + status = ie_infer_request_get_blob(request->infer_request, task->output_name, &output_blob); + if (status != OK) { + //incorrect output name + char *model_output_name = NULL; + char *all_output_names = NULL; + size_t model_output_count = 0; + av_log(ctx, AV_LOG_ERROR, "Failed to get model output data\n"); + status = ie_network_get_outputs_number(task->ov_model->network, &model_output_count); + for (size_t i = 0; i < model_output_count; i++) { + status = ie_network_get_output_name(task->ov_model->network, i, &model_output_name); + APPEND_STRING(all_output_names, model_output_name) + } + av_log(ctx, AV_LOG_ERROR, + "output \"%s\" may not correct, all output(s) are: \"%s\"\n", + task->output_name, all_output_names); + return; + } + + status = ie_blob_get_buffer(output_blob, &blob_buffer); + if (status != OK) { + av_log(ctx, AV_LOG_ERROR, "Failed to access output memory\n"); + return; + } + + status |= ie_blob_get_dims(output_blob, &dims); + status |= ie_blob_get_precision(output_blob, &precision); + if (status != OK) { + av_log(ctx, AV_LOG_ERROR, "Failed to get dims or precision of output\n"); + return; + } + + output.channels = dims.dims[1]; + output.height = dims.dims[2]; + output.width = dims.dims[3]; + output.dt = precision_to_datatype(precision); + output.data = blob_buffer.buffer; + if (task->do_ioproc) { + if (task->ov_model->model->post_proc != NULL) { + task->ov_model->model->post_proc(task->out_frame, &output, task->ov_model->model->userdata); + } else { + proc_from_dnn_to_frame(task->out_frame, &output, ctx); + } + } else { + task->out_frame->width = output.width; + task->out_frame->height = output.height; + } + ie_blob_free(&output_blob); + task->done = 1; +} + +static DNNReturnType execute_model_ov(TaskItem *task, RequestItem *request) +{ + IEStatusCode status; + OVContext *ctx = &task->ov_model->ctx; + + DNNReturnType ret = fill_model_input_ov(task->ov_model, task, request); + if (ret != DNN_SUCCESS) { + return ret; + } + + status = ie_infer_request_infer(request->infer_request); + if (status != OK) { + av_log(ctx, AV_LOG_ERROR, "Failed to start synchronous model inference\n"); + return DNN_ERROR; + } + + request->task = task; + infer_completion_callback(request); + + return task->done ? DNN_SUCCESS : DNN_ERROR; +} + static DNNReturnType get_input_ov(void *model, DNNData *input, const char *input_name) { OVModel *ov_model = (OVModel *)model; @@ -142,6 +283,8 @@ static DNNReturnType get_output_ov(void *model, const char *input_name, int inpu DNNReturnType ret; OVModel *ov_model = (OVModel *)model; OVContext *ctx = &ov_model->ctx; + TaskItem task; + RequestItem request; AVFrame *in_frame = av_frame_alloc(); AVFrame *out_frame = NULL; @@ -158,7 +301,17 @@ static DNNReturnType get_output_ov(void *model, const char *input_name, int inpu in_frame->width = input_width; in_frame->height = input_height; - ret = execute_model_ov(ov_model->model, input_name, in_frame, &output_name, 1, out_frame, 0); + task.done = 0; + task.do_ioproc = 0; + task.input_name = input_name; + task.in_frame = in_frame; + task.output_name = output_name; + task.out_frame = out_frame; + task.ov_model = ov_model; + + request.infer_request = ov_model->infer_request; + + ret = execute_model_ov(&task, &request); *output_width = out_frame->width; *output_height = out_frame->height; @@ -249,55 +402,24 @@ err: return NULL; } -static DNNReturnType execute_model_ov(const DNNModel *model, const char *input_name, AVFrame *in_frame, - const char **output_names, uint32_t nb_output, AVFrame *out_frame, - int do_ioproc) +DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, const char *input_name, AVFrame *in_frame, + const char **output_names, uint32_t nb_output, AVFrame *out_frame) { - char *model_output_name = NULL; - char *all_output_names = NULL; - dimensions_t dims; - precision_e precision; - ie_blob_buffer_t blob_buffer; OVModel *ov_model = (OVModel *)model->model; OVContext *ctx = &ov_model->ctx; - IEStatusCode status; - size_t model_output_count = 0; - DNNData input, output; - ie_blob_t *input_blob = NULL; + TaskItem task; + RequestItem request; - status = ie_infer_request_get_blob(ov_model->infer_request, input_name, &input_blob); - if (status != OK) { - av_log(ctx, AV_LOG_ERROR, "Failed to get input blob\n"); - return DNN_ERROR; - } - - status |= ie_blob_get_dims(input_blob, &dims); - status |= ie_blob_get_precision(input_blob, &precision); - if (status != OK) { - av_log(ctx, AV_LOG_ERROR, "Failed to get input blob dims/precision\n"); + if (!in_frame) { + av_log(ctx, AV_LOG_ERROR, "in frame is NULL when execute model.\n"); return DNN_ERROR; } - status = ie_blob_get_buffer(input_blob, &blob_buffer); - if (status != OK) { - av_log(ctx, AV_LOG_ERROR, "Failed to get input blob buffer\n"); + if (!out_frame) { + av_log(ctx, AV_LOG_ERROR, "out frame is NULL when execute model.\n"); return DNN_ERROR; } - input.height = dims.dims[2]; - input.width = dims.dims[3]; - input.channels = dims.dims[1]; - input.data = blob_buffer.buffer; - input.dt = precision_to_datatype(precision); - if (do_ioproc) { - if (ov_model->model->pre_proc != NULL) { - ov_model->model->pre_proc(in_frame, &input, ov_model->model->userdata); - } else { - proc_from_frame_to_dnn(in_frame, &input, ctx); - } - } - ie_blob_free(&input_blob); - if (nb_output != 1) { // currently, the filter does not need multiple outputs, // so we just pending the support until we really need it. @@ -305,81 +427,17 @@ static DNNReturnType execute_model_ov(const DNNModel *model, const char *input_n return DNN_ERROR; } - status = ie_infer_request_infer(ov_model->infer_request); - if (status != OK) { - av_log(ctx, AV_LOG_ERROR, "Failed to start synchronous model inference\n"); - return DNN_ERROR; - } - - for (uint32_t i = 0; i < nb_output; ++i) { - const char *output_name = output_names[i]; - ie_blob_t *output_blob = NULL; - status = ie_infer_request_get_blob(ov_model->infer_request, output_name, &output_blob); - if (status != OK) { - //incorrect output name - av_log(ctx, AV_LOG_ERROR, "Failed to get model output data\n"); - status = ie_network_get_outputs_number(ov_model->network, &model_output_count); - for (size_t i = 0; i < model_output_count; i++) { - status = ie_network_get_output_name(ov_model->network, i, &model_output_name); - APPEND_STRING(all_output_names, model_output_name) - } - av_log(ctx, AV_LOG_ERROR, - "output \"%s\" may not correct, all output(s) are: \"%s\"\n", - output_name, all_output_names); - return DNN_ERROR; - } - - status = ie_blob_get_buffer(output_blob, &blob_buffer); - if (status != OK) { - av_log(ctx, AV_LOG_ERROR, "Failed to access output memory\n"); - return DNN_ERROR; - } - - status |= ie_blob_get_dims(output_blob, &dims); - status |= ie_blob_get_precision(output_blob, &precision); - if (status != OK) { - av_log(ctx, AV_LOG_ERROR, "Failed to get dims or precision of output\n"); - return DNN_ERROR; - } - - output.channels = dims.dims[1]; - output.height = dims.dims[2]; - output.width = dims.dims[3]; - output.dt = precision_to_datatype(precision); - output.data = blob_buffer.buffer; - if (do_ioproc) { - if (ov_model->model->post_proc != NULL) { - ov_model->model->post_proc(out_frame, &output, ov_model->model->userdata); - } else { - proc_from_dnn_to_frame(out_frame, &output, ctx); - } - } else { - out_frame->width = output.width; - out_frame->height = output.height; - } - ie_blob_free(&output_blob); - } + task.done = 0; + task.do_ioproc = 1; + task.input_name = input_name; + task.in_frame = in_frame; + task.output_name = output_names[0]; + task.out_frame = out_frame; + task.ov_model = ov_model; - return DNN_SUCCESS; -} - -DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, const char *input_name, AVFrame *in_frame, - const char **output_names, uint32_t nb_output, AVFrame *out_frame) -{ - OVModel *ov_model = (OVModel *)model->model; - OVContext *ctx = &ov_model->ctx; - - if (!in_frame) { - av_log(ctx, AV_LOG_ERROR, "in frame is NULL when execute model.\n"); - return DNN_ERROR; - } - - if (!out_frame) { - av_log(ctx, AV_LOG_ERROR, "out frame is NULL when execute model.\n"); - return DNN_ERROR; - } + request.infer_request = ov_model->infer_request; - return execute_model_ov(model, input_name, in_frame, output_names, nb_output, out_frame, 1); + return execute_model_ov(&task, &request); } void ff_dnn_free_model_ov(DNNModel **model)