From patchwork Tue Dec 22 07:46:37 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Guo, Yejun" X-Patchwork-Id: 24618 Return-Path: X-Original-To: patchwork@ffaux-bg.ffmpeg.org Delivered-To: patchwork@ffaux-bg.ffmpeg.org Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org [79.124.17.100]) by ffaux.localdomain (Postfix) with ESMTP id 0C32544A8D7 for ; Tue, 22 Dec 2020 09:47:47 +0200 (EET) Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id E633168A98A; Tue, 22 Dec 2020 09:47:46 +0200 (EET) X-Original-To: ffmpeg-devel@ffmpeg.org Delivered-To: ffmpeg-devel@ffmpeg.org Received: from mga01.intel.com (mga01.intel.com [192.55.52.88]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id D394C68A882 for ; Tue, 22 Dec 2020 09:47:39 +0200 (EET) IronPort-SDR: CEUNfHxyFWip2oeoqRLBZkNnTWxvoIrb2QpX6r8Fv4h/+ygtlQqOM0p2q9z+6EJgJxFc0Sl3wV FPL3fLanbfeg== X-IronPort-AV: E=McAfee;i="6000,8403,9842"; a="194271642" X-IronPort-AV: E=Sophos;i="5.78,438,1599548400"; d="scan'208";a="194271642" Received: from fmsmga005.fm.intel.com ([10.253.24.32]) by fmsmga101.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 21 Dec 2020 23:47:38 -0800 IronPort-SDR: hIJ2N19Cf4mbWrd21Qk0zeRrJySFizbUjkZLj0E8bMxRposisVpcdPdFCkLtaDJlf5ffPySswD FUzOS6MsA7LQ== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.78,438,1599548400"; d="scan'208";a="564205577" Received: from yguo18-skl-u1604.sh.intel.com (HELO localhost.localdomain) ([10.239.159.53]) by fmsmga005.fm.intel.com with ESMTP; 21 Dec 2020 23:47:36 -0800 From: "Guo, Yejun" To: ffmpeg-devel@ffmpeg.org Date: Tue, 22 Dec 2020 15:46:37 +0800 Message-Id: <20201222074637.6408-1-yejun.guo@intel.com> X-Mailer: git-send-email 2.17.1 Subject: [FFmpeg-devel] [PATCH 6/8] dnn: add async execution support for openvino backend X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Cc: yejun.guo@intel.com MIME-Version: 1.0 Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" Signed-off-by: Xie, Lin Signed-off-by: Wu Zhiwen Signed-off-by: Guo, Yejun --- libavfilter/dnn/dnn_backend_openvino.c | 162 +++++++++++++++++++++++-- libavfilter/dnn/dnn_backend_openvino.h | 3 + libavfilter/dnn/dnn_interface.c | 2 + libavfilter/dnn_interface.h | 2 +- 4 files changed, 159 insertions(+), 10 deletions(-) diff --git a/libavfilter/dnn/dnn_backend_openvino.c b/libavfilter/dnn/dnn_backend_openvino.c index da6e640226..8c3ba8a6a8 100644 --- a/libavfilter/dnn/dnn_backend_openvino.c +++ b/libavfilter/dnn/dnn_backend_openvino.c @@ -30,10 +30,13 @@ #include "libavutil/opt.h" #include "libavutil/avstring.h" #include "../internal.h" +#include "queue.h" +#include "safe_queue.h" #include typedef struct OVOptions{ char *device_type; + int nireq; } OVOptions; typedef struct OVContext { @@ -48,6 +51,10 @@ typedef struct OVModel{ ie_network_t *network; ie_executable_network_t *exe_network; ie_infer_request_t *infer_request; + + /* for async execution */ + safe_queue *request_queue; // holds RequestItem + queue *task_queue; // holds TaskItem } OVModel; typedef struct TaskItem { @@ -57,12 +64,14 @@ typedef struct TaskItem { const char *output_name; AVFrame *out_frame; int do_ioproc; + int async; int done; } TaskItem; typedef struct RequestItem { ie_infer_request_t *infer_request; TaskItem *task; + ie_complete_call_back_t callback; } RequestItem; #define APPEND_STRING(generated_string, iterate_string) \ @@ -73,6 +82,7 @@ typedef struct RequestItem { #define FLAGS AV_OPT_FLAG_FILTERING_PARAM static const AVOption dnn_openvino_options[] = { { "device", "device to run model", OFFSET(options.device_type), AV_OPT_TYPE_STRING, { .str = "CPU" }, 0, 0, FLAGS }, + { "nireq", "number of request", OFFSET(options.nireq), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, FLAGS }, { NULL } }; @@ -195,6 +205,12 @@ static void infer_completion_callback(void *args) task->out_frame->height = output.height; } ie_blob_free(&output_blob); + + if (task->async) { + request->task = NULL; + safe_queue_push_back(task->ov_model->request_queue, request); + } + task->done = 1; } @@ -208,16 +224,29 @@ static DNNReturnType execute_model_ov(TaskItem *task, RequestItem *request) return ret; } - status = ie_infer_request_infer(request->infer_request); - if (status != OK) { - av_log(ctx, AV_LOG_ERROR, "Failed to start synchronous model inference\n"); - return DNN_ERROR; + if (task->async) { + request->task = task; + status = ie_infer_set_completion_callback(request->infer_request, &request->callback); + if (status != OK) { + av_log(ctx, AV_LOG_ERROR, "Failed to set completion callback for inference\n"); + return DNN_ERROR; + } + status = ie_infer_request_infer_async(request->infer_request); + if (status != OK) { + av_log(ctx, AV_LOG_ERROR, "Failed to start async inference\n"); + return DNN_ERROR; + } + return DNN_SUCCESS; + } else { + status = ie_infer_request_infer(request->infer_request); + if (status != OK) { + av_log(ctx, AV_LOG_ERROR, "Failed to start synchronous model inference\n"); + return DNN_ERROR; + } + request->task = task; + infer_completion_callback(request); + return task->done ? DNN_SUCCESS : DNN_ERROR; } - - request->task = task; - infer_completion_callback(request); - - return task->done ? DNN_SUCCESS : DNN_ERROR; } static DNNReturnType get_input_ov(void *model, DNNData *input, const char *input_name) @@ -303,6 +332,7 @@ static DNNReturnType get_output_ov(void *model, const char *input_name, int inpu task.done = 0; task.do_ioproc = 0; + task.async = 0; task.input_name = input_name; task.in_frame = in_frame; task.output_name = output_name; @@ -376,10 +406,44 @@ DNNModel *ff_dnn_load_model_ov(const char *model_filename, const char *options, goto err; } + // create infer_request for sync execution status = ie_exec_network_create_infer_request(ov_model->exe_network, &ov_model->infer_request); if (status != OK) goto err; + // create infer_requests for async execution + if (ctx->options.nireq <= 0) { + // the default value is a rough estimation + ctx->options.nireq = av_cpu_count() / 2 + 1; + } + + ov_model->request_queue = safe_queue_create(); + if (!ov_model->request_queue) { + goto err; + } + + for (int i = 0; i < ctx->options.nireq; i++) { + ie_infer_request_t *request; + RequestItem *item = av_mallocz(sizeof(*item)); + if (!item) { + goto err; + } + status = ie_exec_network_create_infer_request(ov_model->exe_network, &request); + if (status != OK) { + av_freep(&item); + goto err; + } + item->infer_request = request; + item->callback.completeCallBackFunc = infer_completion_callback; + item->callback.args = item; + safe_queue_push_back(ov_model->request_queue, item); + } + + ov_model->task_queue = queue_create(); + if (!ov_model->task_queue) { + goto err; + } + model->get_input = &get_input_ov; model->get_output = &get_output_ov; model->options = options; @@ -419,6 +483,7 @@ DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, const char *input_n task.done = 0; task.do_ioproc = 1; + task.async = 0; task.input_name = input_name; task.in_frame = in_frame; task.output_name = output_names[0]; @@ -430,10 +495,89 @@ DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, const char *input_n return execute_model_ov(&task, &request); } +DNNReturnType ff_dnn_execute_model_async_ov(const DNNModel *model, const char *input_name, AVFrame *in_frame, + const char **output_names, uint32_t nb_output, AVFrame *out_frame) +{ + OVModel *ov_model = (OVModel *)model->model; + OVContext *ctx = &ov_model->ctx; + RequestItem *request; + TaskItem *task; + + if (!in_frame) { + av_log(ctx, AV_LOG_ERROR, "in frame is NULL when async execute model.\n"); + return DNN_ERROR; + } + + if (!out_frame) { + av_log(ctx, AV_LOG_ERROR, "out frame is NULL when async execute model.\n"); + return DNN_ERROR; + } + + task = av_malloc(sizeof(*task)); + if (!task) { + av_log(ctx, AV_LOG_ERROR, "unable to alloc memory for task item.\n"); + return DNN_ERROR; + } + + task->done = 0; + task->do_ioproc = 1; + task->async = 1; + task->input_name = input_name; + task->in_frame = in_frame; + task->output_name = output_names[0]; + task->out_frame = out_frame; + task->ov_model = ov_model; + queue_push_back(ov_model->task_queue, task); + + request = safe_queue_pop_front(ov_model->request_queue); + if (!request) { + av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n"); + return DNN_ERROR; + } + + return execute_model_ov(task, request); +} + +DNNAsyncStatusType ff_dnn_get_async_result_ov(const DNNModel *model, AVFrame **in, AVFrame **out) +{ + OVModel *ov_model = (OVModel *)model->model; + TaskItem *task = queue_peek_front(ov_model->task_queue); + + if (!task) { + return DAST_EMPTY_QUEUE; + } + + if (!task->done) { + return DAST_NOT_READY; + } + + *in = task->in_frame; + *out = task->out_frame; + queue_pop_front(ov_model->task_queue); + av_freep(&task); + + return DAST_SUCCESS; +} + void ff_dnn_free_model_ov(DNNModel **model) { if (*model){ OVModel *ov_model = (OVModel *)(*model)->model; + while (safe_queue_size(ov_model->request_queue) != 0) { + RequestItem *item = safe_queue_pop_front(ov_model->request_queue); + if (item && item->infer_request) { + ie_infer_request_free(&item->infer_request); + } + av_freep(&item); + } + safe_queue_destroy(ov_model->request_queue); + + while (queue_size(ov_model->task_queue) != 0) { + TaskItem *item = queue_pop_front(ov_model->task_queue); + av_freep(&item); + } + queue_destroy(ov_model->task_queue); + if (ov_model->infer_request) ie_infer_request_free(&ov_model->infer_request); if (ov_model->exe_network) diff --git a/libavfilter/dnn/dnn_backend_openvino.h b/libavfilter/dnn/dnn_backend_openvino.h index 3f8f01da60..2f88e49a08 100644 --- a/libavfilter/dnn/dnn_backend_openvino.h +++ b/libavfilter/dnn/dnn_backend_openvino.h @@ -33,6 +33,9 @@ DNNModel *ff_dnn_load_model_ov(const char *model_filename, const char *options, DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, const char *input_name, AVFrame *in_frame, const char **output_names, uint32_t nb_output, AVFrame *out_frame); +DNNReturnType ff_dnn_execute_model_async_ov(const DNNModel *model, const char *input_name, AVFrame *in_frame, + const char **output_names, uint32_t nb_output, AVFrame *out_frame); +DNNAsyncStatusType ff_dnn_get_async_result_ov(const DNNModel *model, AVFrame **in, AVFrame **out); void ff_dnn_free_model_ov(DNNModel **model); diff --git a/libavfilter/dnn/dnn_interface.c b/libavfilter/dnn/dnn_interface.c index f82ab12e98..e1b41a21e1 100644 --- a/libavfilter/dnn/dnn_interface.c +++ b/libavfilter/dnn/dnn_interface.c @@ -58,6 +58,8 @@ DNNModule *ff_get_dnn_module(DNNBackendType backend_type) #if (CONFIG_LIBOPENVINO == 1) dnn_module->load_model = &ff_dnn_load_model_ov; dnn_module->execute_model = &ff_dnn_execute_model_ov; + dnn_module->execute_model_async = &ff_dnn_execute_model_async_ov; + dnn_module->get_async_result = &ff_dnn_get_async_result_ov; dnn_module->free_model = &ff_dnn_free_model_ov; #else av_freep(&dnn_module); diff --git a/libavfilter/dnn_interface.h b/libavfilter/dnn_interface.h index 9e54b91d19..33d55703d2 100644 --- a/libavfilter/dnn_interface.h +++ b/libavfilter/dnn_interface.h @@ -80,7 +80,7 @@ typedef struct DNNModule{ DNNReturnType (*execute_model_async)(const DNNModel *model, const char *input_name, AVFrame *in_frame, const char **output_names, uint32_t nb_output, AVFrame *out_frame); // Retrieve inference result. - DNNAsyncStatusType (*get_async_result)(const DNNModel *model, AVFrame **out); + DNNAsyncStatusType (*get_async_result)(const DNNModel *model, AVFrame **in, AVFrame **out); // Frees memory allocated for model. void (*free_model)(DNNModel **model); } DNNModule;