From patchwork Fri Aug 7 07:00:26 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Guo, Yejun" X-Patchwork-Id: 21520 Return-Path: X-Original-To: patchwork@ffaux-bg.ffmpeg.org Delivered-To: patchwork@ffaux-bg.ffmpeg.org Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org [79.124.17.100]) by ffaux.localdomain (Postfix) with ESMTP id C2594448F7C for ; Fri, 7 Aug 2020 10:05:32 +0300 (EEST) Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 9D7F268B597; Fri, 7 Aug 2020 10:05:32 +0300 (EEST) X-Original-To: ffmpeg-devel@ffmpeg.org Delivered-To: ffmpeg-devel@ffmpeg.org Received: from mga05.intel.com (mga05.intel.com [192.55.52.43]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id 7BD1D68B3F4 for ; Fri, 7 Aug 2020 10:05:24 +0300 (EEST) IronPort-SDR: lo14R7AGGOi/9vfplGzpLMAnmdtG/ZoC3hVkEQF4OR7CJCxVygo+JrdKZyIehDCTZBCNEAEd/S fZSbvqoozPKQ== X-IronPort-AV: E=McAfee;i="6000,8403,9705"; a="237869399" X-IronPort-AV: E=Sophos;i="5.75,444,1589266800"; d="scan'208";a="237869399" X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga006.jf.intel.com ([10.7.209.51]) by fmsmga105.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 07 Aug 2020 00:05:21 -0700 IronPort-SDR: uqTU+Qbhzm2JeouthJSoYBbU3wPV/AidUwVLvsAQyu81Ir3llwCoYxUmOBt91u/QU0NfEkzWw5 H2zAYSdPTxOQ== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.75,444,1589266800"; d="scan'208";a="293573901" Received: from yguo18-skl-u1604.sh.intel.com ([10.239.159.53]) by orsmga006.jf.intel.com with ESMTP; 07 Aug 2020 00:05:17 -0700 From: "Guo, Yejun" To: ffmpeg-devel@ffmpeg.org Date: Fri, 7 Aug 2020 15:00:26 +0800 Message-Id: <1596783626-11483-1-git-send-email-yejun.guo@intel.com> X-Mailer: git-send-email 2.7.4 Subject: [FFmpeg-devel] [PATCH] dnn: add backend options when load the model X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches MIME-Version: 1.0 Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" different backend might need different options for a better performance, so, add the parameter into dnn interface, as a preparation. Signed-off-by: Guo, Yejun --- libavfilter/dnn/dnn_backend_native.c | 3 ++- libavfilter/dnn/dnn_backend_native.h | 2 +- libavfilter/dnn/dnn_backend_openvino.c | 3 ++- libavfilter/dnn/dnn_backend_openvino.h | 2 +- libavfilter/dnn/dnn_backend_tf.c | 3 ++- libavfilter/dnn/dnn_backend_tf.h | 2 +- libavfilter/dnn_interface.h | 4 +++- libavfilter/vf_derain.c | 2 +- libavfilter/vf_dnn_processing.c | 4 +++- libavfilter/vf_sr.c | 2 +- 10 files changed, 17 insertions(+), 10 deletions(-) diff --git a/libavfilter/dnn/dnn_backend_native.c b/libavfilter/dnn/dnn_backend_native.c index a685efb..adc652a 100644 --- a/libavfilter/dnn/dnn_backend_native.c +++ b/libavfilter/dnn/dnn_backend_native.c @@ -115,7 +115,7 @@ static DNNReturnType set_input_output_native(void *model, DNNData *input, const // layers_num,layer_type,layer_parameterss,layer_type,layer_parameters... // For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases // For DEPTH_TO_SPACE layer: block_size -DNNModel *ff_dnn_load_model_native(const char *model_filename) +DNNModel *ff_dnn_load_model_native(const char *model_filename, const char *options) { DNNModel *model = NULL; char header_expected[] = "FFMPEGDNNNATIVE"; @@ -245,6 +245,7 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) model->set_input_output = &set_input_output_native; model->get_input = &get_input_native; + model->options = options; return model; diff --git a/libavfilter/dnn/dnn_backend_native.h b/libavfilter/dnn/dnn_backend_native.h index 62191ff..5254f9a 100644 --- a/libavfilter/dnn/dnn_backend_native.h +++ b/libavfilter/dnn/dnn_backend_native.h @@ -114,7 +114,7 @@ typedef struct ConvolutionalNetwork{ uint32_t nb_output; } ConvolutionalNetwork; -DNNModel *ff_dnn_load_model_native(const char *model_filename); +DNNModel *ff_dnn_load_model_native(const char *model_filename, const char *options); DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output); diff --git a/libavfilter/dnn/dnn_backend_openvino.c b/libavfilter/dnn/dnn_backend_openvino.c index f048bc2..d343bf2 100644 --- a/libavfilter/dnn/dnn_backend_openvino.c +++ b/libavfilter/dnn/dnn_backend_openvino.c @@ -155,7 +155,7 @@ err: return DNN_ERROR; } -DNNModel *ff_dnn_load_model_ov(const char *model_filename) +DNNModel *ff_dnn_load_model_ov(const char *model_filename, const char *options) { DNNModel *model = NULL; OVModel *ov_model = NULL; @@ -186,6 +186,7 @@ DNNModel *ff_dnn_load_model_ov(const char *model_filename) model->model = (void *)ov_model; model->set_input_output = &set_input_output_ov; model->get_input = &get_input_ov; + model->options = options; return model; diff --git a/libavfilter/dnn/dnn_backend_openvino.h b/libavfilter/dnn/dnn_backend_openvino.h index 397847a..0a8fef9 100644 --- a/libavfilter/dnn/dnn_backend_openvino.h +++ b/libavfilter/dnn/dnn_backend_openvino.h @@ -29,7 +29,7 @@ #include "../dnn_interface.h" -DNNModel *ff_dnn_load_model_ov(const char *model_filename); +DNNModel *ff_dnn_load_model_ov(const char *model_filename, const char *options); DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, DNNData *outputs, uint32_t nb_output); diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c index 9ceca5c..86da991 100644 --- a/libavfilter/dnn/dnn_backend_tf.c +++ b/libavfilter/dnn/dnn_backend_tf.c @@ -572,7 +572,7 @@ static DNNReturnType load_native_model(TFModel *tf_model, const char *model_file return DNN_SUCCESS; } -DNNModel *ff_dnn_load_model_tf(const char *model_filename) +DNNModel *ff_dnn_load_model_tf(const char *model_filename, const char *options) { DNNModel *model = NULL; TFModel *tf_model = NULL; @@ -600,6 +600,7 @@ DNNModel *ff_dnn_load_model_tf(const char *model_filename) model->model = (void *)tf_model; model->set_input_output = &set_input_output_tf; model->get_input = &get_input_tf; + model->options = options; return model; } diff --git a/libavfilter/dnn/dnn_backend_tf.h b/libavfilter/dnn/dnn_backend_tf.h index 3e450891..d7ee1df 100644 --- a/libavfilter/dnn/dnn_backend_tf.h +++ b/libavfilter/dnn/dnn_backend_tf.h @@ -29,7 +29,7 @@ #include "../dnn_interface.h" -DNNModel *ff_dnn_load_model_tf(const char *model_filename); +DNNModel *ff_dnn_load_model_tf(const char *model_filename, const char *options); DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, uint32_t nb_output); diff --git a/libavfilter/dnn_interface.h b/libavfilter/dnn_interface.h index f914265..c5a72b4 100644 --- a/libavfilter/dnn_interface.h +++ b/libavfilter/dnn_interface.h @@ -43,6 +43,8 @@ typedef struct DNNData{ typedef struct DNNModel{ // Stores model that can be different for different backends. void *model; + // Stores options when the model is executed by the backend + const char *options; // Gets model input information // Just reuse struct DNNData here, actually the DNNData.data field is not needed. DNNReturnType (*get_input)(void *model, DNNData *input, const char *input_name); @@ -54,7 +56,7 @@ typedef struct DNNModel{ // Stores pointers to functions for loading, executing, freeing DNN models for one of the backends. typedef struct DNNModule{ // Loads model and parameters from given file. Returns NULL if it is not possible. - DNNModel *(*load_model)(const char *model_filename); + DNNModel *(*load_model)(const char *model_filename, const char *options); // Executes model with specified input and output. Returns DNN_ERROR otherwise. DNNReturnType (*execute_model)(const DNNModel *model, DNNData *outputs, uint32_t nb_output); // Frees memory allocated for model. diff --git a/libavfilter/vf_derain.c b/libavfilter/vf_derain.c index 7432260..e175bf4 100644 --- a/libavfilter/vf_derain.c +++ b/libavfilter/vf_derain.c @@ -161,7 +161,7 @@ static av_cold int init(AVFilterContext *ctx) return AVERROR(EINVAL); } - dr_context->model = (dr_context->dnn_module->load_model)(dr_context->model_filename); + dr_context->model = (dr_context->dnn_module->load_model)(dr_context->model_filename, NULL); if (!dr_context->model) { av_log(ctx, AV_LOG_ERROR, "could not load DNN model\n"); return AVERROR(EINVAL); diff --git a/libavfilter/vf_dnn_processing.c b/libavfilter/vf_dnn_processing.c index 4b31808..b6dfc4e 100644 --- a/libavfilter/vf_dnn_processing.c +++ b/libavfilter/vf_dnn_processing.c @@ -41,6 +41,7 @@ typedef struct DnnProcessingContext { DNNBackendType backend_type; char *model_inputname; char *model_outputname; + char *backend_options; DNNModule *dnn_module; DNNModel *model; @@ -69,6 +70,7 @@ static const AVOption dnn_processing_options[] = { { "model", "path to model file", OFFSET(model_filename), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, { "input", "input name of the model", OFFSET(model_inputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, { "output", "output name of the model", OFFSET(model_outputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, + { "options", "backend options", OFFSET(backend_options), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, { NULL } }; @@ -101,7 +103,7 @@ static av_cold int init(AVFilterContext *context) return AVERROR(EINVAL); } - ctx->model = (ctx->dnn_module->load_model)(ctx->model_filename); + ctx->model = (ctx->dnn_module->load_model)(ctx->model_filename, ctx->backend_options); if (!ctx->model) { av_log(ctx, AV_LOG_ERROR, "could not load DNN model\n"); return AVERROR(EINVAL); diff --git a/libavfilter/vf_sr.c b/libavfilter/vf_sr.c index f000eda..1dee317 100644 --- a/libavfilter/vf_sr.c +++ b/libavfilter/vf_sr.c @@ -81,7 +81,7 @@ static av_cold int init(AVFilterContext *context) av_log(context, AV_LOG_ERROR, "load_model for network was not specified\n"); return AVERROR(EIO); } - sr_context->model = (sr_context->dnn_module->load_model)(sr_context->model_filename); + sr_context->model = (sr_context->dnn_module->load_model)(sr_context->model_filename, NULL); if (!sr_context->model){ av_log(context, AV_LOG_ERROR, "could not load DNN model\n"); return AVERROR(EIO);