From patchwork Mon Oct 21 12:38:17 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Guo, Yejun" X-Patchwork-Id: 15884 Return-Path: X-Original-To: patchwork@ffaux-bg.ffmpeg.org Delivered-To: patchwork@ffaux-bg.ffmpeg.org Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org [79.124.17.100]) by ffaux.localdomain (Postfix) with ESMTP id 0E53A448415 for ; Mon, 21 Oct 2019 15:44:15 +0300 (EEST) Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id E625F68B0CE; Mon, 21 Oct 2019 15:44:14 +0300 (EEST) X-Original-To: ffmpeg-devel@ffmpeg.org Delivered-To: ffmpeg-devel@ffmpeg.org Received: from mga04.intel.com (mga04.intel.com [192.55.52.120]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id 4F9A968B0C1 for ; Mon, 21 Oct 2019 15:44:08 +0300 (EEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga006.jf.intel.com ([10.7.209.51]) by fmsmga104.fm.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 21 Oct 2019 05:44:06 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.67,323,1566889200"; d="scan'208";a="201357121" Received: from yguo18-skl-u1604.sh.intel.com ([10.239.13.25]) by orsmga006.jf.intel.com with ESMTP; 21 Oct 2019 05:44:05 -0700 From: "Guo, Yejun" To: ffmpeg-devel@ffmpeg.org Date: Mon, 21 Oct 2019 20:38:17 +0800 Message-Id: <1571661497-21510-1-git-send-email-yejun.guo@intel.com> X-Mailer: git-send-email 2.7.4 Subject: [FFmpeg-devel] [PATCH V2 3/4] avfilter/dnn: add a new interface to query dnn model's input info X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Cc: yejun.guo@intel.com MIME-Version: 1.0 Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" to support dnn networks more general, we need to know the input info of the dnn model. background: The data type of dnn model's input could be float32, uint8 or fp16, etc. And the w/h of input image could be fixed or variable. Signed-off-by: Guo, Yejun --- libavfilter/dnn/dnn_backend_native.c | 24 +++++++++++++++++++++++- libavfilter/dnn/dnn_backend_tf.c | 32 ++++++++++++++++++++++++++++++++ libavfilter/dnn_interface.h | 3 +++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/libavfilter/dnn/dnn_backend_native.c b/libavfilter/dnn/dnn_backend_native.c index add1db4..94634b3 100644 --- a/libavfilter/dnn/dnn_backend_native.c +++ b/libavfilter/dnn/dnn_backend_native.c @@ -28,6 +28,28 @@ #include "dnn_backend_native_layer_conv2d.h" #include "dnn_backend_native_layers.h" +static DNNReturnType get_input_native(void *model, DNNData *input, const char *input_name) +{ + ConvolutionalNetwork *network = (ConvolutionalNetwork *)model; + + for (int i = 0; i < network->operands_num; ++i) { + DnnOperand *oprd = &network->operands[i]; + if (strcmp(oprd->name, input_name) == 0) { + if (oprd->type != DOT_INPUT) + return DNN_ERROR; + input->dt = oprd->data_type; + av_assert0(oprd->dims[0] == 1); + input->height = oprd->dims[1]; + input->width = oprd->dims[2]; + input->channels = oprd->dims[3]; + return DNN_SUCCESS; + } + } + + // do not find the input operand + return DNN_ERROR; +} + static DNNReturnType set_input_output_native(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output) { ConvolutionalNetwork *network = (ConvolutionalNetwork *)model; @@ -37,7 +59,6 @@ static DNNReturnType set_input_output_native(void *model, DNNData *input, const return DNN_ERROR; /* inputs */ - av_assert0(input->dt == DNN_FLOAT); for (int i = 0; i < network->operands_num; ++i) { oprd = &network->operands[i]; if (strcmp(oprd->name, input_name) == 0) { @@ -234,6 +255,7 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) } model->set_input_output = &set_input_output_native; + model->get_input = &get_input_native; return model; } diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c index ed91d05..a921667 100644 --- a/libavfilter/dnn/dnn_backend_tf.c +++ b/libavfilter/dnn/dnn_backend_tf.c @@ -105,6 +105,37 @@ static TF_Tensor *allocate_input_tensor(const DNNData *input) input_dims[1] * input_dims[2] * input_dims[3] * size); } +static DNNReturnType get_input_tf(void *model, DNNData *input, const char *input_name) +{ + TFModel *tf_model = (TFModel *)model; + TF_Status *status; + int64_t dims[4]; + + TF_Output tf_output; + tf_output.oper = TF_GraphOperationByName(tf_model->graph, input_name); + if (!tf_output.oper) + return DNN_ERROR; + + tf_output.index = 0; + input->dt = TF_OperationOutputType(tf_output); + + status = TF_NewStatus(); + TF_GraphGetTensorShape(tf_model->graph, tf_output, dims, 4, status); + if (TF_GetCode(status) != TF_OK){ + TF_DeleteStatus(status); + return DNN_ERROR; + } + TF_DeleteStatus(status); + + // currently only NHWC is supported + av_assert0(dims[0] == 1); + input->height = dims[1]; + input->width = dims[2]; + input->channels = dims[3]; + + return DNN_SUCCESS; +} + static DNNReturnType set_input_output_tf(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output) { TFModel *tf_model = (TFModel *)model; @@ -568,6 +599,7 @@ DNNModel *ff_dnn_load_model_tf(const char *model_filename) model->model = (void *)tf_model; model->set_input_output = &set_input_output_tf; + model->get_input = &get_input_tf; return model; } diff --git a/libavfilter/dnn_interface.h b/libavfilter/dnn_interface.h index fdefcb7..b20e5c8 100644 --- a/libavfilter/dnn_interface.h +++ b/libavfilter/dnn_interface.h @@ -43,6 +43,9 @@ typedef struct DNNData{ typedef struct DNNModel{ // Stores model that can be different for different backends. void *model; + // Gets model input information + // Just reuse struct DNNData here, actually the DNNData.data field is not needed. + DNNReturnType (*get_input)(void *model, DNNData *input, const char *input_name); // Sets model input and output. // Should be called at least once before model execution. DNNReturnType (*set_input_output)(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output);