From patchwork Mon Oct 21 12:38:10 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Guo, Yejun" X-Patchwork-Id: 15883 Return-Path: X-Original-To: patchwork@ffaux-bg.ffmpeg.org Delivered-To: patchwork@ffaux-bg.ffmpeg.org Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org [79.124.17.100]) by ffaux.localdomain (Postfix) with ESMTP id 34519448415 for ; Mon, 21 Oct 2019 15:44:10 +0300 (EEST) Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 204AD68B0C4; Mon, 21 Oct 2019 15:44:10 +0300 (EEST) X-Original-To: ffmpeg-devel@ffmpeg.org Delivered-To: ffmpeg-devel@ffmpeg.org Received: from mga03.intel.com (mga03.intel.com [134.134.136.65]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id 5CAAA68B0AA for ; Mon, 21 Oct 2019 15:44:03 +0300 (EEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from fmsmga007.fm.intel.com ([10.253.24.52]) by orsmga103.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 21 Oct 2019 05:43:59 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.67,323,1566889200"; d="scan'208";a="196801335" Received: from yguo18-skl-u1604.sh.intel.com ([10.239.13.25]) by fmsmga007.fm.intel.com with ESMTP; 21 Oct 2019 05:43:58 -0700 From: "Guo, Yejun" To: ffmpeg-devel@ffmpeg.org Date: Mon, 21 Oct 2019 20:38:10 +0800 Message-Id: <1571661490-21463-1-git-send-email-yejun.guo@intel.com> X-Mailer: git-send-email 2.7.4 Subject: [FFmpeg-devel] [PATCH V2 2/4] avfilter/dnn: get the data type of network output from dnn execution result X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Cc: yejun.guo@intel.com MIME-Version: 1.0 Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" so, we can make a filter more general to accept different network models, by adding a data type convertion after getting data from network. After we add dt field into struct DNNData, it becomes the same as DNNInputData, so merge them with one struct: DNNData. Signed-off-by: Guo, Yejun --- libavfilter/dnn/dnn_backend_native.c | 3 ++- libavfilter/dnn/dnn_backend_native_layer_conv2d.c | 1 + libavfilter/dnn/dnn_backend_native_layer_depth2space.c | 1 + libavfilter/dnn/dnn_backend_native_layer_pad.c | 1 + libavfilter/dnn/dnn_backend_tf.c | 5 +++-- libavfilter/dnn_interface.h | 9 ++------- libavfilter/vf_derain.c | 4 ++-- libavfilter/vf_sr.c | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/libavfilter/dnn/dnn_backend_native.c b/libavfilter/dnn/dnn_backend_native.c index ff280b5..add1db4 100644 --- a/libavfilter/dnn/dnn_backend_native.c +++ b/libavfilter/dnn/dnn_backend_native.c @@ -28,7 +28,7 @@ #include "dnn_backend_native_layer_conv2d.h" #include "dnn_backend_native_layers.h" -static DNNReturnType set_input_output_native(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output) +static DNNReturnType set_input_output_native(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output) { ConvolutionalNetwork *network = (ConvolutionalNetwork *)model; DnnOperand *oprd = NULL; @@ -263,6 +263,7 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *output outputs[i].height = oprd->dims[1]; outputs[i].width = oprd->dims[2]; outputs[i].channels = oprd->dims[3]; + outputs[i].dt = oprd->data_type; } return DNN_SUCCESS; diff --git a/libavfilter/dnn/dnn_backend_native_layer_conv2d.c b/libavfilter/dnn/dnn_backend_native_layer_conv2d.c index 6ec0fa7..7b29697 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_conv2d.c +++ b/libavfilter/dnn/dnn_backend_native_layer_conv2d.c @@ -106,6 +106,7 @@ int dnn_execute_layer_conv2d(DnnOperand *operands, const int32_t *input_operand_ output_operand->dims[1] = height - pad_size * 2; output_operand->dims[2] = width - pad_size * 2; output_operand->dims[3] = conv_params->output_num; + output_operand->data_type = operands[input_operand_index].data_type; output_operand->length = calculate_operand_data_length(output_operand); output_operand->data = av_realloc(output_operand->data, output_operand->length); if (!output_operand->data) diff --git a/libavfilter/dnn/dnn_backend_native_layer_depth2space.c b/libavfilter/dnn/dnn_backend_native_layer_depth2space.c index 174676e..7dab19d 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_depth2space.c +++ b/libavfilter/dnn/dnn_backend_native_layer_depth2space.c @@ -69,6 +69,7 @@ int dnn_execute_layer_depth2space(DnnOperand *operands, const int32_t *input_ope output_operand->dims[1] = height * block_size; output_operand->dims[2] = width * block_size; output_operand->dims[3] = new_channels; + output_operand->data_type = operands[input_operand_index].data_type; output_operand->length = calculate_operand_data_length(output_operand); output_operand->data = av_realloc(output_operand->data, output_operand->length); if (!output_operand->data) diff --git a/libavfilter/dnn/dnn_backend_native_layer_pad.c b/libavfilter/dnn/dnn_backend_native_layer_pad.c index 8fa35de..8e5959b 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_pad.c +++ b/libavfilter/dnn/dnn_backend_native_layer_pad.c @@ -105,6 +105,7 @@ int dnn_execute_layer_pad(DnnOperand *operands, const int32_t *input_operand_ind output_operand->dims[1] = new_height; output_operand->dims[2] = new_width; output_operand->dims[3] = new_channel; + output_operand->data_type = operands[input_operand_index].data_type; output_operand->length = calculate_operand_data_length(output_operand); output_operand->data = av_realloc(output_operand->data, output_operand->length); if (!output_operand->data) diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c index c8dff51..ed91d05 100644 --- a/libavfilter/dnn/dnn_backend_tf.c +++ b/libavfilter/dnn/dnn_backend_tf.c @@ -83,7 +83,7 @@ static TF_Buffer *read_graph(const char *model_filename) return graph_buf; } -static TF_Tensor *allocate_input_tensor(const DNNInputData *input) +static TF_Tensor *allocate_input_tensor(const DNNData *input) { TF_DataType dt; size_t size; @@ -105,7 +105,7 @@ static TF_Tensor *allocate_input_tensor(const DNNInputData *input) input_dims[1] * input_dims[2] * input_dims[3] * size); } -static DNNReturnType set_input_output_tf(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output) +static DNNReturnType set_input_output_tf(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output) { TFModel *tf_model = (TFModel *)model; TF_SessionOptions *sess_opts; @@ -603,6 +603,7 @@ DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, u outputs[i].width = TF_Dim(tf_model->output_tensors[i], 2); outputs[i].channels = TF_Dim(tf_model->output_tensors[i], 3); outputs[i].data = TF_TensorData(tf_model->output_tensors[i]); + outputs[i].dt = TF_TensorType(tf_model->output_tensors[i]); } return DNN_SUCCESS; diff --git a/libavfilter/dnn_interface.h b/libavfilter/dnn_interface.h index 057005f..fdefcb7 100644 --- a/libavfilter/dnn_interface.h +++ b/libavfilter/dnn_interface.h @@ -34,15 +34,10 @@ typedef enum {DNN_NATIVE, DNN_TF} DNNBackendType; typedef enum {DNN_FLOAT = 1, DNN_UINT8 = 4} DNNDataType; -typedef struct DNNInputData{ +typedef struct DNNData{ void *data; DNNDataType dt; int width, height, channels; -} DNNInputData; - -typedef struct DNNData{ - float *data; - int width, height, channels; } DNNData; typedef struct DNNModel{ @@ -50,7 +45,7 @@ typedef struct DNNModel{ void *model; // Sets model input and output. // Should be called at least once before model execution. - DNNReturnType (*set_input_output)(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output); + DNNReturnType (*set_input_output)(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output); } DNNModel; // Stores pointers to functions for loading, executing, freeing DNN models for one of the backends. diff --git a/libavfilter/vf_derain.c b/libavfilter/vf_derain.c index b7bba09..89f9d5a 100644 --- a/libavfilter/vf_derain.c +++ b/libavfilter/vf_derain.c @@ -39,7 +39,7 @@ typedef struct DRContext { DNNBackendType backend_type; DNNModule *dnn_module; DNNModel *model; - DNNInputData input; + DNNData input; DNNData output; } DRContext; @@ -137,7 +137,7 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in) int t = i * out->width * 3 + j; int t_in = (i + pad_size) * in->width * 3 + j + pad_size * 3; - out->data[0][k] = CLIP((int)((((float *)dr_context->input.data)[t_in] - dr_context->output.data[t]) * 255), 0, 255); + out->data[0][k] = CLIP((int)((((float *)dr_context->input.data)[t_in] - ((float *)dr_context->output.data)[t]) * 255), 0, 255); } } diff --git a/libavfilter/vf_sr.c b/libavfilter/vf_sr.c index 0433246..fff19ea 100644 --- a/libavfilter/vf_sr.c +++ b/libavfilter/vf_sr.c @@ -41,7 +41,7 @@ typedef struct SRContext { DNNBackendType backend_type; DNNModule *dnn_module; DNNModel *model; - DNNInputData input; + DNNData input; DNNData output; int scale_factor; struct SwsContext *sws_contexts[3];