From patchwork Mon Jul 29 01:56:54 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Guo, Yejun" X-Patchwork-Id: 14116 Return-Path: X-Original-To: patchwork@ffaux-bg.ffmpeg.org Delivered-To: patchwork@ffaux-bg.ffmpeg.org Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org [79.124.17.100]) by ffaux.localdomain (Postfix) with ESMTP id 587AA44A2C3 for ; Mon, 29 Jul 2019 05:00:07 +0300 (EEST) Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 4200A6882F4; Mon, 29 Jul 2019 05:00:07 +0300 (EEST) X-Original-To: ffmpeg-devel@ffmpeg.org Delivered-To: ffmpeg-devel@ffmpeg.org Received: from mga06.intel.com (mga06.intel.com [134.134.136.31]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id 8AC15680911 for ; Mon, 29 Jul 2019 05:00:00 +0300 (EEST) X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga008.jf.intel.com ([10.7.209.65]) by orsmga104.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384; 28 Jul 2019 18:59:58 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.64,320,1559545200"; d="scan'208";a="165296751" Received: from yguo18-skl-u1604.sh.intel.com ([10.239.13.25]) by orsmga008.jf.intel.com with ESMTP; 28 Jul 2019 18:59:57 -0700 From: "Guo, Yejun" To: ffmpeg-devel@ffmpeg.org Date: Mon, 29 Jul 2019 09:56:54 +0800 Message-Id: <1564365414-27068-1-git-send-email-yejun.guo@intel.com> X-Mailer: git-send-email 2.7.4 Subject: [FFmpeg-devel] [PATCH V2 3/3] dnn: convert tf.pad to native model in python script, and load/execute it in the c code. X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Cc: yejun.guo@intel.com MIME-Version: 1.0 Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" since tf.pad is enabled, the conv2d(valid) changes back to its original behavior. Signed-off-by: Guo, Yejun --- libavfilter/dnn/dnn_backend_native.c | 35 +++++++++++++++++++++++++++++++++ libavfilter/dnn/dnn_backend_native.h | 2 +- tools/python/convert_from_tensorflow.py | 23 +++++++++++++++++----- 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/libavfilter/dnn/dnn_backend_native.c b/libavfilter/dnn/dnn_backend_native.c index 82e900b..09c583b 100644 --- a/libavfilter/dnn/dnn_backend_native.c +++ b/libavfilter/dnn/dnn_backend_native.c @@ -25,6 +25,7 @@ #include "dnn_backend_native.h" #include "libavutil/avassert.h" +#include "dnn_backend_native_layer_pad.h" static DNNReturnType set_input_output_native(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output) { @@ -32,6 +33,7 @@ static DNNReturnType set_input_output_native(void *model, DNNInputData *input, c InputParams *input_params; ConvolutionalParams *conv_params; DepthToSpaceParams *depth_to_space_params; + LayerPadParams *pad_params; int cur_width, cur_height, cur_channels; int32_t layer; @@ -77,6 +79,12 @@ static DNNReturnType set_input_output_native(void *model, DNNInputData *input, c cur_height *= depth_to_space_params->block_size; cur_width *= depth_to_space_params->block_size; break; + case MIRROR_PAD: + pad_params = (LayerPadParams *)network->layers[layer].params; + cur_height = cur_height + pad_params->paddings[1][0] + pad_params->paddings[1][1]; + cur_width = cur_width + pad_params->paddings[2][0] + pad_params->paddings[2][1]; + cur_channels = cur_channels + pad_params->paddings[3][0] + pad_params->paddings[3][1]; + break; default: return DNN_ERROR; } @@ -110,6 +118,7 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) DNNLayerType layer_type; ConvolutionalParams *conv_params; DepthToSpaceParams *depth_to_space_params; + LayerPadParams *pad_params; model = av_malloc(sizeof(DNNModel)); if (!model){ @@ -207,6 +216,23 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) network->layers[layer].type = DEPTH_TO_SPACE; network->layers[layer].params = depth_to_space_params; break; + case MIRROR_PAD: + pad_params = av_malloc(sizeof(LayerPadParams)); + if (!pad_params){ + avio_closep(&model_file_context); + ff_dnn_free_model_native(&model); + return NULL; + } + pad_params->mode = (int32_t)avio_rl32(model_file_context); + dnn_size += 4; + for (i = 0; i < 4; ++i) { + pad_params->paddings[i][0] = avio_rl32(model_file_context); + pad_params->paddings[i][1] = avio_rl32(model_file_context); + dnn_size += 8; + } + network->layers[layer].type = MIRROR_PAD; + network->layers[layer].params = pad_params; + break; default: avio_closep(&model_file_context); ff_dnn_free_model_native(&model); @@ -314,6 +340,7 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *output InputParams *input_params; ConvolutionalParams *conv_params; DepthToSpaceParams *depth_to_space_params; + LayerPadParams *pad_params; if (network->layers_num <= 0 || network->layers[0].type != INPUT || !network->layers[0].output){ return DNN_ERROR; @@ -348,6 +375,14 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *output cur_width *= depth_to_space_params->block_size; cur_channels /= depth_to_space_params->block_size * depth_to_space_params->block_size; break; + case MIRROR_PAD: + pad_params = (LayerPadParams *)network->layers[layer].params; + dnn_execute_layer_pad(network->layers[layer - 1].output, network->layers[layer].output, + pad_params, 1, cur_height, cur_width, cur_channels); + cur_height = cur_height + pad_params->paddings[1][0] + pad_params->paddings[1][1]; + cur_width = cur_width + pad_params->paddings[2][0] + pad_params->paddings[2][1]; + cur_channels = cur_channels + pad_params->paddings[3][0] + pad_params->paddings[3][1]; + break; case INPUT: return DNN_ERROR; } diff --git a/libavfilter/dnn/dnn_backend_native.h b/libavfilter/dnn/dnn_backend_native.h index 8ef1855..b6f9533 100644 --- a/libavfilter/dnn/dnn_backend_native.h +++ b/libavfilter/dnn/dnn_backend_native.h @@ -30,7 +30,7 @@ #include "../dnn_interface.h" #include "libavformat/avio.h" -typedef enum {INPUT, CONV, DEPTH_TO_SPACE} DNNLayerType; +typedef enum {INPUT, CONV, DEPTH_TO_SPACE, MIRROR_PAD} DNNLayerType; typedef enum {RELU, TANH, SIGMOID, NONE, LEAKY_RELU} DNNActivationFunc; diff --git a/tools/python/convert_from_tensorflow.py b/tools/python/convert_from_tensorflow.py index 37049e5..041c82c 100644 --- a/tools/python/convert_from_tensorflow.py +++ b/tools/python/convert_from_tensorflow.py @@ -23,9 +23,6 @@ import sys, struct __all__ = ['convert_from_tensorflow'] -# as the first step to be compatible with vf_sr, it is not general. -# it will be refined step by step. - class TFConverter: def __init__(self, graph_def, nodes, outfile): self.graph_def = graph_def @@ -36,9 +33,10 @@ class TFConverter: self.name_node_dict = {} self.edges = {} self.conv_activations = {'Relu':0, 'Tanh':1, 'Sigmoid':2, 'LeakyRelu':4} - self.conv_paddings = {'VALID':2, 'SAME':1} + self.conv_paddings = {'VALID':0, 'SAME':1} self.converted_nodes = set() - self.op2code = {'Conv2D':1, 'DepthToSpace':2} + self.op2code = {'Conv2D':1, 'DepthToSpace':2, 'MirrorPad':3} + self.mirrorpad_mode = {'CONSTANT':0, 'REFLECT':1, 'SYMMETRIC':2} def dump_for_tensorboard(self): @@ -101,6 +99,19 @@ class TFConverter: self.converted_nodes.add(node.name) + def dump_mirrorpad_to_file(self, node, f): + assert(node.op == 'MirrorPad') + self.layer_number = self.layer_number + 1 + mode = node.attr['mode'].s + mode = self.mirrorpad_mode[mode.decode("utf-8")] + np.array([self.op2code[node.op], mode], dtype=np.uint32).tofile(f) + pnode = self.name_node_dict[node.input[1]] + self.converted_nodes.add(pnode.name) + paddings = pnode.attr['value'].tensor.tensor_content + f.write(paddings) + self.converted_nodes.add(node.name) + + def generate_layer_number(self): # in current hard code implementation, the layer number is the first data written to the native model file # it is not easy to know it at the beginning time in the general converter, so first do a dry run for compatibility @@ -118,6 +129,8 @@ class TFConverter: self.dump_conv2d_to_file(node, f) elif node.op == 'DepthToSpace': self.dump_depth2space_to_file(node, f) + elif node.op == 'MirrorPad': + self.dump_mirrorpad_to_file(node, f) def dump_to_file(self):