From patchwork Sat Apr 11 08:39:31 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Guo, Yejun" X-Patchwork-Id: 18872 Delivered-To: andriy.gelman@gmail.com Received: by 2002:ad4:45d0:0:0:0:0:0 with SMTP id v16csp608244qvt; Sat, 11 Apr 2020 01:41:13 -0700 (PDT) X-Google-Smtp-Source: APiQypLx8y7uvJbTaaxL2qr6sqBOpM0l5evlaTnei3IUVdFLhsaQAtzu6NFoH5Zjw9erz6JmQQlc X-Received: by 2002:a1c:2506:: with SMTP id l6mr8623445wml.44.1586594473705; Sat, 11 Apr 2020 01:41:13 -0700 (PDT) ARC-Seal: i=1; a=rsa-sha256; t=1586594473; cv=none; d=google.com; s=arc-20160816; b=Ff4BhgsiKivm5kEBMjZubEm+WufYiLdl0GC5RiapxU0oj21kjbLiZaMs7YAFOfMHwU A8M0CMp7y7K5q4Y8bR7gT3hBEebJg+l2i7hsOc8JUdz1WPY5BzdXYkcCUCdpHFNvZW9L xadcrAkOoAXDQkhIGlumEwOhN1cxelDtiVyVaviOlb33oeq8pO9tRMqgVb8vvRx6/Fvz kaKwsTXjQyZMDmfY2ngLR28XWyj7ad0a2jzNNggUxIsJ7oCX2W8hU1Jb5Vk5frZC16R2 dIleC1Pmmtqf9LhRa9QvMx71NMcs8foOJCpkXM/AcPnKk5u7zUuCV9VA4Mi0cNP7TywV PM9A== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=sender:errors-to:content-transfer-encoding:mime-version:cc:reply-to :list-subscribe:list-help:list-post:list-archive:list-unsubscribe :list-id:precedence:subject:message-id:date:to:from:ironport-sdr :ironport-sdr:delivered-to; bh=Y/h0wP/VOrGqvoz/omlhgWTVEwbfdPlKXDe4bdz4CBs=; b=a4gdMvptUIWv6ZvKIf4Ng0S/73QjOmTtAN9kHZ+K+g3PXEy2hqsJkow3s13DwX6GV+ HIvs2hkxW/NJVj8rLLLyh9boatqv+V0eD4maWkaTUjN1z/L5i67yg4TTjq5dieZZ+F01 ua0x1Xe8fqNWx7wN6l5QLlw4X5mlYCoWfDQ2LuJyB9Us7OZElCPDVp7V+8b3fvtGmHX5 jGGkUpepTzooi9EogqDJm0XFhP6TBZMMDgOaaciK+I94OpWzKNxdnGZaEhRtdO6NRjFM P/NNMnyyWvOsFr9NOj0YT69y6CadcLlWmoyZbAUBJBbDb0U3ScHhqldPfVF4ip07m6um nrZw== ARC-Authentication-Results: i=1; mx.google.com; spf=pass (google.com: domain of ffmpeg-devel-bounces@ffmpeg.org designates 79.124.17.100 as permitted sender) smtp.mailfrom=ffmpeg-devel-bounces@ffmpeg.org; dmarc=fail (p=NONE sp=NONE dis=NONE) header.from=intel.com Return-Path: Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org. [79.124.17.100]) by mx.google.com with ESMTP id z8si4569169wmc.146.2020.04.11.01.41.13; Sat, 11 Apr 2020 01:41:13 -0700 (PDT) Received-SPF: pass (google.com: domain of ffmpeg-devel-bounces@ffmpeg.org designates 79.124.17.100 as permitted sender) client-ip=79.124.17.100; Authentication-Results: mx.google.com; spf=pass (google.com: domain of ffmpeg-devel-bounces@ffmpeg.org designates 79.124.17.100 as permitted sender) smtp.mailfrom=ffmpeg-devel-bounces@ffmpeg.org; dmarc=fail (p=NONE sp=NONE dis=NONE) header.from=intel.com Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 8F0FB68B096; Sat, 11 Apr 2020 11:41:10 +0300 (EEST) X-Original-To: ffmpeg-devel@ffmpeg.org Delivered-To: ffmpeg-devel@ffmpeg.org Received: from mga05.intel.com (mga05.intel.com [192.55.52.43]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id 4989468AAD8 for ; Sat, 11 Apr 2020 11:41:03 +0300 (EEST) IronPort-SDR: 9tH6XweLUMQCX5peQw+1X317FpIUdOMAwu9fC81H642oqu6puNUxfQzewSYscTzCQQDboTKfPS KTb3vcuNtXpw== X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga005.jf.intel.com ([10.7.209.41]) by fmsmga105.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 11 Apr 2020 01:41:00 -0700 IronPort-SDR: dggZK/uIosEWYsyekd/A98DpRTUSn1KsfiHznpYx8x+X/yZWY6QID30xhWWimQRnwpVjpB5qJU GAN265ZZBX4Q== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.72,370,1580803200"; d="scan'208";a="426151464" Received: from yguo18-skl-u1604.sh.intel.com ([10.239.159.53]) by orsmga005.jf.intel.com with ESMTP; 11 Apr 2020 01:40:59 -0700 From: "Guo, Yejun" To: ffmpeg-devel@ffmpeg.org Date: Sat, 11 Apr 2020 16:39:31 +0800 Message-Id: <1586594371-16799-1-git-send-email-yejun.guo@intel.com> X-Mailer: git-send-email 2.7.4 Subject: [FFmpeg-devel] [PATCH 1/6] dnn/native: add native support for add X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Cc: yejun.guo@intel.com MIME-Version: 1.0 Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" X-TUID: nRweFpCl/SxW Content-Length: 7810 It can be tested with the model file generated with below python script: import tensorflow as tf import numpy as np import imageio in_img = imageio.imread('input.jpg') in_img = in_img.astype(np.float32)/255.0 in_data = in_img[np.newaxis, :] x = tf.placeholder(tf.float32, shape=[1, None, None, 3], name='dnn_in') z1 = 0.039 + x z2 = x + 0.042 z3 = z1 + z2 z4 = z3 - 0.381 z5 = z4 - x y = tf.math.maximum(z5, 0.0, name='dnn_out') sess=tf.Session() sess.run(tf.global_variables_initializer()) graph_def = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, ['dnn_out']) tf.train.write_graph(graph_def, '.', 'image_process.pb', as_text=False) print("image_process.pb generated, please use \ path_to_ffmpeg/tools/python/convert.py to generate image_process.model\n") output = sess.run(y, feed_dict={x: in_data}) imageio.imsave("out.jpg", np.squeeze(output)) Signed-off-by: Guo, Yejun --- libavfilter/dnn/dnn_backend_native_layer_mathbinary.c | 13 +++++++++++++ libavfilter/dnn/dnn_backend_native_layer_mathbinary.h | 1 + tools/python/convert_from_tensorflow.py | 15 +++++++-------- tools/python/convert_header.py | 2 +- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c b/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c index 3b8bab8..3fe337f 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c +++ b/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c @@ -107,6 +107,19 @@ int dnn_execute_layer_math_binary(DnnOperand *operands, const int32_t *input_ope } } return 0; + case DMBO_ADD: + if (params->input0_broadcast || params->input1_broadcast) { + for (int i = 0; i < dims_count; ++i) { + dst[i] = params->v + src[i]; + } + } else { + const DnnOperand *input1 = &operands[input_operand_indexes[1]]; + const float *src1 = input1->data; + for (int i = 0; i < dims_count; ++i) { + dst[i] = src[i] + src1[i]; + } + } + return 0; default: return -1; } diff --git a/libavfilter/dnn/dnn_backend_native_layer_mathbinary.h b/libavfilter/dnn/dnn_backend_native_layer_mathbinary.h index 6b684d1..3c5bc6b 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_mathbinary.h +++ b/libavfilter/dnn/dnn_backend_native_layer_mathbinary.h @@ -32,6 +32,7 @@ typedef enum { DMBO_SUB = 0, + DMBO_ADD = 1, DMBO_COUNT } DNNMathBinaryOperation; diff --git a/tools/python/convert_from_tensorflow.py b/tools/python/convert_from_tensorflow.py index 2485f16..9a495c0 100644 --- a/tools/python/convert_from_tensorflow.py +++ b/tools/python/convert_from_tensorflow.py @@ -71,7 +71,7 @@ class TFConverter: self.conv2d_scope_names = set() self.conv2d_scopename_inputname_dict = {} self.op2code = {'Conv2D':1, 'DepthToSpace':2, 'MirrorPad':3, 'Maximum':4, 'MathBinary':5} - self.mathbin2code = {'Sub':0} + self.mathbin2code = {'Sub':0, 'Add':1} self.mirrorpad_mode = {'CONSTANT':0, 'REFLECT':1, 'SYMMETRIC':2} self.name_operand_dict = {} @@ -255,8 +255,7 @@ class TFConverter: np.array([input_operand_index, output_operand_index], dtype=np.uint32).tofile(f) - def dump_sub_to_file(self, node, f): - assert(node.op == 'Sub') + def dump_mathbinary_to_file(self, node, f): self.layer_number = self.layer_number + 1 self.converted_nodes.add(node.name) i0_node = self.name_node_dict[node.input[0]] @@ -264,15 +263,13 @@ class TFConverter: np.array([self.op2code['MathBinary'], self.mathbin2code[node.op]], dtype=np.uint32).tofile(f) if i0_node.op == 'Const': scalar = i0_node.attr['value'].tensor.float_val[0] - assert(i0_node.name.find('sub/x')) - np.array([1], dtype=np.uint32).tofile(f) + np.array([1], dtype=np.uint32).tofile(f) # broadcast: 1 np.array([scalar], dtype=np.float32).tofile(f) - np.array([0], dtype=np.uint32).tofile(f) + np.array([0], dtype=np.uint32).tofile(f) # broadcast: 0 input_operand_index = self.add_operand(i1_node.name, Operand.IOTYPE_INPUT) np.array([input_operand_index], dtype=np.uint32).tofile(f) elif i1_node.op == 'Const': scalar = i1_node.attr['value'].tensor.float_val[0] - assert(i1_node.name.find('sub/y')) np.array([0], dtype=np.uint32).tofile(f) input_operand_index = self.add_operand(i0_node.name, Operand.IOTYPE_INPUT) np.array([input_operand_index], dtype=np.uint32).tofile(f) @@ -309,7 +306,9 @@ class TFConverter: elif node.op == 'Maximum': self.dump_maximum_to_file(node, f) elif node.op == 'Sub': - self.dump_sub_to_file(node, f) + self.dump_mathbinary_to_file(node, f) + elif node.op == 'Add': + self.dump_mathbinary_to_file(node, f) def dump_operands_to_file(self, f): diff --git a/tools/python/convert_header.py b/tools/python/convert_header.py index 6576fca..7027022 100644 --- a/tools/python/convert_header.py +++ b/tools/python/convert_header.py @@ -23,4 +23,4 @@ str = 'FFMPEGDNNNATIVE' major = 1 # increase minor when we don't have to re-convert the model file -minor = 1 +minor = 2