From patchwork Sat Apr 11 08:39:51 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Guo, Yejun" X-Patchwork-Id: 18870 Delivered-To: andriy.gelman@gmail.com Received: by 2002:ad4:45d0:0:0:0:0:0 with SMTP id v16csp608455qvt; Sat, 11 Apr 2020 01:41:33 -0700 (PDT) X-Google-Smtp-Source: APiQypIW3NxdvIKXVlPXTVF59ytvHNdR1tVp3vk7C/qe0p/ky7b9uG7Mi97+u5vlht18H5+oYYC1 X-Received: by 2002:a5d:4ecf:: with SMTP id s15mr9243993wrv.226.1586594493777; Sat, 11 Apr 2020 01:41:33 -0700 (PDT) ARC-Seal: i=1; a=rsa-sha256; t=1586594493; cv=none; d=google.com; s=arc-20160816; b=iIEhRG2glSqf0mXAcCOBwdBarvFRuoubTRdYmv3JrQ5W10UGHA9EYanDZ0CCLhkj1T 3AESEpt1fGMaL0Tug4NWSqp+3+iIohEay2O+AgXyfE2pikklLNg23tW0Ozas4Eb+Q2w0 zFefOvZEuxqiu6z6JgzwNRDcu7OSCcsFAh/R0CXfXPOQxpNE4FNdae7k6y7YFejdEOW9 /Xdi2JEoRAX8bBK4UF5f5AeAhL2dWiiX+GiO6VeWbmPp8cPK8t8kChWBMNDFQuyuHbxb +9b3eXDiApzFeaJpqrrvF12QYq9fq7+MbT8eQv0Ln+JISAJnO0MaVUU06UPi331/nK0W co/g== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=sender:errors-to:content-transfer-encoding:mime-version:cc:reply-to :list-subscribe:list-help:list-post:list-archive:list-unsubscribe :list-id:precedence:subject:message-id:date:to:from:ironport-sdr :ironport-sdr:delivered-to; bh=TWEXw5P3qxIrxjNKJD8rJmQh2U04TggbSlIiklHYIc8=; b=Mk5qgOnVfkKEPHfxuYtrc5HvMSq4mMic+8v7Scbs/6lVCnJNs/e2wYpEVYJDTDu3dF EdeN+MMSiNMibSj0kwNXiPOrkBV++kBLiGPxnf7+bT00Ec7PiK4pN3n0WqdAIIFf+nfD /2BdlRVfA6nf8+7Uky5Vksg/Yu/NJ0oHhe9cNiV2wPKQCVrNPjbids65zGHIeKTK4MPj +bx34UIW9oH2j23x8OR1zJ5GOkWpywz264h7rARLpRzYOVN1gQ/sJ22+T1CLKmRG3iqg g4xmuvSYjNHs3mvdUs7jsNNLq0ViB0Y2DoibfCyzcWkXzATC/Cr2aI3dfK45+j9vdUsc juPA== ARC-Authentication-Results: i=1; mx.google.com; spf=pass (google.com: domain of ffmpeg-devel-bounces@ffmpeg.org designates 79.124.17.100 as permitted sender) smtp.mailfrom=ffmpeg-devel-bounces@ffmpeg.org; dmarc=fail (p=NONE sp=NONE dis=NONE) header.from=intel.com Return-Path: Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org. [79.124.17.100]) by mx.google.com with ESMTP id i127si4611755wma.83.2020.04.11.01.41.33; Sat, 11 Apr 2020 01:41:33 -0700 (PDT) Received-SPF: pass (google.com: domain of ffmpeg-devel-bounces@ffmpeg.org designates 79.124.17.100 as permitted sender) client-ip=79.124.17.100; Authentication-Results: mx.google.com; spf=pass (google.com: domain of ffmpeg-devel-bounces@ffmpeg.org designates 79.124.17.100 as permitted sender) smtp.mailfrom=ffmpeg-devel-bounces@ffmpeg.org; dmarc=fail (p=NONE sp=NONE dis=NONE) header.from=intel.com Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id CB48968B096; Sat, 11 Apr 2020 11:41:29 +0300 (EEST) X-Original-To: ffmpeg-devel@ffmpeg.org Delivered-To: ffmpeg-devel@ffmpeg.org Received: from mga02.intel.com (mga02.intel.com [134.134.136.20]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id 7B9EA68ABB7 for ; Sat, 11 Apr 2020 11:41:22 +0300 (EEST) IronPort-SDR: JEsqncp6osEwvzbARO3bBCqZGJkNl/w3PesxYB7y4/N1S3z5gXV7cMhdHW50iPBtnU9wJWRMNM /cO1JHxgH9Qw== X-Amp-Result: SKIPPED(no attachment in message) X-Amp-File-Uploaded: False Received: from orsmga005.jf.intel.com ([10.7.209.41]) by orsmga101.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 11 Apr 2020 01:41:20 -0700 IronPort-SDR: HolWEqGKPwwcXl+NZrZpc5xbuqwkFU+vwEJ2G0PKewWfYDsV1324OmqdprnqRxKi3V7+E9xHzQ F5AX3wY8yDjw== X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.72,370,1580803200"; d="scan'208";a="426151558" Received: from yguo18-skl-u1604.sh.intel.com ([10.239.159.53]) by orsmga005.jf.intel.com with ESMTP; 11 Apr 2020 01:41:19 -0700 From: "Guo, Yejun" To: ffmpeg-devel@ffmpeg.org Date: Sat, 11 Apr 2020 16:39:51 +0800 Message-Id: <1586594391-16891-1-git-send-email-yejun.guo@intel.com> X-Mailer: git-send-email 2.7.4 Subject: [FFmpeg-devel] [PATCH 3/6] dnn/native: add native support for mul X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Cc: yejun.guo@intel.com MIME-Version: 1.0 Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" X-TUID: jBxRMULLeW02 Content-Length: 5585 it can be tested with model file generated from above python script: import tensorflow as tf import numpy as np import imageio in_img = imageio.imread('input.jpg') in_img = in_img.astype(np.float32)/255.0 in_data = in_img[np.newaxis, :] x = tf.placeholder(tf.float32, shape=[1, None, None, 3], name='dnn_in') z1 = 0.5 + 0.3 * x z2 = z1 * 4 z3 = z2 - x - 2.0 y = tf.identity(z3, name='dnn_out') sess=tf.Session() sess.run(tf.global_variables_initializer()) graph_def = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, ['dnn_out']) tf.train.write_graph(graph_def, '.', 'image_process.pb', as_text=False) print("image_process.pb generated, please use \ path_to_ffmpeg/tools/python/convert.py to generate image_process.model\n") output = sess.run(y, feed_dict={x: in_data}) imageio.imsave("out.jpg", np.squeeze(output)) Signed-off-by: Guo, Yejun --- libavfilter/dnn/dnn_backend_native_layer_mathbinary.c | 13 +++++++++++++ libavfilter/dnn/dnn_backend_native_layer_mathbinary.h | 1 + tools/python/convert_from_tensorflow.py | 4 +++- tools/python/convert_header.py | 2 +- 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c b/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c index 3fe337f..222941e 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c +++ b/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c @@ -120,6 +120,19 @@ int dnn_execute_layer_math_binary(DnnOperand *operands, const int32_t *input_ope } } return 0; + case DMBO_MUL: + if (params->input0_broadcast || params->input1_broadcast) { + for (int i = 0; i < dims_count; ++i) { + dst[i] = params->v * src[i]; + } + } else { + const DnnOperand *input1 = &operands[input_operand_indexes[1]]; + const float *src1 = input1->data; + for (int i = 0; i < dims_count; ++i) { + dst[i] = src[i] * src1[i]; + } + } + return 0; default: return -1; } diff --git a/libavfilter/dnn/dnn_backend_native_layer_mathbinary.h b/libavfilter/dnn/dnn_backend_native_layer_mathbinary.h index 3c5bc6b..d58b48c 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_mathbinary.h +++ b/libavfilter/dnn/dnn_backend_native_layer_mathbinary.h @@ -33,6 +33,7 @@ typedef enum { DMBO_SUB = 0, DMBO_ADD = 1, + DMBO_MUL = 2, DMBO_COUNT } DNNMathBinaryOperation; diff --git a/tools/python/convert_from_tensorflow.py b/tools/python/convert_from_tensorflow.py index 9a495c0..dc3b4e3 100644 --- a/tools/python/convert_from_tensorflow.py +++ b/tools/python/convert_from_tensorflow.py @@ -71,7 +71,7 @@ class TFConverter: self.conv2d_scope_names = set() self.conv2d_scopename_inputname_dict = {} self.op2code = {'Conv2D':1, 'DepthToSpace':2, 'MirrorPad':3, 'Maximum':4, 'MathBinary':5} - self.mathbin2code = {'Sub':0, 'Add':1} + self.mathbin2code = {'Sub':0, 'Add':1, 'Mul':2} self.mirrorpad_mode = {'CONSTANT':0, 'REFLECT':1, 'SYMMETRIC':2} self.name_operand_dict = {} @@ -309,6 +309,8 @@ class TFConverter: self.dump_mathbinary_to_file(node, f) elif node.op == 'Add': self.dump_mathbinary_to_file(node, f) + elif node.op == 'Mul': + self.dump_mathbinary_to_file(node, f) def dump_operands_to_file(self, f): diff --git a/tools/python/convert_header.py b/tools/python/convert_header.py index 7027022..87899fe 100644 --- a/tools/python/convert_header.py +++ b/tools/python/convert_header.py @@ -23,4 +23,4 @@ str = 'FFMPEGDNNNATIVE' major = 1 # increase minor when we don't have to re-convert the model file -minor = 2 +minor = 3