| Message ID | 1563256545-28027-1-git-send-email-yejun.guo@intel.com |
|---|---|
| State | New |
| Headers | show |
Hi, It fails fate source guard header tests, The headers should be changed from AVFILTER_DNN_BACKEND_xxx to AVFILTER_DNN_DNN_BACKEND_xxx. Other than that it LGTM. Em ter, 16 de jul de 2019 às 02:58, Guo, Yejun <yejun.guo@intel.com> escreveu: > > it is expected that there will be more files to support native mode, > so put all the dnn codes under libavfilter/dnn > > The main change of this patch is to move the file location, see below: > modified: libavfilter/Makefile > new file: libavfilter/dnn/Makefile > renamed: libavfilter/dnn_backend_native.c -> libavfilter/dnn/dnn_backend_native.c > renamed: libavfilter/dnn_backend_native.h -> libavfilter/dnn/dnn_backend_native.h > renamed: libavfilter/dnn_backend_tf.c -> libavfilter/dnn/dnn_backend_tf.c > renamed: libavfilter/dnn_backend_tf.h -> libavfilter/dnn/dnn_backend_tf.h > renamed: libavfilter/dnn_interface.c -> libavfilter/dnn/dnn_interface.c > > Signed-off-by: Guo, Yejun <yejun.guo@intel.com> > --- > libavfilter/Makefile | 3 +- > libavfilter/dnn/Makefile | 6 + > libavfilter/dnn/dnn_backend_native.c | 389 ++++++++++++++++++++++ > libavfilter/dnn/dnn_backend_native.h | 74 +++++ > libavfilter/dnn/dnn_backend_tf.c | 603 +++++++++++++++++++++++++++++++++++ > libavfilter/dnn/dnn_backend_tf.h | 38 +++ > libavfilter/dnn/dnn_interface.c | 63 ++++ > libavfilter/dnn_backend_native.c | 389 ---------------------- > libavfilter/dnn_backend_native.h | 74 ----- > libavfilter/dnn_backend_tf.c | 603 ----------------------------------- > libavfilter/dnn_backend_tf.h | 38 --- > libavfilter/dnn_interface.c | 63 ---- > 12 files changed, 1174 insertions(+), 1169 deletions(-) > create mode 100644 libavfilter/dnn/Makefile > create mode 100644 libavfilter/dnn/dnn_backend_native.c > create mode 100644 libavfilter/dnn/dnn_backend_native.h > create mode 100644 libavfilter/dnn/dnn_backend_tf.c > create mode 100644 libavfilter/dnn/dnn_backend_tf.h > create mode 100644 libavfilter/dnn/dnn_interface.c > delete mode 100644 libavfilter/dnn_backend_native.c > delete mode 100644 libavfilter/dnn_backend_native.h > delete mode 100644 libavfilter/dnn_backend_tf.c > delete mode 100644 libavfilter/dnn_backend_tf.h > delete mode 100644 libavfilter/dnn_interface.c > > diff --git a/libavfilter/Makefile b/libavfilter/Makefile > index 455c809..450d781 100644 > --- a/libavfilter/Makefile > +++ b/libavfilter/Makefile > @@ -26,9 +26,8 @@ OBJS-$(HAVE_THREADS) += pthread.o > > # subsystems > OBJS-$(CONFIG_QSVVPP) += qsvvpp.o > -DNN-OBJS-$(CONFIG_LIBTENSORFLOW) += dnn_backend_tf.o > -OBJS-$(CONFIG_DNN) += dnn_interface.o dnn_backend_native.o $(DNN-OBJS-yes) > OBJS-$(CONFIG_SCENE_SAD) += scene_sad.o > +include $(SRC_PATH)/libavfilter/dnn/Makefile > > # audio filters > OBJS-$(CONFIG_ABENCH_FILTER) += f_bench.o > diff --git a/libavfilter/dnn/Makefile b/libavfilter/dnn/Makefile > new file mode 100644 > index 0000000..1d12ade > --- /dev/null > +++ b/libavfilter/dnn/Makefile > @@ -0,0 +1,6 @@ > +OBJS-$(CONFIG_DNN) += dnn/dnn_interface.o > +OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native.o > + > +DNN-OBJS-$(CONFIG_LIBTENSORFLOW) += dnn/dnn_backend_tf.o > + > +OBJS-$(CONFIG_DNN) += $(DNN-OBJS-yes) > diff --git a/libavfilter/dnn/dnn_backend_native.c b/libavfilter/dnn/dnn_backend_native.c > new file mode 100644 > index 0000000..82e900b > --- /dev/null > +++ b/libavfilter/dnn/dnn_backend_native.c > @@ -0,0 +1,389 @@ > +/* > + * Copyright (c) 2018 Sergey Lavrushkin > + * > + * This file is part of FFmpeg. > + * > + * FFmpeg is free software; you can redistribute it and/or > + * modify it under the terms of the GNU Lesser General Public > + * License as published by the Free Software Foundation; either > + * version 2.1 of the License, or (at your option) any later version. > + * > + * FFmpeg is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + * Lesser General Public License for more details. > + * > + * You should have received a copy of the GNU Lesser General Public > + * License along with FFmpeg; if not, write to the Free Software > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > + */ > + > +/** > + * @file > + * DNN native backend implementation. > + */ > + > +#include "dnn_backend_native.h" > +#include "libavutil/avassert.h" > + > +static DNNReturnType set_input_output_native(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output) > +{ > + ConvolutionalNetwork *network = (ConvolutionalNetwork *)model; > + InputParams *input_params; > + ConvolutionalParams *conv_params; > + DepthToSpaceParams *depth_to_space_params; > + int cur_width, cur_height, cur_channels; > + int32_t layer; > + > + if (network->layers_num <= 0 || network->layers[0].type != INPUT){ > + return DNN_ERROR; > + } > + else{ > + input_params = (InputParams *)network->layers[0].params; > + input_params->width = cur_width = input->width; > + input_params->height = cur_height = input->height; > + input_params->channels = cur_channels = input->channels; > + if (input->data){ > + av_freep(&input->data); > + } > + av_assert0(input->dt == DNN_FLOAT); > + network->layers[0].output = input->data = av_malloc(cur_height * cur_width * cur_channels * sizeof(float)); > + if (!network->layers[0].output){ > + return DNN_ERROR; > + } > + } > + > + for (layer = 1; layer < network->layers_num; ++layer){ > + switch (network->layers[layer].type){ > + case CONV: > + conv_params = (ConvolutionalParams *)network->layers[layer].params; > + if (conv_params->input_num != cur_channels){ > + return DNN_ERROR; > + } > + cur_channels = conv_params->output_num; > + > + if (conv_params->padding_method == VALID) { > + int pad_size = (conv_params->kernel_size - 1) * conv_params->dilation; > + cur_height -= pad_size; > + cur_width -= pad_size; > + } > + break; > + case DEPTH_TO_SPACE: > + depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params; > + if (cur_channels % (depth_to_space_params->block_size * depth_to_space_params->block_size) != 0){ > + return DNN_ERROR; > + } > + cur_channels = cur_channels / (depth_to_space_params->block_size * depth_to_space_params->block_size); > + cur_height *= depth_to_space_params->block_size; > + cur_width *= depth_to_space_params->block_size; > + break; > + default: > + return DNN_ERROR; > + } > + if (network->layers[layer].output){ > + av_freep(&network->layers[layer].output); > + } > + > + if (cur_height <= 0 || cur_width <= 0) > + return DNN_ERROR; > + > + network->layers[layer].output = av_malloc(cur_height * cur_width * cur_channels * sizeof(float)); > + if (!network->layers[layer].output){ > + return DNN_ERROR; > + } > + } > + > + return DNN_SUCCESS; > +} > + > +// Loads model and its parameters that are stored in a binary file with following structure: > +// layers_num,layer_type,layer_parameterss,layer_type,layer_parameters... > +// For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases > +// For DEPTH_TO_SPACE layer: block_size > +DNNModel *ff_dnn_load_model_native(const char *model_filename) > +{ > + DNNModel *model = NULL; > + ConvolutionalNetwork *network = NULL; > + AVIOContext *model_file_context; > + int file_size, dnn_size, kernel_size, i; > + int32_t layer; > + DNNLayerType layer_type; > + ConvolutionalParams *conv_params; > + DepthToSpaceParams *depth_to_space_params; > + > + model = av_malloc(sizeof(DNNModel)); > + if (!model){ > + return NULL; > + } > + > + if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){ > + av_freep(&model); > + return NULL; > + } > + file_size = avio_size(model_file_context); > + > + network = av_malloc(sizeof(ConvolutionalNetwork)); > + if (!network){ > + avio_closep(&model_file_context); > + av_freep(&model); > + return NULL; > + } > + model->model = (void *)network; > + > + network->layers_num = 1 + (int32_t)avio_rl32(model_file_context); > + dnn_size = 4; > + > + network->layers = av_malloc(network->layers_num * sizeof(Layer)); > + if (!network->layers){ > + av_freep(&network); > + avio_closep(&model_file_context); > + av_freep(&model); > + return NULL; > + } > + > + for (layer = 0; layer < network->layers_num; ++layer){ > + network->layers[layer].output = NULL; > + network->layers[layer].params = NULL; > + } > + network->layers[0].type = INPUT; > + network->layers[0].params = av_malloc(sizeof(InputParams)); > + if (!network->layers[0].params){ > + avio_closep(&model_file_context); > + ff_dnn_free_model_native(&model); > + return NULL; > + } > + > + for (layer = 1; layer < network->layers_num; ++layer){ > + layer_type = (int32_t)avio_rl32(model_file_context); > + dnn_size += 4; > + switch (layer_type){ > + case CONV: > + conv_params = av_malloc(sizeof(ConvolutionalParams)); > + if (!conv_params){ > + avio_closep(&model_file_context); > + ff_dnn_free_model_native(&model); > + return NULL; > + } > + conv_params->dilation = (int32_t)avio_rl32(model_file_context); > + conv_params->padding_method = (int32_t)avio_rl32(model_file_context); > + conv_params->activation = (int32_t)avio_rl32(model_file_context); > + conv_params->input_num = (int32_t)avio_rl32(model_file_context); > + conv_params->output_num = (int32_t)avio_rl32(model_file_context); > + conv_params->kernel_size = (int32_t)avio_rl32(model_file_context); > + kernel_size = conv_params->input_num * conv_params->output_num * > + conv_params->kernel_size * conv_params->kernel_size; > + dnn_size += 24 + (kernel_size + conv_params->output_num << 2); > + if (dnn_size > file_size || conv_params->input_num <= 0 || > + conv_params->output_num <= 0 || conv_params->kernel_size <= 0){ > + avio_closep(&model_file_context); > + ff_dnn_free_model_native(&model); > + return NULL; > + } > + conv_params->kernel = av_malloc(kernel_size * sizeof(float)); > + conv_params->biases = av_malloc(conv_params->output_num * sizeof(float)); > + if (!conv_params->kernel || !conv_params->biases){ > + avio_closep(&model_file_context); > + ff_dnn_free_model_native(&model); > + return NULL; > + } > + for (i = 0; i < kernel_size; ++i){ > + conv_params->kernel[i] = av_int2float(avio_rl32(model_file_context)); > + } > + for (i = 0; i < conv_params->output_num; ++i){ > + conv_params->biases[i] = av_int2float(avio_rl32(model_file_context)); > + } > + network->layers[layer].type = CONV; > + network->layers[layer].params = conv_params; > + break; > + case DEPTH_TO_SPACE: > + depth_to_space_params = av_malloc(sizeof(DepthToSpaceParams)); > + if (!depth_to_space_params){ > + avio_closep(&model_file_context); > + ff_dnn_free_model_native(&model); > + return NULL; > + } > + depth_to_space_params->block_size = (int32_t)avio_rl32(model_file_context); > + dnn_size += 4; > + network->layers[layer].type = DEPTH_TO_SPACE; > + network->layers[layer].params = depth_to_space_params; > + break; > + default: > + avio_closep(&model_file_context); > + ff_dnn_free_model_native(&model); > + return NULL; > + } > + } > + > + avio_closep(&model_file_context); > + > + if (dnn_size != file_size){ > + ff_dnn_free_model_native(&model); > + return NULL; > + } > + > + model->set_input_output = &set_input_output_native; > + > + return model; > +} > + > +#define CLAMP_TO_EDGE(x, w) ((x) < 0 ? 0 : ((x) >= (w) ? (w - 1) : (x))) > + > +static void convolve(const float *input, float *output, const ConvolutionalParams *conv_params, int width, int height) > +{ > + int radius = conv_params->kernel_size >> 1; > + int src_linesize = width * conv_params->input_num; > + int filter_linesize = conv_params->kernel_size * conv_params->input_num; > + int filter_size = conv_params->kernel_size * filter_linesize; > + int pad_size = (conv_params->padding_method == VALID) ? (conv_params->kernel_size - 1) / 2 * conv_params->dilation : 0; > + > + for (int y = pad_size; y < height - pad_size; ++y) { > + for (int x = pad_size; x < width - pad_size; ++x) { > + for (int n_filter = 0; n_filter < conv_params->output_num; ++n_filter) { > + output[n_filter] = conv_params->biases[n_filter]; > + > + for (int ch = 0; ch < conv_params->input_num; ++ch) { > + for (int kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y) { > + for (int kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x) { > + float input_pel; > + if (conv_params->padding_method == SAME_CLAMP_TO_EDGE) { > + int y_pos = CLAMP_TO_EDGE(y + (kernel_y - radius) * conv_params->dilation, height); > + int x_pos = CLAMP_TO_EDGE(x + (kernel_x - radius) * conv_params->dilation, width); > + input_pel = input[y_pos * src_linesize + x_pos * conv_params->input_num + ch]; > + } else { > + int y_pos = y + (kernel_y - radius) * conv_params->dilation; > + int x_pos = x + (kernel_x - radius) * conv_params->dilation; > + input_pel = (x_pos < 0 || x_pos >= width || y_pos < 0 || y_pos >= height) ? 0.0 : > + input[y_pos * src_linesize + x_pos * conv_params->input_num + ch]; > + } > + > + > + output[n_filter] += input_pel * conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize + > + kernel_x * conv_params->input_num + ch]; > + } > + } > + } > + switch (conv_params->activation){ > + case RELU: > + output[n_filter] = FFMAX(output[n_filter], 0.0); > + break; > + case TANH: > + output[n_filter] = 2.0f / (1.0f + exp(-2.0f * output[n_filter])) - 1.0f; > + break; > + case SIGMOID: > + output[n_filter] = 1.0f / (1.0f + exp(-output[n_filter])); > + break; > + case NONE: > + break; > + case LEAKY_RELU: > + output[n_filter] = FFMAX(output[n_filter], 0.0) + 0.2 * FFMIN(output[n_filter], 0.0); > + } > + } > + output += conv_params->output_num; > + } > + } > +} > + > +static void depth_to_space(const float *input, float *output, int block_size, int width, int height, int channels) > +{ > + int y, x, by, bx, ch; > + int new_channels = channels / (block_size * block_size); > + int output_linesize = width * channels; > + int by_linesize = output_linesize / block_size; > + int x_linesize = new_channels * block_size; > + > + for (y = 0; y < height; ++y){ > + for (x = 0; x < width; ++x){ > + for (by = 0; by < block_size; ++by){ > + for (bx = 0; bx < block_size; ++bx){ > + for (ch = 0; ch < new_channels; ++ch){ > + output[by * by_linesize + x * x_linesize + bx * new_channels + ch] = input[ch]; > + } > + input += new_channels; > + } > + } > + } > + output += output_linesize; > + } > +} > + > +DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output) > +{ > + ConvolutionalNetwork *network = (ConvolutionalNetwork *)model->model; > + int cur_width, cur_height, cur_channels; > + int32_t layer; > + InputParams *input_params; > + ConvolutionalParams *conv_params; > + DepthToSpaceParams *depth_to_space_params; > + > + if (network->layers_num <= 0 || network->layers[0].type != INPUT || !network->layers[0].output){ > + return DNN_ERROR; > + } > + else{ > + input_params = (InputParams *)network->layers[0].params; > + cur_width = input_params->width; > + cur_height = input_params->height; > + cur_channels = input_params->channels; > + } > + > + for (layer = 1; layer < network->layers_num; ++layer){ > + if (!network->layers[layer].output){ > + return DNN_ERROR; > + } > + switch (network->layers[layer].type){ > + case CONV: > + conv_params = (ConvolutionalParams *)network->layers[layer].params; > + convolve(network->layers[layer - 1].output, network->layers[layer].output, conv_params, cur_width, cur_height); > + cur_channels = conv_params->output_num; > + if (conv_params->padding_method == VALID) { > + int pad_size = (conv_params->kernel_size - 1) * conv_params->dilation; > + cur_height -= pad_size; > + cur_width -= pad_size; > + } > + break; > + case DEPTH_TO_SPACE: > + depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params; > + depth_to_space(network->layers[layer - 1].output, network->layers[layer].output, > + depth_to_space_params->block_size, cur_width, cur_height, cur_channels); > + cur_height *= depth_to_space_params->block_size; > + cur_width *= depth_to_space_params->block_size; > + cur_channels /= depth_to_space_params->block_size * depth_to_space_params->block_size; > + break; > + case INPUT: > + return DNN_ERROR; > + } > + } > + > + // native mode does not support multiple outputs yet > + if (nb_output > 1) > + return DNN_ERROR; > + outputs[0].data = network->layers[network->layers_num - 1].output; > + outputs[0].height = cur_height; > + outputs[0].width = cur_width; > + outputs[0].channels = cur_channels; > + > + return DNN_SUCCESS; > +} > + > +void ff_dnn_free_model_native(DNNModel **model) > +{ > + ConvolutionalNetwork *network; > + ConvolutionalParams *conv_params; > + int32_t layer; > + > + if (*model) > + { > + network = (ConvolutionalNetwork *)(*model)->model; > + for (layer = 0; layer < network->layers_num; ++layer){ > + av_freep(&network->layers[layer].output); > + if (network->layers[layer].type == CONV){ > + conv_params = (ConvolutionalParams *)network->layers[layer].params; > + av_freep(&conv_params->kernel); > + av_freep(&conv_params->biases); > + } > + av_freep(&network->layers[layer].params); > + } > + av_freep(&network->layers); > + av_freep(&network); > + av_freep(model); > + } > +} > diff --git a/libavfilter/dnn/dnn_backend_native.h b/libavfilter/dnn/dnn_backend_native.h > new file mode 100644 > index 0000000..532103c > --- /dev/null > +++ b/libavfilter/dnn/dnn_backend_native.h > @@ -0,0 +1,74 @@ > +/* > + * Copyright (c) 2018 Sergey Lavrushkin > + * > + * This file is part of FFmpeg. > + * > + * FFmpeg is free software; you can redistribute it and/or > + * modify it under the terms of the GNU Lesser General Public > + * License as published by the Free Software Foundation; either > + * version 2.1 of the License, or (at your option) any later version. > + * > + * FFmpeg is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + * Lesser General Public License for more details. > + * > + * You should have received a copy of the GNU Lesser General Public > + * License along with FFmpeg; if not, write to the Free Software > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > + */ > + > +/** > + * @file > + * DNN inference functions interface for native backend. > + */ > + > + > +#ifndef AVFILTER_DNN_BACKEND_NATIVE_H > +#define AVFILTER_DNN_BACKEND_NATIVE_H > + > +#include "../dnn_interface.h" > +#include "libavformat/avio.h" > + > +typedef enum {INPUT, CONV, DEPTH_TO_SPACE} DNNLayerType; > + > +typedef enum {RELU, TANH, SIGMOID, NONE, LEAKY_RELU} DNNActivationFunc; > + > +typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} DNNConvPaddingParam; > + > +typedef struct Layer{ > + DNNLayerType type; > + float *output; > + void *params; > +} Layer; > + > +typedef struct ConvolutionalParams{ > + int32_t input_num, output_num, kernel_size; > + DNNActivationFunc activation; > + DNNConvPaddingParam padding_method; > + int32_t dilation; > + float *kernel; > + float *biases; > +} ConvolutionalParams; > + > +typedef struct InputParams{ > + int height, width, channels; > +} InputParams; > + > +typedef struct DepthToSpaceParams{ > + int block_size; > +} DepthToSpaceParams; > + > +// Represents simple feed-forward convolutional network. > +typedef struct ConvolutionalNetwork{ > + Layer *layers; > + int32_t layers_num; > +} ConvolutionalNetwork; > + > +DNNModel *ff_dnn_load_model_native(const char *model_filename); > + > +DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output); > + > +void ff_dnn_free_model_native(DNNModel **model); > + > +#endif > diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c > new file mode 100644 > index 0000000..ba959ae > --- /dev/null > +++ b/libavfilter/dnn/dnn_backend_tf.c > @@ -0,0 +1,603 @@ > +/* > + * Copyright (c) 2018 Sergey Lavrushkin > + * > + * This file is part of FFmpeg. > + * > + * FFmpeg is free software; you can redistribute it and/or > + * modify it under the terms of the GNU Lesser General Public > + * License as published by the Free Software Foundation; either > + * version 2.1 of the License, or (at your option) any later version. > + * > + * FFmpeg is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + * Lesser General Public License for more details. > + * > + * You should have received a copy of the GNU Lesser General Public > + * License along with FFmpeg; if not, write to the Free Software > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > + */ > + > +/** > + * @file > + * DNN tensorflow backend implementation. > + */ > + > +#include "dnn_backend_tf.h" > +#include "dnn_backend_native.h" > +#include "libavformat/avio.h" > +#include "libavutil/avassert.h" > + > +#include <tensorflow/c/c_api.h> > + > +typedef struct TFModel{ > + TF_Graph *graph; > + TF_Session *session; > + TF_Status *status; > + TF_Output input; > + TF_Tensor *input_tensor; > + TF_Output *outputs; > + TF_Tensor **output_tensors; > + uint32_t nb_output; > +} TFModel; > + > +static void free_buffer(void *data, size_t length) > +{ > + av_freep(&data); > +} > + > +static TF_Buffer *read_graph(const char *model_filename) > +{ > + TF_Buffer *graph_buf; > + unsigned char *graph_data = NULL; > + AVIOContext *model_file_context; > + long size, bytes_read; > + > + if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){ > + return NULL; > + } > + > + size = avio_size(model_file_context); > + > + graph_data = av_malloc(size); > + if (!graph_data){ > + avio_closep(&model_file_context); > + return NULL; > + } > + bytes_read = avio_read(model_file_context, graph_data, size); > + avio_closep(&model_file_context); > + if (bytes_read != size){ > + av_freep(&graph_data); > + return NULL; > + } > + > + graph_buf = TF_NewBuffer(); > + graph_buf->data = (void *)graph_data; > + graph_buf->length = size; > + graph_buf->data_deallocator = free_buffer; > + > + return graph_buf; > +} > + > +static TF_Tensor *allocate_input_tensor(const DNNInputData *input) > +{ > + TF_DataType dt; > + size_t size; > + int64_t input_dims[] = {1, input->height, input->width, input->channels}; > + switch (input->dt) { > + case DNN_FLOAT: > + dt = TF_FLOAT; > + size = sizeof(float); > + break; > + case DNN_UINT8: > + dt = TF_UINT8; > + size = sizeof(char); > + break; > + default: > + av_assert0(!"should not reach here"); > + } > + > + return TF_AllocateTensor(dt, input_dims, 4, > + input_dims[1] * input_dims[2] * input_dims[3] * size); > +} > + > +static DNNReturnType set_input_output_tf(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output) > +{ > + TFModel *tf_model = (TFModel *)model; > + TF_SessionOptions *sess_opts; > + const TF_Operation *init_op = TF_GraphOperationByName(tf_model->graph, "init"); > + > + // Input operation > + tf_model->input.oper = TF_GraphOperationByName(tf_model->graph, input_name); > + if (!tf_model->input.oper){ > + return DNN_ERROR; > + } > + tf_model->input.index = 0; > + if (tf_model->input_tensor){ > + TF_DeleteTensor(tf_model->input_tensor); > + } > + tf_model->input_tensor = allocate_input_tensor(input); > + if (!tf_model->input_tensor){ > + return DNN_ERROR; > + } > + input->data = (float *)TF_TensorData(tf_model->input_tensor); > + > + // Output operation > + if (nb_output == 0) > + return DNN_ERROR; > + > + av_freep(&tf_model->outputs); > + tf_model->outputs = av_malloc_array(nb_output, sizeof(*tf_model->outputs)); > + if (!tf_model->outputs) > + return DNN_ERROR; > + for (int i = 0; i < nb_output; ++i) { > + tf_model->outputs[i].oper = TF_GraphOperationByName(tf_model->graph, output_names[i]); > + if (!tf_model->outputs[i].oper){ > + av_freep(&tf_model->outputs); > + return DNN_ERROR; > + } > + tf_model->outputs[i].index = 0; > + } > + > + if (tf_model->output_tensors) { > + for (uint32_t i = 0; i < tf_model->nb_output; ++i) { > + if (tf_model->output_tensors[i]) { > + TF_DeleteTensor(tf_model->output_tensors[i]); > + tf_model->output_tensors[i] = NULL; > + } > + } > + } > + av_freep(&tf_model->output_tensors); > + tf_model->output_tensors = av_mallocz_array(nb_output, sizeof(*tf_model->output_tensors)); > + if (!tf_model->output_tensors) { > + av_freep(&tf_model->outputs); > + return DNN_ERROR; > + } > + > + tf_model->nb_output = nb_output; > + > + if (tf_model->session){ > + TF_CloseSession(tf_model->session, tf_model->status); > + TF_DeleteSession(tf_model->session, tf_model->status); > + } > + > + sess_opts = TF_NewSessionOptions(); > + tf_model->session = TF_NewSession(tf_model->graph, sess_opts, tf_model->status); > + TF_DeleteSessionOptions(sess_opts); > + if (TF_GetCode(tf_model->status) != TF_OK) > + { > + return DNN_ERROR; > + } > + > + // Run initialization operation with name "init" if it is present in graph > + if (init_op){ > + TF_SessionRun(tf_model->session, NULL, > + NULL, NULL, 0, > + NULL, NULL, 0, > + &init_op, 1, NULL, tf_model->status); > + if (TF_GetCode(tf_model->status) != TF_OK) > + { > + return DNN_ERROR; > + } > + } > + > + return DNN_SUCCESS; > +} > + > +static DNNReturnType load_tf_model(TFModel *tf_model, const char *model_filename) > +{ > + TF_Buffer *graph_def; > + TF_ImportGraphDefOptions *graph_opts; > + > + graph_def = read_graph(model_filename); > + if (!graph_def){ > + return DNN_ERROR; > + } > + tf_model->graph = TF_NewGraph(); > + tf_model->status = TF_NewStatus(); > + graph_opts = TF_NewImportGraphDefOptions(); > + TF_GraphImportGraphDef(tf_model->graph, graph_def, graph_opts, tf_model->status); > + TF_DeleteImportGraphDefOptions(graph_opts); > + TF_DeleteBuffer(graph_def); > + if (TF_GetCode(tf_model->status) != TF_OK){ > + TF_DeleteGraph(tf_model->graph); > + TF_DeleteStatus(tf_model->status); > + return DNN_ERROR; > + } > + > + return DNN_SUCCESS; > +} > + > +#define NAME_BUFFER_SIZE 256 > + > +static DNNReturnType add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op, > + ConvolutionalParams* params, const int layer) > +{ > + TF_Operation *op; > + TF_OperationDescription *op_desc; > + TF_Output input; > + int64_t strides[] = {1, 1, 1, 1}; > + TF_Tensor *tensor; > + int64_t dims[4]; > + int dims_len; > + char name_buffer[NAME_BUFFER_SIZE]; > + int32_t size; > + > + size = params->input_num * params->output_num * params->kernel_size * params->kernel_size; > + input.index = 0; > + > + snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_kernel%d", layer); > + op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer); > + TF_SetAttrType(op_desc, "dtype", TF_FLOAT); > + dims[0] = params->output_num; > + dims[1] = params->kernel_size; > + dims[2] = params->kernel_size; > + dims[3] = params->input_num; > + dims_len = 4; > + tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, size * sizeof(float)); > + memcpy(TF_TensorData(tensor), params->kernel, size * sizeof(float)); > + TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); > + if (TF_GetCode(tf_model->status) != TF_OK){ > + return DNN_ERROR; > + } > + op = TF_FinishOperation(op_desc, tf_model->status); > + if (TF_GetCode(tf_model->status) != TF_OK){ > + return DNN_ERROR; > + } > + > + snprintf(name_buffer, NAME_BUFFER_SIZE, "transpose%d", layer); > + op_desc = TF_NewOperation(tf_model->graph, "Transpose", name_buffer); > + input.oper = op; > + TF_AddInput(op_desc, input); > + input.oper = transpose_op; > + TF_AddInput(op_desc, input); > + TF_SetAttrType(op_desc, "T", TF_FLOAT); > + TF_SetAttrType(op_desc, "Tperm", TF_INT32); > + op = TF_FinishOperation(op_desc, tf_model->status); > + if (TF_GetCode(tf_model->status) != TF_OK){ > + return DNN_ERROR; > + } > + > + snprintf(name_buffer, NAME_BUFFER_SIZE, "conv2d%d", layer); > + op_desc = TF_NewOperation(tf_model->graph, "Conv2D", name_buffer); > + input.oper = *cur_op; > + TF_AddInput(op_desc, input); > + input.oper = op; > + TF_AddInput(op_desc, input); > + TF_SetAttrType(op_desc, "T", TF_FLOAT); > + TF_SetAttrIntList(op_desc, "strides", strides, 4); > + TF_SetAttrString(op_desc, "padding", "VALID", 5); > + *cur_op = TF_FinishOperation(op_desc, tf_model->status); > + if (TF_GetCode(tf_model->status) != TF_OK){ > + return DNN_ERROR; > + } > + > + snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_biases%d", layer); > + op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer); > + TF_SetAttrType(op_desc, "dtype", TF_FLOAT); > + dims[0] = params->output_num; > + dims_len = 1; > + tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, params->output_num * sizeof(float)); > + memcpy(TF_TensorData(tensor), params->biases, params->output_num * sizeof(float)); > + TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); > + if (TF_GetCode(tf_model->status) != TF_OK){ > + return DNN_ERROR; > + } > + op = TF_FinishOperation(op_desc, tf_model->status); > + if (TF_GetCode(tf_model->status) != TF_OK){ > + return DNN_ERROR; > + } > + > + snprintf(name_buffer, NAME_BUFFER_SIZE, "bias_add%d", layer); > + op_desc = TF_NewOperation(tf_model->graph, "BiasAdd", name_buffer); > + input.oper = *cur_op; > + TF_AddInput(op_desc, input); > + input.oper = op; > + TF_AddInput(op_desc, input); > + TF_SetAttrType(op_desc, "T", TF_FLOAT); > + *cur_op = TF_FinishOperation(op_desc, tf_model->status); > + if (TF_GetCode(tf_model->status) != TF_OK){ > + return DNN_ERROR; > + } > + > + snprintf(name_buffer, NAME_BUFFER_SIZE, "activation%d", layer); > + switch (params->activation){ > + case RELU: > + op_desc = TF_NewOperation(tf_model->graph, "Relu", name_buffer); > + break; > + case TANH: > + op_desc = TF_NewOperation(tf_model->graph, "Tanh", name_buffer); > + break; > + case SIGMOID: > + op_desc = TF_NewOperation(tf_model->graph, "Sigmoid", name_buffer); > + break; > + default: > + return DNN_ERROR; > + } > + input.oper = *cur_op; > + TF_AddInput(op_desc, input); > + TF_SetAttrType(op_desc, "T", TF_FLOAT); > + *cur_op = TF_FinishOperation(op_desc, tf_model->status); > + if (TF_GetCode(tf_model->status) != TF_OK){ > + return DNN_ERROR; > + } > + > + return DNN_SUCCESS; > +} > + > +static DNNReturnType add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op, > + DepthToSpaceParams *params, const int layer) > +{ > + TF_OperationDescription *op_desc; > + TF_Output input; > + char name_buffer[NAME_BUFFER_SIZE]; > + > + snprintf(name_buffer, NAME_BUFFER_SIZE, "depth_to_space%d", layer); > + op_desc = TF_NewOperation(tf_model->graph, "DepthToSpace", name_buffer); > + input.oper = *cur_op; > + input.index = 0; > + TF_AddInput(op_desc, input); > + TF_SetAttrType(op_desc, "T", TF_FLOAT); > + TF_SetAttrInt(op_desc, "block_size", params->block_size); > + *cur_op = TF_FinishOperation(op_desc, tf_model->status); > + if (TF_GetCode(tf_model->status) != TF_OK){ > + return DNN_ERROR; > + } > + > + return DNN_SUCCESS; > +} > + > +static int calculate_pad(const ConvolutionalNetwork *conv_network) > +{ > + ConvolutionalParams *params; > + int32_t layer; > + int pad = 0; > + > + for (layer = 0; layer < conv_network->layers_num; ++layer){ > + if (conv_network->layers[layer].type == CONV){ > + params = (ConvolutionalParams *)conv_network->layers[layer].params; > + pad += params->kernel_size >> 1; > + } > + } > + > + return pad; > +} > + > +static DNNReturnType add_pad_op(TFModel *tf_model, TF_Operation **cur_op, const int32_t pad) > +{ > + TF_Operation *op; > + TF_Tensor *tensor; > + TF_OperationDescription *op_desc; > + TF_Output input; > + int32_t *pads; > + int64_t pads_shape[] = {4, 2}; > + > + input.index = 0; > + > + op_desc = TF_NewOperation(tf_model->graph, "Const", "pads"); > + TF_SetAttrType(op_desc, "dtype", TF_INT32); > + tensor = TF_AllocateTensor(TF_INT32, pads_shape, 2, 4 * 2 * sizeof(int32_t)); > + pads = (int32_t *)TF_TensorData(tensor); > + pads[0] = 0; pads[1] = 0; > + pads[2] = pad; pads[3] = pad; > + pads[4] = pad; pads[5] = pad; > + pads[6] = 0; pads[7] = 0; > + TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); > + if (TF_GetCode(tf_model->status) != TF_OK){ > + return DNN_ERROR; > + } > + op = TF_FinishOperation(op_desc, tf_model->status); > + if (TF_GetCode(tf_model->status) != TF_OK){ > + return DNN_ERROR; > + } > + > + op_desc = TF_NewOperation(tf_model->graph, "MirrorPad", "mirror_pad"); > + input.oper = *cur_op; > + TF_AddInput(op_desc, input); > + input.oper = op; > + TF_AddInput(op_desc, input); > + TF_SetAttrType(op_desc, "T", TF_FLOAT); > + TF_SetAttrType(op_desc, "Tpaddings", TF_INT32); > + TF_SetAttrString(op_desc, "mode", "SYMMETRIC", 9); > + *cur_op = TF_FinishOperation(op_desc, tf_model->status); > + if (TF_GetCode(tf_model->status) != TF_OK){ > + return DNN_ERROR; > + } > + > + return DNN_SUCCESS; > +} > + > +static DNNReturnType load_native_model(TFModel *tf_model, const char *model_filename) > +{ > + int32_t layer; > + TF_OperationDescription *op_desc; > + TF_Operation *op; > + TF_Operation *transpose_op; > + TF_Tensor *tensor; > + TF_Output input; > + int32_t *transpose_perm; > + int64_t transpose_perm_shape[] = {4}; > + int64_t input_shape[] = {1, -1, -1, -1}; > + int32_t pad; > + DNNReturnType layer_add_res; > + DNNModel *native_model = NULL; > + ConvolutionalNetwork *conv_network; > + > + native_model = ff_dnn_load_model_native(model_filename); > + if (!native_model){ > + return DNN_ERROR; > + } > + > + conv_network = (ConvolutionalNetwork *)native_model->model; > + pad = calculate_pad(conv_network); > + tf_model->graph = TF_NewGraph(); > + tf_model->status = TF_NewStatus(); > + > +#define CLEANUP_ON_ERROR(tf_model) \ > + { \ > + TF_DeleteGraph(tf_model->graph); \ > + TF_DeleteStatus(tf_model->status); \ > + return DNN_ERROR; \ > + } > + > + op_desc = TF_NewOperation(tf_model->graph, "Placeholder", "x"); > + TF_SetAttrType(op_desc, "dtype", TF_FLOAT); > + TF_SetAttrShape(op_desc, "shape", input_shape, 4); > + op = TF_FinishOperation(op_desc, tf_model->status); > + if (TF_GetCode(tf_model->status) != TF_OK){ > + CLEANUP_ON_ERROR(tf_model); > + } > + > + if (add_pad_op(tf_model, &op, pad) != DNN_SUCCESS){ > + CLEANUP_ON_ERROR(tf_model); > + } > + > + op_desc = TF_NewOperation(tf_model->graph, "Const", "transpose_perm"); > + TF_SetAttrType(op_desc, "dtype", TF_INT32); > + tensor = TF_AllocateTensor(TF_INT32, transpose_perm_shape, 1, 4 * sizeof(int32_t)); > + transpose_perm = (int32_t *)TF_TensorData(tensor); > + transpose_perm[0] = 1; > + transpose_perm[1] = 2; > + transpose_perm[2] = 3; > + transpose_perm[3] = 0; > + TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); > + if (TF_GetCode(tf_model->status) != TF_OK){ > + CLEANUP_ON_ERROR(tf_model); > + } > + transpose_op = TF_FinishOperation(op_desc, tf_model->status); > + > + for (layer = 0; layer < conv_network->layers_num; ++layer){ > + switch (conv_network->layers[layer].type){ > + case INPUT: > + layer_add_res = DNN_SUCCESS; > + break; > + case CONV: > + layer_add_res = add_conv_layer(tf_model, transpose_op, &op, > + (ConvolutionalParams *)conv_network->layers[layer].params, layer); > + break; > + case DEPTH_TO_SPACE: > + layer_add_res = add_depth_to_space_layer(tf_model, &op, > + (DepthToSpaceParams *)conv_network->layers[layer].params, layer); > + break; > + default: > + CLEANUP_ON_ERROR(tf_model); > + } > + > + if (layer_add_res != DNN_SUCCESS){ > + CLEANUP_ON_ERROR(tf_model); > + } > + } > + > + op_desc = TF_NewOperation(tf_model->graph, "Identity", "y"); > + input.oper = op; > + TF_AddInput(op_desc, input); > + TF_FinishOperation(op_desc, tf_model->status); > + if (TF_GetCode(tf_model->status) != TF_OK){ > + CLEANUP_ON_ERROR(tf_model); > + } > + > + ff_dnn_free_model_native(&native_model); > + > + return DNN_SUCCESS; > +} > + > +DNNModel *ff_dnn_load_model_tf(const char *model_filename) > +{ > + DNNModel *model = NULL; > + TFModel *tf_model = NULL; > + > + model = av_malloc(sizeof(DNNModel)); > + if (!model){ > + return NULL; > + } > + > + tf_model = av_mallocz(sizeof(TFModel)); > + if (!tf_model){ > + av_freep(&model); > + return NULL; > + } > + > + if (load_tf_model(tf_model, model_filename) != DNN_SUCCESS){ > + if (load_native_model(tf_model, model_filename) != DNN_SUCCESS){ > + av_freep(&tf_model); > + av_freep(&model); > + > + return NULL; > + } > + } > + > + model->model = (void *)tf_model; > + model->set_input_output = &set_input_output_tf; > + > + return model; > +} > + > + > + > +DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, uint32_t nb_output) > +{ > + TFModel *tf_model = (TFModel *)model->model; > + uint32_t nb = FFMIN(nb_output, tf_model->nb_output); > + if (nb == 0) > + return DNN_ERROR; > + > + av_assert0(tf_model->output_tensors); > + for (uint32_t i = 0; i < tf_model->nb_output; ++i) { > + if (tf_model->output_tensors[i]) { > + TF_DeleteTensor(tf_model->output_tensors[i]); > + tf_model->output_tensors[i] = NULL; > + } > + } > + > + TF_SessionRun(tf_model->session, NULL, > + &tf_model->input, &tf_model->input_tensor, 1, > + tf_model->outputs, tf_model->output_tensors, nb, > + NULL, 0, NULL, tf_model->status); > + > + if (TF_GetCode(tf_model->status) != TF_OK){ > + return DNN_ERROR; > + } > + > + for (uint32_t i = 0; i < nb; ++i) { > + outputs[i].height = TF_Dim(tf_model->output_tensors[i], 1); > + outputs[i].width = TF_Dim(tf_model->output_tensors[i], 2); > + outputs[i].channels = TF_Dim(tf_model->output_tensors[i], 3); > + outputs[i].data = TF_TensorData(tf_model->output_tensors[i]); > + } > + > + return DNN_SUCCESS; > +} > + > +void ff_dnn_free_model_tf(DNNModel **model) > +{ > + TFModel *tf_model; > + > + if (*model){ > + tf_model = (TFModel *)(*model)->model; > + if (tf_model->graph){ > + TF_DeleteGraph(tf_model->graph); > + } > + if (tf_model->session){ > + TF_CloseSession(tf_model->session, tf_model->status); > + TF_DeleteSession(tf_model->session, tf_model->status); > + } > + if (tf_model->status){ > + TF_DeleteStatus(tf_model->status); > + } > + if (tf_model->input_tensor){ > + TF_DeleteTensor(tf_model->input_tensor); > + } > + if (tf_model->output_tensors) { > + for (uint32_t i = 0; i < tf_model->nb_output; ++i) { > + if (tf_model->output_tensors[i]) { > + TF_DeleteTensor(tf_model->output_tensors[i]); > + tf_model->output_tensors[i] = NULL; > + } > + } > + } > + av_freep(&tf_model->outputs); > + av_freep(&tf_model->output_tensors); > + av_freep(&tf_model); > + av_freep(model); > + } > +} > diff --git a/libavfilter/dnn/dnn_backend_tf.h b/libavfilter/dnn/dnn_backend_tf.h > new file mode 100644 > index 0000000..bb1c85f > --- /dev/null > +++ b/libavfilter/dnn/dnn_backend_tf.h > @@ -0,0 +1,38 @@ > +/* > + * Copyright (c) 2018 Sergey Lavrushkin > + * > + * This file is part of FFmpeg. > + * > + * FFmpeg is free software; you can redistribute it and/or > + * modify it under the terms of the GNU Lesser General Public > + * License as published by the Free Software Foundation; either > + * version 2.1 of the License, or (at your option) any later version. > + * > + * FFmpeg is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + * Lesser General Public License for more details. > + * > + * You should have received a copy of the GNU Lesser General Public > + * License along with FFmpeg; if not, write to the Free Software > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > + */ > + > +/** > + * @file > + * DNN inference functions interface for TensorFlow backend. > + */ > + > + > +#ifndef AVFILTER_DNN_BACKEND_TF_H > +#define AVFILTER_DNN_BACKEND_TF_H > + > +#include "../dnn_interface.h" > + > +DNNModel *ff_dnn_load_model_tf(const char *model_filename); > + > +DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, uint32_t nb_output); > + > +void ff_dnn_free_model_tf(DNNModel **model); > + > +#endif > diff --git a/libavfilter/dnn/dnn_interface.c b/libavfilter/dnn/dnn_interface.c > new file mode 100644 > index 0000000..62da55f > --- /dev/null > +++ b/libavfilter/dnn/dnn_interface.c > @@ -0,0 +1,63 @@ > +/* > + * Copyright (c) 2018 Sergey Lavrushkin > + * > + * This file is part of FFmpeg. > + * > + * FFmpeg is free software; you can redistribute it and/or > + * modify it under the terms of the GNU Lesser General Public > + * License as published by the Free Software Foundation; either > + * version 2.1 of the License, or (at your option) any later version. > + * > + * FFmpeg is distributed in the hope that it will be useful, > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + * Lesser General Public License for more details. > + * > + * You should have received a copy of the GNU Lesser General Public > + * License along with FFmpeg; if not, write to the Free Software > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > + */ > + > +/** > + * @file > + * Implements DNN module initialization with specified backend. > + */ > + > +#include "../dnn_interface.h" > +#include "dnn_backend_native.h" > +#include "dnn_backend_tf.h" > +#include "libavutil/mem.h" > + > +DNNModule *ff_get_dnn_module(DNNBackendType backend_type) > +{ > + DNNModule *dnn_module; > + > + dnn_module = av_malloc(sizeof(DNNModule)); > + if(!dnn_module){ > + return NULL; > + } > + > + switch(backend_type){ > + case DNN_NATIVE: > + dnn_module->load_model = &ff_dnn_load_model_native; > + dnn_module->execute_model = &ff_dnn_execute_model_native; > + dnn_module->free_model = &ff_dnn_free_model_native; > + break; > + case DNN_TF: > + #if (CONFIG_LIBTENSORFLOW == 1) > + dnn_module->load_model = &ff_dnn_load_model_tf; > + dnn_module->execute_model = &ff_dnn_execute_model_tf; > + dnn_module->free_model = &ff_dnn_free_model_tf; > + #else > + av_freep(&dnn_module); > + return NULL; > + #endif > + break; > + default: > + av_log(NULL, AV_LOG_ERROR, "Module backend_type is not native or tensorflow\n"); > + av_freep(&dnn_module); > + return NULL; > + } > + > + return dnn_module; > +} > diff --git a/libavfilter/dnn_backend_native.c b/libavfilter/dnn_backend_native.c > deleted file mode 100644 > index 82e900b..0000000 > --- a/libavfilter/dnn_backend_native.c > +++ /dev/null > @@ -1,389 +0,0 @@ > -/* > - * Copyright (c) 2018 Sergey Lavrushkin > - * > - * This file is part of FFmpeg. > - * > - * FFmpeg is free software; you can redistribute it and/or > - * modify it under the terms of the GNU Lesser General Public > - * License as published by the Free Software Foundation; either > - * version 2.1 of the License, or (at your option) any later version. > - * > - * FFmpeg is distributed in the hope that it will be useful, > - * but WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * Lesser General Public License for more details. > - * > - * You should have received a copy of the GNU Lesser General Public > - * License along with FFmpeg; if not, write to the Free Software > - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > - */ > - > -/** > - * @file > - * DNN native backend implementation. > - */ > - > -#include "dnn_backend_native.h" > -#include "libavutil/avassert.h" > - > -static DNNReturnType set_input_output_native(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output) > -{ > - ConvolutionalNetwork *network = (ConvolutionalNetwork *)model; > - InputParams *input_params; > - ConvolutionalParams *conv_params; > - DepthToSpaceParams *depth_to_space_params; > - int cur_width, cur_height, cur_channels; > - int32_t layer; > - > - if (network->layers_num <= 0 || network->layers[0].type != INPUT){ > - return DNN_ERROR; > - } > - else{ > - input_params = (InputParams *)network->layers[0].params; > - input_params->width = cur_width = input->width; > - input_params->height = cur_height = input->height; > - input_params->channels = cur_channels = input->channels; > - if (input->data){ > - av_freep(&input->data); > - } > - av_assert0(input->dt == DNN_FLOAT); > - network->layers[0].output = input->data = av_malloc(cur_height * cur_width * cur_channels * sizeof(float)); > - if (!network->layers[0].output){ > - return DNN_ERROR; > - } > - } > - > - for (layer = 1; layer < network->layers_num; ++layer){ > - switch (network->layers[layer].type){ > - case CONV: > - conv_params = (ConvolutionalParams *)network->layers[layer].params; > - if (conv_params->input_num != cur_channels){ > - return DNN_ERROR; > - } > - cur_channels = conv_params->output_num; > - > - if (conv_params->padding_method == VALID) { > - int pad_size = (conv_params->kernel_size - 1) * conv_params->dilation; > - cur_height -= pad_size; > - cur_width -= pad_size; > - } > - break; > - case DEPTH_TO_SPACE: > - depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params; > - if (cur_channels % (depth_to_space_params->block_size * depth_to_space_params->block_size) != 0){ > - return DNN_ERROR; > - } > - cur_channels = cur_channels / (depth_to_space_params->block_size * depth_to_space_params->block_size); > - cur_height *= depth_to_space_params->block_size; > - cur_width *= depth_to_space_params->block_size; > - break; > - default: > - return DNN_ERROR; > - } > - if (network->layers[layer].output){ > - av_freep(&network->layers[layer].output); > - } > - > - if (cur_height <= 0 || cur_width <= 0) > - return DNN_ERROR; > - > - network->layers[layer].output = av_malloc(cur_height * cur_width * cur_channels * sizeof(float)); > - if (!network->layers[layer].output){ > - return DNN_ERROR; > - } > - } > - > - return DNN_SUCCESS; > -} > - > -// Loads model and its parameters that are stored in a binary file with following structure: > -// layers_num,layer_type,layer_parameterss,layer_type,layer_parameters... > -// For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases > -// For DEPTH_TO_SPACE layer: block_size > -DNNModel *ff_dnn_load_model_native(const char *model_filename) > -{ > - DNNModel *model = NULL; > - ConvolutionalNetwork *network = NULL; > - AVIOContext *model_file_context; > - int file_size, dnn_size, kernel_size, i; > - int32_t layer; > - DNNLayerType layer_type; > - ConvolutionalParams *conv_params; > - DepthToSpaceParams *depth_to_space_params; > - > - model = av_malloc(sizeof(DNNModel)); > - if (!model){ > - return NULL; > - } > - > - if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){ > - av_freep(&model); > - return NULL; > - } > - file_size = avio_size(model_file_context); > - > - network = av_malloc(sizeof(ConvolutionalNetwork)); > - if (!network){ > - avio_closep(&model_file_context); > - av_freep(&model); > - return NULL; > - } > - model->model = (void *)network; > - > - network->layers_num = 1 + (int32_t)avio_rl32(model_file_context); > - dnn_size = 4; > - > - network->layers = av_malloc(network->layers_num * sizeof(Layer)); > - if (!network->layers){ > - av_freep(&network); > - avio_closep(&model_file_context); > - av_freep(&model); > - return NULL; > - } > - > - for (layer = 0; layer < network->layers_num; ++layer){ > - network->layers[layer].output = NULL; > - network->layers[layer].params = NULL; > - } > - network->layers[0].type = INPUT; > - network->layers[0].params = av_malloc(sizeof(InputParams)); > - if (!network->layers[0].params){ > - avio_closep(&model_file_context); > - ff_dnn_free_model_native(&model); > - return NULL; > - } > - > - for (layer = 1; layer < network->layers_num; ++layer){ > - layer_type = (int32_t)avio_rl32(model_file_context); > - dnn_size += 4; > - switch (layer_type){ > - case CONV: > - conv_params = av_malloc(sizeof(ConvolutionalParams)); > - if (!conv_params){ > - avio_closep(&model_file_context); > - ff_dnn_free_model_native(&model); > - return NULL; > - } > - conv_params->dilation = (int32_t)avio_rl32(model_file_context); > - conv_params->padding_method = (int32_t)avio_rl32(model_file_context); > - conv_params->activation = (int32_t)avio_rl32(model_file_context); > - conv_params->input_num = (int32_t)avio_rl32(model_file_context); > - conv_params->output_num = (int32_t)avio_rl32(model_file_context); > - conv_params->kernel_size = (int32_t)avio_rl32(model_file_context); > - kernel_size = conv_params->input_num * conv_params->output_num * > - conv_params->kernel_size * conv_params->kernel_size; > - dnn_size += 24 + (kernel_size + conv_params->output_num << 2); > - if (dnn_size > file_size || conv_params->input_num <= 0 || > - conv_params->output_num <= 0 || conv_params->kernel_size <= 0){ > - avio_closep(&model_file_context); > - ff_dnn_free_model_native(&model); > - return NULL; > - } > - conv_params->kernel = av_malloc(kernel_size * sizeof(float)); > - conv_params->biases = av_malloc(conv_params->output_num * sizeof(float)); > - if (!conv_params->kernel || !conv_params->biases){ > - avio_closep(&model_file_context); > - ff_dnn_free_model_native(&model); > - return NULL; > - } > - for (i = 0; i < kernel_size; ++i){ > - conv_params->kernel[i] = av_int2float(avio_rl32(model_file_context)); > - } > - for (i = 0; i < conv_params->output_num; ++i){ > - conv_params->biases[i] = av_int2float(avio_rl32(model_file_context)); > - } > - network->layers[layer].type = CONV; > - network->layers[layer].params = conv_params; > - break; > - case DEPTH_TO_SPACE: > - depth_to_space_params = av_malloc(sizeof(DepthToSpaceParams)); > - if (!depth_to_space_params){ > - avio_closep(&model_file_context); > - ff_dnn_free_model_native(&model); > - return NULL; > - } > - depth_to_space_params->block_size = (int32_t)avio_rl32(model_file_context); > - dnn_size += 4; > - network->layers[layer].type = DEPTH_TO_SPACE; > - network->layers[layer].params = depth_to_space_params; > - break; > - default: > - avio_closep(&model_file_context); > - ff_dnn_free_model_native(&model); > - return NULL; > - } > - } > - > - avio_closep(&model_file_context); > - > - if (dnn_size != file_size){ > - ff_dnn_free_model_native(&model); > - return NULL; > - } > - > - model->set_input_output = &set_input_output_native; > - > - return model; > -} > - > -#define CLAMP_TO_EDGE(x, w) ((x) < 0 ? 0 : ((x) >= (w) ? (w - 1) : (x))) > - > -static void convolve(const float *input, float *output, const ConvolutionalParams *conv_params, int width, int height) > -{ > - int radius = conv_params->kernel_size >> 1; > - int src_linesize = width * conv_params->input_num; > - int filter_linesize = conv_params->kernel_size * conv_params->input_num; > - int filter_size = conv_params->kernel_size * filter_linesize; > - int pad_size = (conv_params->padding_method == VALID) ? (conv_params->kernel_size - 1) / 2 * conv_params->dilation : 0; > - > - for (int y = pad_size; y < height - pad_size; ++y) { > - for (int x = pad_size; x < width - pad_size; ++x) { > - for (int n_filter = 0; n_filter < conv_params->output_num; ++n_filter) { > - output[n_filter] = conv_params->biases[n_filter]; > - > - for (int ch = 0; ch < conv_params->input_num; ++ch) { > - for (int kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y) { > - for (int kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x) { > - float input_pel; > - if (conv_params->padding_method == SAME_CLAMP_TO_EDGE) { > - int y_pos = CLAMP_TO_EDGE(y + (kernel_y - radius) * conv_params->dilation, height); > - int x_pos = CLAMP_TO_EDGE(x + (kernel_x - radius) * conv_params->dilation, width); > - input_pel = input[y_pos * src_linesize + x_pos * conv_params->input_num + ch]; > - } else { > - int y_pos = y + (kernel_y - radius) * conv_params->dilation; > - int x_pos = x + (kernel_x - radius) * conv_params->dilation; > - input_pel = (x_pos < 0 || x_pos >= width || y_pos < 0 || y_pos >= height) ? 0.0 : > - input[y_pos * src_linesize + x_pos * conv_params->input_num + ch]; > - } > - > - > - output[n_filter] += input_pel * conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize + > - kernel_x * conv_params->input_num + ch]; > - } > - } > - } > - switch (conv_params->activation){ > - case RELU: > - output[n_filter] = FFMAX(output[n_filter], 0.0); > - break; > - case TANH: > - output[n_filter] = 2.0f / (1.0f + exp(-2.0f * output[n_filter])) - 1.0f; > - break; > - case SIGMOID: > - output[n_filter] = 1.0f / (1.0f + exp(-output[n_filter])); > - break; > - case NONE: > - break; > - case LEAKY_RELU: > - output[n_filter] = FFMAX(output[n_filter], 0.0) + 0.2 * FFMIN(output[n_filter], 0.0); > - } > - } > - output += conv_params->output_num; > - } > - } > -} > - > -static void depth_to_space(const float *input, float *output, int block_size, int width, int height, int channels) > -{ > - int y, x, by, bx, ch; > - int new_channels = channels / (block_size * block_size); > - int output_linesize = width * channels; > - int by_linesize = output_linesize / block_size; > - int x_linesize = new_channels * block_size; > - > - for (y = 0; y < height; ++y){ > - for (x = 0; x < width; ++x){ > - for (by = 0; by < block_size; ++by){ > - for (bx = 0; bx < block_size; ++bx){ > - for (ch = 0; ch < new_channels; ++ch){ > - output[by * by_linesize + x * x_linesize + bx * new_channels + ch] = input[ch]; > - } > - input += new_channels; > - } > - } > - } > - output += output_linesize; > - } > -} > - > -DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output) > -{ > - ConvolutionalNetwork *network = (ConvolutionalNetwork *)model->model; > - int cur_width, cur_height, cur_channels; > - int32_t layer; > - InputParams *input_params; > - ConvolutionalParams *conv_params; > - DepthToSpaceParams *depth_to_space_params; > - > - if (network->layers_num <= 0 || network->layers[0].type != INPUT || !network->layers[0].output){ > - return DNN_ERROR; > - } > - else{ > - input_params = (InputParams *)network->layers[0].params; > - cur_width = input_params->width; > - cur_height = input_params->height; > - cur_channels = input_params->channels; > - } > - > - for (layer = 1; layer < network->layers_num; ++layer){ > - if (!network->layers[layer].output){ > - return DNN_ERROR; > - } > - switch (network->layers[layer].type){ > - case CONV: > - conv_params = (ConvolutionalParams *)network->layers[layer].params; > - convolve(network->layers[layer - 1].output, network->layers[layer].output, conv_params, cur_width, cur_height); > - cur_channels = conv_params->output_num; > - if (conv_params->padding_method == VALID) { > - int pad_size = (conv_params->kernel_size - 1) * conv_params->dilation; > - cur_height -= pad_size; > - cur_width -= pad_size; > - } > - break; > - case DEPTH_TO_SPACE: > - depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params; > - depth_to_space(network->layers[layer - 1].output, network->layers[layer].output, > - depth_to_space_params->block_size, cur_width, cur_height, cur_channels); > - cur_height *= depth_to_space_params->block_size; > - cur_width *= depth_to_space_params->block_size; > - cur_channels /= depth_to_space_params->block_size * depth_to_space_params->block_size; > - break; > - case INPUT: > - return DNN_ERROR; > - } > - } > - > - // native mode does not support multiple outputs yet > - if (nb_output > 1) > - return DNN_ERROR; > - outputs[0].data = network->layers[network->layers_num - 1].output; > - outputs[0].height = cur_height; > - outputs[0].width = cur_width; > - outputs[0].channels = cur_channels; > - > - return DNN_SUCCESS; > -} > - > -void ff_dnn_free_model_native(DNNModel **model) > -{ > - ConvolutionalNetwork *network; > - ConvolutionalParams *conv_params; > - int32_t layer; > - > - if (*model) > - { > - network = (ConvolutionalNetwork *)(*model)->model; > - for (layer = 0; layer < network->layers_num; ++layer){ > - av_freep(&network->layers[layer].output); > - if (network->layers[layer].type == CONV){ > - conv_params = (ConvolutionalParams *)network->layers[layer].params; > - av_freep(&conv_params->kernel); > - av_freep(&conv_params->biases); > - } > - av_freep(&network->layers[layer].params); > - } > - av_freep(&network->layers); > - av_freep(&network); > - av_freep(model); > - } > -} > diff --git a/libavfilter/dnn_backend_native.h b/libavfilter/dnn_backend_native.h > deleted file mode 100644 > index 5917955..0000000 > --- a/libavfilter/dnn_backend_native.h > +++ /dev/null > @@ -1,74 +0,0 @@ > -/* > - * Copyright (c) 2018 Sergey Lavrushkin > - * > - * This file is part of FFmpeg. > - * > - * FFmpeg is free software; you can redistribute it and/or > - * modify it under the terms of the GNU Lesser General Public > - * License as published by the Free Software Foundation; either > - * version 2.1 of the License, or (at your option) any later version. > - * > - * FFmpeg is distributed in the hope that it will be useful, > - * but WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * Lesser General Public License for more details. > - * > - * You should have received a copy of the GNU Lesser General Public > - * License along with FFmpeg; if not, write to the Free Software > - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > - */ > - > -/** > - * @file > - * DNN inference functions interface for native backend. > - */ > - > - > -#ifndef AVFILTER_DNN_BACKEND_NATIVE_H > -#define AVFILTER_DNN_BACKEND_NATIVE_H > - > -#include "dnn_interface.h" > -#include "libavformat/avio.h" > - > -typedef enum {INPUT, CONV, DEPTH_TO_SPACE} DNNLayerType; > - > -typedef enum {RELU, TANH, SIGMOID, NONE, LEAKY_RELU} DNNActivationFunc; > - > -typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} DNNConvPaddingParam; > - > -typedef struct Layer{ > - DNNLayerType type; > - float *output; > - void *params; > -} Layer; > - > -typedef struct ConvolutionalParams{ > - int32_t input_num, output_num, kernel_size; > - DNNActivationFunc activation; > - DNNConvPaddingParam padding_method; > - int32_t dilation; > - float *kernel; > - float *biases; > -} ConvolutionalParams; > - > -typedef struct InputParams{ > - int height, width, channels; > -} InputParams; > - > -typedef struct DepthToSpaceParams{ > - int block_size; > -} DepthToSpaceParams; > - > -// Represents simple feed-forward convolutional network. > -typedef struct ConvolutionalNetwork{ > - Layer *layers; > - int32_t layers_num; > -} ConvolutionalNetwork; > - > -DNNModel *ff_dnn_load_model_native(const char *model_filename); > - > -DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output); > - > -void ff_dnn_free_model_native(DNNModel **model); > - > -#endif > diff --git a/libavfilter/dnn_backend_tf.c b/libavfilter/dnn_backend_tf.c > deleted file mode 100644 > index ba959ae..0000000 > --- a/libavfilter/dnn_backend_tf.c > +++ /dev/null > @@ -1,603 +0,0 @@ > -/* > - * Copyright (c) 2018 Sergey Lavrushkin > - * > - * This file is part of FFmpeg. > - * > - * FFmpeg is free software; you can redistribute it and/or > - * modify it under the terms of the GNU Lesser General Public > - * License as published by the Free Software Foundation; either > - * version 2.1 of the License, or (at your option) any later version. > - * > - * FFmpeg is distributed in the hope that it will be useful, > - * but WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * Lesser General Public License for more details. > - * > - * You should have received a copy of the GNU Lesser General Public > - * License along with FFmpeg; if not, write to the Free Software > - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > - */ > - > -/** > - * @file > - * DNN tensorflow backend implementation. > - */ > - > -#include "dnn_backend_tf.h" > -#include "dnn_backend_native.h" > -#include "libavformat/avio.h" > -#include "libavutil/avassert.h" > - > -#include <tensorflow/c/c_api.h> > - > -typedef struct TFModel{ > - TF_Graph *graph; > - TF_Session *session; > - TF_Status *status; > - TF_Output input; > - TF_Tensor *input_tensor; > - TF_Output *outputs; > - TF_Tensor **output_tensors; > - uint32_t nb_output; > -} TFModel; > - > -static void free_buffer(void *data, size_t length) > -{ > - av_freep(&data); > -} > - > -static TF_Buffer *read_graph(const char *model_filename) > -{ > - TF_Buffer *graph_buf; > - unsigned char *graph_data = NULL; > - AVIOContext *model_file_context; > - long size, bytes_read; > - > - if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){ > - return NULL; > - } > - > - size = avio_size(model_file_context); > - > - graph_data = av_malloc(size); > - if (!graph_data){ > - avio_closep(&model_file_context); > - return NULL; > - } > - bytes_read = avio_read(model_file_context, graph_data, size); > - avio_closep(&model_file_context); > - if (bytes_read != size){ > - av_freep(&graph_data); > - return NULL; > - } > - > - graph_buf = TF_NewBuffer(); > - graph_buf->data = (void *)graph_data; > - graph_buf->length = size; > - graph_buf->data_deallocator = free_buffer; > - > - return graph_buf; > -} > - > -static TF_Tensor *allocate_input_tensor(const DNNInputData *input) > -{ > - TF_DataType dt; > - size_t size; > - int64_t input_dims[] = {1, input->height, input->width, input->channels}; > - switch (input->dt) { > - case DNN_FLOAT: > - dt = TF_FLOAT; > - size = sizeof(float); > - break; > - case DNN_UINT8: > - dt = TF_UINT8; > - size = sizeof(char); > - break; > - default: > - av_assert0(!"should not reach here"); > - } > - > - return TF_AllocateTensor(dt, input_dims, 4, > - input_dims[1] * input_dims[2] * input_dims[3] * size); > -} > - > -static DNNReturnType set_input_output_tf(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output) > -{ > - TFModel *tf_model = (TFModel *)model; > - TF_SessionOptions *sess_opts; > - const TF_Operation *init_op = TF_GraphOperationByName(tf_model->graph, "init"); > - > - // Input operation > - tf_model->input.oper = TF_GraphOperationByName(tf_model->graph, input_name); > - if (!tf_model->input.oper){ > - return DNN_ERROR; > - } > - tf_model->input.index = 0; > - if (tf_model->input_tensor){ > - TF_DeleteTensor(tf_model->input_tensor); > - } > - tf_model->input_tensor = allocate_input_tensor(input); > - if (!tf_model->input_tensor){ > - return DNN_ERROR; > - } > - input->data = (float *)TF_TensorData(tf_model->input_tensor); > - > - // Output operation > - if (nb_output == 0) > - return DNN_ERROR; > - > - av_freep(&tf_model->outputs); > - tf_model->outputs = av_malloc_array(nb_output, sizeof(*tf_model->outputs)); > - if (!tf_model->outputs) > - return DNN_ERROR; > - for (int i = 0; i < nb_output; ++i) { > - tf_model->outputs[i].oper = TF_GraphOperationByName(tf_model->graph, output_names[i]); > - if (!tf_model->outputs[i].oper){ > - av_freep(&tf_model->outputs); > - return DNN_ERROR; > - } > - tf_model->outputs[i].index = 0; > - } > - > - if (tf_model->output_tensors) { > - for (uint32_t i = 0; i < tf_model->nb_output; ++i) { > - if (tf_model->output_tensors[i]) { > - TF_DeleteTensor(tf_model->output_tensors[i]); > - tf_model->output_tensors[i] = NULL; > - } > - } > - } > - av_freep(&tf_model->output_tensors); > - tf_model->output_tensors = av_mallocz_array(nb_output, sizeof(*tf_model->output_tensors)); > - if (!tf_model->output_tensors) { > - av_freep(&tf_model->outputs); > - return DNN_ERROR; > - } > - > - tf_model->nb_output = nb_output; > - > - if (tf_model->session){ > - TF_CloseSession(tf_model->session, tf_model->status); > - TF_DeleteSession(tf_model->session, tf_model->status); > - } > - > - sess_opts = TF_NewSessionOptions(); > - tf_model->session = TF_NewSession(tf_model->graph, sess_opts, tf_model->status); > - TF_DeleteSessionOptions(sess_opts); > - if (TF_GetCode(tf_model->status) != TF_OK) > - { > - return DNN_ERROR; > - } > - > - // Run initialization operation with name "init" if it is present in graph > - if (init_op){ > - TF_SessionRun(tf_model->session, NULL, > - NULL, NULL, 0, > - NULL, NULL, 0, > - &init_op, 1, NULL, tf_model->status); > - if (TF_GetCode(tf_model->status) != TF_OK) > - { > - return DNN_ERROR; > - } > - } > - > - return DNN_SUCCESS; > -} > - > -static DNNReturnType load_tf_model(TFModel *tf_model, const char *model_filename) > -{ > - TF_Buffer *graph_def; > - TF_ImportGraphDefOptions *graph_opts; > - > - graph_def = read_graph(model_filename); > - if (!graph_def){ > - return DNN_ERROR; > - } > - tf_model->graph = TF_NewGraph(); > - tf_model->status = TF_NewStatus(); > - graph_opts = TF_NewImportGraphDefOptions(); > - TF_GraphImportGraphDef(tf_model->graph, graph_def, graph_opts, tf_model->status); > - TF_DeleteImportGraphDefOptions(graph_opts); > - TF_DeleteBuffer(graph_def); > - if (TF_GetCode(tf_model->status) != TF_OK){ > - TF_DeleteGraph(tf_model->graph); > - TF_DeleteStatus(tf_model->status); > - return DNN_ERROR; > - } > - > - return DNN_SUCCESS; > -} > - > -#define NAME_BUFFER_SIZE 256 > - > -static DNNReturnType add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op, > - ConvolutionalParams* params, const int layer) > -{ > - TF_Operation *op; > - TF_OperationDescription *op_desc; > - TF_Output input; > - int64_t strides[] = {1, 1, 1, 1}; > - TF_Tensor *tensor; > - int64_t dims[4]; > - int dims_len; > - char name_buffer[NAME_BUFFER_SIZE]; > - int32_t size; > - > - size = params->input_num * params->output_num * params->kernel_size * params->kernel_size; > - input.index = 0; > - > - snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_kernel%d", layer); > - op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer); > - TF_SetAttrType(op_desc, "dtype", TF_FLOAT); > - dims[0] = params->output_num; > - dims[1] = params->kernel_size; > - dims[2] = params->kernel_size; > - dims[3] = params->input_num; > - dims_len = 4; > - tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, size * sizeof(float)); > - memcpy(TF_TensorData(tensor), params->kernel, size * sizeof(float)); > - TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); > - if (TF_GetCode(tf_model->status) != TF_OK){ > - return DNN_ERROR; > - } > - op = TF_FinishOperation(op_desc, tf_model->status); > - if (TF_GetCode(tf_model->status) != TF_OK){ > - return DNN_ERROR; > - } > - > - snprintf(name_buffer, NAME_BUFFER_SIZE, "transpose%d", layer); > - op_desc = TF_NewOperation(tf_model->graph, "Transpose", name_buffer); > - input.oper = op; > - TF_AddInput(op_desc, input); > - input.oper = transpose_op; > - TF_AddInput(op_desc, input); > - TF_SetAttrType(op_desc, "T", TF_FLOAT); > - TF_SetAttrType(op_desc, "Tperm", TF_INT32); > - op = TF_FinishOperation(op_desc, tf_model->status); > - if (TF_GetCode(tf_model->status) != TF_OK){ > - return DNN_ERROR; > - } > - > - snprintf(name_buffer, NAME_BUFFER_SIZE, "conv2d%d", layer); > - op_desc = TF_NewOperation(tf_model->graph, "Conv2D", name_buffer); > - input.oper = *cur_op; > - TF_AddInput(op_desc, input); > - input.oper = op; > - TF_AddInput(op_desc, input); > - TF_SetAttrType(op_desc, "T", TF_FLOAT); > - TF_SetAttrIntList(op_desc, "strides", strides, 4); > - TF_SetAttrString(op_desc, "padding", "VALID", 5); > - *cur_op = TF_FinishOperation(op_desc, tf_model->status); > - if (TF_GetCode(tf_model->status) != TF_OK){ > - return DNN_ERROR; > - } > - > - snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_biases%d", layer); > - op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer); > - TF_SetAttrType(op_desc, "dtype", TF_FLOAT); > - dims[0] = params->output_num; > - dims_len = 1; > - tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, params->output_num * sizeof(float)); > - memcpy(TF_TensorData(tensor), params->biases, params->output_num * sizeof(float)); > - TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); > - if (TF_GetCode(tf_model->status) != TF_OK){ > - return DNN_ERROR; > - } > - op = TF_FinishOperation(op_desc, tf_model->status); > - if (TF_GetCode(tf_model->status) != TF_OK){ > - return DNN_ERROR; > - } > - > - snprintf(name_buffer, NAME_BUFFER_SIZE, "bias_add%d", layer); > - op_desc = TF_NewOperation(tf_model->graph, "BiasAdd", name_buffer); > - input.oper = *cur_op; > - TF_AddInput(op_desc, input); > - input.oper = op; > - TF_AddInput(op_desc, input); > - TF_SetAttrType(op_desc, "T", TF_FLOAT); > - *cur_op = TF_FinishOperation(op_desc, tf_model->status); > - if (TF_GetCode(tf_model->status) != TF_OK){ > - return DNN_ERROR; > - } > - > - snprintf(name_buffer, NAME_BUFFER_SIZE, "activation%d", layer); > - switch (params->activation){ > - case RELU: > - op_desc = TF_NewOperation(tf_model->graph, "Relu", name_buffer); > - break; > - case TANH: > - op_desc = TF_NewOperation(tf_model->graph, "Tanh", name_buffer); > - break; > - case SIGMOID: > - op_desc = TF_NewOperation(tf_model->graph, "Sigmoid", name_buffer); > - break; > - default: > - return DNN_ERROR; > - } > - input.oper = *cur_op; > - TF_AddInput(op_desc, input); > - TF_SetAttrType(op_desc, "T", TF_FLOAT); > - *cur_op = TF_FinishOperation(op_desc, tf_model->status); > - if (TF_GetCode(tf_model->status) != TF_OK){ > - return DNN_ERROR; > - } > - > - return DNN_SUCCESS; > -} > - > -static DNNReturnType add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op, > - DepthToSpaceParams *params, const int layer) > -{ > - TF_OperationDescription *op_desc; > - TF_Output input; > - char name_buffer[NAME_BUFFER_SIZE]; > - > - snprintf(name_buffer, NAME_BUFFER_SIZE, "depth_to_space%d", layer); > - op_desc = TF_NewOperation(tf_model->graph, "DepthToSpace", name_buffer); > - input.oper = *cur_op; > - input.index = 0; > - TF_AddInput(op_desc, input); > - TF_SetAttrType(op_desc, "T", TF_FLOAT); > - TF_SetAttrInt(op_desc, "block_size", params->block_size); > - *cur_op = TF_FinishOperation(op_desc, tf_model->status); > - if (TF_GetCode(tf_model->status) != TF_OK){ > - return DNN_ERROR; > - } > - > - return DNN_SUCCESS; > -} > - > -static int calculate_pad(const ConvolutionalNetwork *conv_network) > -{ > - ConvolutionalParams *params; > - int32_t layer; > - int pad = 0; > - > - for (layer = 0; layer < conv_network->layers_num; ++layer){ > - if (conv_network->layers[layer].type == CONV){ > - params = (ConvolutionalParams *)conv_network->layers[layer].params; > - pad += params->kernel_size >> 1; > - } > - } > - > - return pad; > -} > - > -static DNNReturnType add_pad_op(TFModel *tf_model, TF_Operation **cur_op, const int32_t pad) > -{ > - TF_Operation *op; > - TF_Tensor *tensor; > - TF_OperationDescription *op_desc; > - TF_Output input; > - int32_t *pads; > - int64_t pads_shape[] = {4, 2}; > - > - input.index = 0; > - > - op_desc = TF_NewOperation(tf_model->graph, "Const", "pads"); > - TF_SetAttrType(op_desc, "dtype", TF_INT32); > - tensor = TF_AllocateTensor(TF_INT32, pads_shape, 2, 4 * 2 * sizeof(int32_t)); > - pads = (int32_t *)TF_TensorData(tensor); > - pads[0] = 0; pads[1] = 0; > - pads[2] = pad; pads[3] = pad; > - pads[4] = pad; pads[5] = pad; > - pads[6] = 0; pads[7] = 0; > - TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); > - if (TF_GetCode(tf_model->status) != TF_OK){ > - return DNN_ERROR; > - } > - op = TF_FinishOperation(op_desc, tf_model->status); > - if (TF_GetCode(tf_model->status) != TF_OK){ > - return DNN_ERROR; > - } > - > - op_desc = TF_NewOperation(tf_model->graph, "MirrorPad", "mirror_pad"); > - input.oper = *cur_op; > - TF_AddInput(op_desc, input); > - input.oper = op; > - TF_AddInput(op_desc, input); > - TF_SetAttrType(op_desc, "T", TF_FLOAT); > - TF_SetAttrType(op_desc, "Tpaddings", TF_INT32); > - TF_SetAttrString(op_desc, "mode", "SYMMETRIC", 9); > - *cur_op = TF_FinishOperation(op_desc, tf_model->status); > - if (TF_GetCode(tf_model->status) != TF_OK){ > - return DNN_ERROR; > - } > - > - return DNN_SUCCESS; > -} > - > -static DNNReturnType load_native_model(TFModel *tf_model, const char *model_filename) > -{ > - int32_t layer; > - TF_OperationDescription *op_desc; > - TF_Operation *op; > - TF_Operation *transpose_op; > - TF_Tensor *tensor; > - TF_Output input; > - int32_t *transpose_perm; > - int64_t transpose_perm_shape[] = {4}; > - int64_t input_shape[] = {1, -1, -1, -1}; > - int32_t pad; > - DNNReturnType layer_add_res; > - DNNModel *native_model = NULL; > - ConvolutionalNetwork *conv_network; > - > - native_model = ff_dnn_load_model_native(model_filename); > - if (!native_model){ > - return DNN_ERROR; > - } > - > - conv_network = (ConvolutionalNetwork *)native_model->model; > - pad = calculate_pad(conv_network); > - tf_model->graph = TF_NewGraph(); > - tf_model->status = TF_NewStatus(); > - > -#define CLEANUP_ON_ERROR(tf_model) \ > - { \ > - TF_DeleteGraph(tf_model->graph); \ > - TF_DeleteStatus(tf_model->status); \ > - return DNN_ERROR; \ > - } > - > - op_desc = TF_NewOperation(tf_model->graph, "Placeholder", "x"); > - TF_SetAttrType(op_desc, "dtype", TF_FLOAT); > - TF_SetAttrShape(op_desc, "shape", input_shape, 4); > - op = TF_FinishOperation(op_desc, tf_model->status); > - if (TF_GetCode(tf_model->status) != TF_OK){ > - CLEANUP_ON_ERROR(tf_model); > - } > - > - if (add_pad_op(tf_model, &op, pad) != DNN_SUCCESS){ > - CLEANUP_ON_ERROR(tf_model); > - } > - > - op_desc = TF_NewOperation(tf_model->graph, "Const", "transpose_perm"); > - TF_SetAttrType(op_desc, "dtype", TF_INT32); > - tensor = TF_AllocateTensor(TF_INT32, transpose_perm_shape, 1, 4 * sizeof(int32_t)); > - transpose_perm = (int32_t *)TF_TensorData(tensor); > - transpose_perm[0] = 1; > - transpose_perm[1] = 2; > - transpose_perm[2] = 3; > - transpose_perm[3] = 0; > - TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); > - if (TF_GetCode(tf_model->status) != TF_OK){ > - CLEANUP_ON_ERROR(tf_model); > - } > - transpose_op = TF_FinishOperation(op_desc, tf_model->status); > - > - for (layer = 0; layer < conv_network->layers_num; ++layer){ > - switch (conv_network->layers[layer].type){ > - case INPUT: > - layer_add_res = DNN_SUCCESS; > - break; > - case CONV: > - layer_add_res = add_conv_layer(tf_model, transpose_op, &op, > - (ConvolutionalParams *)conv_network->layers[layer].params, layer); > - break; > - case DEPTH_TO_SPACE: > - layer_add_res = add_depth_to_space_layer(tf_model, &op, > - (DepthToSpaceParams *)conv_network->layers[layer].params, layer); > - break; > - default: > - CLEANUP_ON_ERROR(tf_model); > - } > - > - if (layer_add_res != DNN_SUCCESS){ > - CLEANUP_ON_ERROR(tf_model); > - } > - } > - > - op_desc = TF_NewOperation(tf_model->graph, "Identity", "y"); > - input.oper = op; > - TF_AddInput(op_desc, input); > - TF_FinishOperation(op_desc, tf_model->status); > - if (TF_GetCode(tf_model->status) != TF_OK){ > - CLEANUP_ON_ERROR(tf_model); > - } > - > - ff_dnn_free_model_native(&native_model); > - > - return DNN_SUCCESS; > -} > - > -DNNModel *ff_dnn_load_model_tf(const char *model_filename) > -{ > - DNNModel *model = NULL; > - TFModel *tf_model = NULL; > - > - model = av_malloc(sizeof(DNNModel)); > - if (!model){ > - return NULL; > - } > - > - tf_model = av_mallocz(sizeof(TFModel)); > - if (!tf_model){ > - av_freep(&model); > - return NULL; > - } > - > - if (load_tf_model(tf_model, model_filename) != DNN_SUCCESS){ > - if (load_native_model(tf_model, model_filename) != DNN_SUCCESS){ > - av_freep(&tf_model); > - av_freep(&model); > - > - return NULL; > - } > - } > - > - model->model = (void *)tf_model; > - model->set_input_output = &set_input_output_tf; > - > - return model; > -} > - > - > - > -DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, uint32_t nb_output) > -{ > - TFModel *tf_model = (TFModel *)model->model; > - uint32_t nb = FFMIN(nb_output, tf_model->nb_output); > - if (nb == 0) > - return DNN_ERROR; > - > - av_assert0(tf_model->output_tensors); > - for (uint32_t i = 0; i < tf_model->nb_output; ++i) { > - if (tf_model->output_tensors[i]) { > - TF_DeleteTensor(tf_model->output_tensors[i]); > - tf_model->output_tensors[i] = NULL; > - } > - } > - > - TF_SessionRun(tf_model->session, NULL, > - &tf_model->input, &tf_model->input_tensor, 1, > - tf_model->outputs, tf_model->output_tensors, nb, > - NULL, 0, NULL, tf_model->status); > - > - if (TF_GetCode(tf_model->status) != TF_OK){ > - return DNN_ERROR; > - } > - > - for (uint32_t i = 0; i < nb; ++i) { > - outputs[i].height = TF_Dim(tf_model->output_tensors[i], 1); > - outputs[i].width = TF_Dim(tf_model->output_tensors[i], 2); > - outputs[i].channels = TF_Dim(tf_model->output_tensors[i], 3); > - outputs[i].data = TF_TensorData(tf_model->output_tensors[i]); > - } > - > - return DNN_SUCCESS; > -} > - > -void ff_dnn_free_model_tf(DNNModel **model) > -{ > - TFModel *tf_model; > - > - if (*model){ > - tf_model = (TFModel *)(*model)->model; > - if (tf_model->graph){ > - TF_DeleteGraph(tf_model->graph); > - } > - if (tf_model->session){ > - TF_CloseSession(tf_model->session, tf_model->status); > - TF_DeleteSession(tf_model->session, tf_model->status); > - } > - if (tf_model->status){ > - TF_DeleteStatus(tf_model->status); > - } > - if (tf_model->input_tensor){ > - TF_DeleteTensor(tf_model->input_tensor); > - } > - if (tf_model->output_tensors) { > - for (uint32_t i = 0; i < tf_model->nb_output; ++i) { > - if (tf_model->output_tensors[i]) { > - TF_DeleteTensor(tf_model->output_tensors[i]); > - tf_model->output_tensors[i] = NULL; > - } > - } > - } > - av_freep(&tf_model->outputs); > - av_freep(&tf_model->output_tensors); > - av_freep(&tf_model); > - av_freep(model); > - } > -} > diff --git a/libavfilter/dnn_backend_tf.h b/libavfilter/dnn_backend_tf.h > deleted file mode 100644 > index 07877b1..0000000 > --- a/libavfilter/dnn_backend_tf.h > +++ /dev/null > @@ -1,38 +0,0 @@ > -/* > - * Copyright (c) 2018 Sergey Lavrushkin > - * > - * This file is part of FFmpeg. > - * > - * FFmpeg is free software; you can redistribute it and/or > - * modify it under the terms of the GNU Lesser General Public > - * License as published by the Free Software Foundation; either > - * version 2.1 of the License, or (at your option) any later version. > - * > - * FFmpeg is distributed in the hope that it will be useful, > - * but WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * Lesser General Public License for more details. > - * > - * You should have received a copy of the GNU Lesser General Public > - * License along with FFmpeg; if not, write to the Free Software > - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > - */ > - > -/** > - * @file > - * DNN inference functions interface for TensorFlow backend. > - */ > - > - > -#ifndef AVFILTER_DNN_BACKEND_TF_H > -#define AVFILTER_DNN_BACKEND_TF_H > - > -#include "dnn_interface.h" > - > -DNNModel *ff_dnn_load_model_tf(const char *model_filename); > - > -DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, uint32_t nb_output); > - > -void ff_dnn_free_model_tf(DNNModel **model); > - > -#endif > diff --git a/libavfilter/dnn_interface.c b/libavfilter/dnn_interface.c > deleted file mode 100644 > index 86fc283..0000000 > --- a/libavfilter/dnn_interface.c > +++ /dev/null > @@ -1,63 +0,0 @@ > -/* > - * Copyright (c) 2018 Sergey Lavrushkin > - * > - * This file is part of FFmpeg. > - * > - * FFmpeg is free software; you can redistribute it and/or > - * modify it under the terms of the GNU Lesser General Public > - * License as published by the Free Software Foundation; either > - * version 2.1 of the License, or (at your option) any later version. > - * > - * FFmpeg is distributed in the hope that it will be useful, > - * but WITHOUT ANY WARRANTY; without even the implied warranty of > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - * Lesser General Public License for more details. > - * > - * You should have received a copy of the GNU Lesser General Public > - * License along with FFmpeg; if not, write to the Free Software > - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > - */ > - > -/** > - * @file > - * Implements DNN module initialization with specified backend. > - */ > - > -#include "dnn_interface.h" > -#include "dnn_backend_native.h" > -#include "dnn_backend_tf.h" > -#include "libavutil/mem.h" > - > -DNNModule *ff_get_dnn_module(DNNBackendType backend_type) > -{ > - DNNModule *dnn_module; > - > - dnn_module = av_malloc(sizeof(DNNModule)); > - if(!dnn_module){ > - return NULL; > - } > - > - switch(backend_type){ > - case DNN_NATIVE: > - dnn_module->load_model = &ff_dnn_load_model_native; > - dnn_module->execute_model = &ff_dnn_execute_model_native; > - dnn_module->free_model = &ff_dnn_free_model_native; > - break; > - case DNN_TF: > - #if (CONFIG_LIBTENSORFLOW == 1) > - dnn_module->load_model = &ff_dnn_load_model_tf; > - dnn_module->execute_model = &ff_dnn_execute_model_tf; > - dnn_module->free_model = &ff_dnn_free_model_tf; > - #else > - av_freep(&dnn_module); > - return NULL; > - #endif > - break; > - default: > - av_log(NULL, AV_LOG_ERROR, "Module backend_type is not native or tensorflow\n"); > - av_freep(&dnn_module); > - return NULL; > - } > - > - return dnn_module; > -} > -- > 2.7.4 > > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
Em sex, 26 de jul de 2019 às 13:02, Pedro Arthur <bygrandao@gmail.com> escreveu: > > Hi, > It fails fate source guard header tests, > The headers should be changed from AVFILTER_DNN_BACKEND_xxx to > AVFILTER_DNN_DNN_BACKEND_xxx. Changed locally and pushed. > Other than that it LGTM. > > Em ter, 16 de jul de 2019 às 02:58, Guo, Yejun <yejun.guo@intel.com> escreveu: > > > > it is expected that there will be more files to support native mode, > > so put all the dnn codes under libavfilter/dnn > > > > The main change of this patch is to move the file location, see below: > > modified: libavfilter/Makefile > > new file: libavfilter/dnn/Makefile > > renamed: libavfilter/dnn_backend_native.c -> libavfilter/dnn/dnn_backend_native.c > > renamed: libavfilter/dnn_backend_native.h -> libavfilter/dnn/dnn_backend_native.h > > renamed: libavfilter/dnn_backend_tf.c -> libavfilter/dnn/dnn_backend_tf.c > > renamed: libavfilter/dnn_backend_tf.h -> libavfilter/dnn/dnn_backend_tf.h > > renamed: libavfilter/dnn_interface.c -> libavfilter/dnn/dnn_interface.c > > > > Signed-off-by: Guo, Yejun <yejun.guo@intel.com> > > --- > > libavfilter/Makefile | 3 +- > > libavfilter/dnn/Makefile | 6 + > > libavfilter/dnn/dnn_backend_native.c | 389 ++++++++++++++++++++++ > > libavfilter/dnn/dnn_backend_native.h | 74 +++++ > > libavfilter/dnn/dnn_backend_tf.c | 603 +++++++++++++++++++++++++++++++++++ > > libavfilter/dnn/dnn_backend_tf.h | 38 +++ > > libavfilter/dnn/dnn_interface.c | 63 ++++ > > libavfilter/dnn_backend_native.c | 389 ---------------------- > > libavfilter/dnn_backend_native.h | 74 ----- > > libavfilter/dnn_backend_tf.c | 603 ----------------------------------- > > libavfilter/dnn_backend_tf.h | 38 --- > > libavfilter/dnn_interface.c | 63 ---- > > 12 files changed, 1174 insertions(+), 1169 deletions(-) > > create mode 100644 libavfilter/dnn/Makefile > > create mode 100644 libavfilter/dnn/dnn_backend_native.c > > create mode 100644 libavfilter/dnn/dnn_backend_native.h > > create mode 100644 libavfilter/dnn/dnn_backend_tf.c > > create mode 100644 libavfilter/dnn/dnn_backend_tf.h > > create mode 100644 libavfilter/dnn/dnn_interface.c > > delete mode 100644 libavfilter/dnn_backend_native.c > > delete mode 100644 libavfilter/dnn_backend_native.h > > delete mode 100644 libavfilter/dnn_backend_tf.c > > delete mode 100644 libavfilter/dnn_backend_tf.h > > delete mode 100644 libavfilter/dnn_interface.c > > > > diff --git a/libavfilter/Makefile b/libavfilter/Makefile > > index 455c809..450d781 100644 > > --- a/libavfilter/Makefile > > +++ b/libavfilter/Makefile > > @@ -26,9 +26,8 @@ OBJS-$(HAVE_THREADS) += pthread.o > > > > # subsystems > > OBJS-$(CONFIG_QSVVPP) += qsvvpp.o > > -DNN-OBJS-$(CONFIG_LIBTENSORFLOW) += dnn_backend_tf.o > > -OBJS-$(CONFIG_DNN) += dnn_interface.o dnn_backend_native.o $(DNN-OBJS-yes) > > OBJS-$(CONFIG_SCENE_SAD) += scene_sad.o > > +include $(SRC_PATH)/libavfilter/dnn/Makefile > > > > # audio filters > > OBJS-$(CONFIG_ABENCH_FILTER) += f_bench.o > > diff --git a/libavfilter/dnn/Makefile b/libavfilter/dnn/Makefile > > new file mode 100644 > > index 0000000..1d12ade > > --- /dev/null > > +++ b/libavfilter/dnn/Makefile > > @@ -0,0 +1,6 @@ > > +OBJS-$(CONFIG_DNN) += dnn/dnn_interface.o > > +OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native.o > > + > > +DNN-OBJS-$(CONFIG_LIBTENSORFLOW) += dnn/dnn_backend_tf.o > > + > > +OBJS-$(CONFIG_DNN) += $(DNN-OBJS-yes) > > diff --git a/libavfilter/dnn/dnn_backend_native.c b/libavfilter/dnn/dnn_backend_native.c > > new file mode 100644 > > index 0000000..82e900b > > --- /dev/null > > +++ b/libavfilter/dnn/dnn_backend_native.c > > @@ -0,0 +1,389 @@ > > +/* > > + * Copyright (c) 2018 Sergey Lavrushkin > > + * > > + * This file is part of FFmpeg. > > + * > > + * FFmpeg is free software; you can redistribute it and/or > > + * modify it under the terms of the GNU Lesser General Public > > + * License as published by the Free Software Foundation; either > > + * version 2.1 of the License, or (at your option) any later version. > > + * > > + * FFmpeg is distributed in the hope that it will be useful, > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > + * Lesser General Public License for more details. > > + * > > + * You should have received a copy of the GNU Lesser General Public > > + * License along with FFmpeg; if not, write to the Free Software > > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > > + */ > > + > > +/** > > + * @file > > + * DNN native backend implementation. > > + */ > > + > > +#include "dnn_backend_native.h" > > +#include "libavutil/avassert.h" > > + > > +static DNNReturnType set_input_output_native(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output) > > +{ > > + ConvolutionalNetwork *network = (ConvolutionalNetwork *)model; > > + InputParams *input_params; > > + ConvolutionalParams *conv_params; > > + DepthToSpaceParams *depth_to_space_params; > > + int cur_width, cur_height, cur_channels; > > + int32_t layer; > > + > > + if (network->layers_num <= 0 || network->layers[0].type != INPUT){ > > + return DNN_ERROR; > > + } > > + else{ > > + input_params = (InputParams *)network->layers[0].params; > > + input_params->width = cur_width = input->width; > > + input_params->height = cur_height = input->height; > > + input_params->channels = cur_channels = input->channels; > > + if (input->data){ > > + av_freep(&input->data); > > + } > > + av_assert0(input->dt == DNN_FLOAT); > > + network->layers[0].output = input->data = av_malloc(cur_height * cur_width * cur_channels * sizeof(float)); > > + if (!network->layers[0].output){ > > + return DNN_ERROR; > > + } > > + } > > + > > + for (layer = 1; layer < network->layers_num; ++layer){ > > + switch (network->layers[layer].type){ > > + case CONV: > > + conv_params = (ConvolutionalParams *)network->layers[layer].params; > > + if (conv_params->input_num != cur_channels){ > > + return DNN_ERROR; > > + } > > + cur_channels = conv_params->output_num; > > + > > + if (conv_params->padding_method == VALID) { > > + int pad_size = (conv_params->kernel_size - 1) * conv_params->dilation; > > + cur_height -= pad_size; > > + cur_width -= pad_size; > > + } > > + break; > > + case DEPTH_TO_SPACE: > > + depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params; > > + if (cur_channels % (depth_to_space_params->block_size * depth_to_space_params->block_size) != 0){ > > + return DNN_ERROR; > > + } > > + cur_channels = cur_channels / (depth_to_space_params->block_size * depth_to_space_params->block_size); > > + cur_height *= depth_to_space_params->block_size; > > + cur_width *= depth_to_space_params->block_size; > > + break; > > + default: > > + return DNN_ERROR; > > + } > > + if (network->layers[layer].output){ > > + av_freep(&network->layers[layer].output); > > + } > > + > > + if (cur_height <= 0 || cur_width <= 0) > > + return DNN_ERROR; > > + > > + network->layers[layer].output = av_malloc(cur_height * cur_width * cur_channels * sizeof(float)); > > + if (!network->layers[layer].output){ > > + return DNN_ERROR; > > + } > > + } > > + > > + return DNN_SUCCESS; > > +} > > + > > +// Loads model and its parameters that are stored in a binary file with following structure: > > +// layers_num,layer_type,layer_parameterss,layer_type,layer_parameters... > > +// For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases > > +// For DEPTH_TO_SPACE layer: block_size > > +DNNModel *ff_dnn_load_model_native(const char *model_filename) > > +{ > > + DNNModel *model = NULL; > > + ConvolutionalNetwork *network = NULL; > > + AVIOContext *model_file_context; > > + int file_size, dnn_size, kernel_size, i; > > + int32_t layer; > > + DNNLayerType layer_type; > > + ConvolutionalParams *conv_params; > > + DepthToSpaceParams *depth_to_space_params; > > + > > + model = av_malloc(sizeof(DNNModel)); > > + if (!model){ > > + return NULL; > > + } > > + > > + if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){ > > + av_freep(&model); > > + return NULL; > > + } > > + file_size = avio_size(model_file_context); > > + > > + network = av_malloc(sizeof(ConvolutionalNetwork)); > > + if (!network){ > > + avio_closep(&model_file_context); > > + av_freep(&model); > > + return NULL; > > + } > > + model->model = (void *)network; > > + > > + network->layers_num = 1 + (int32_t)avio_rl32(model_file_context); > > + dnn_size = 4; > > + > > + network->layers = av_malloc(network->layers_num * sizeof(Layer)); > > + if (!network->layers){ > > + av_freep(&network); > > + avio_closep(&model_file_context); > > + av_freep(&model); > > + return NULL; > > + } > > + > > + for (layer = 0; layer < network->layers_num; ++layer){ > > + network->layers[layer].output = NULL; > > + network->layers[layer].params = NULL; > > + } > > + network->layers[0].type = INPUT; > > + network->layers[0].params = av_malloc(sizeof(InputParams)); > > + if (!network->layers[0].params){ > > + avio_closep(&model_file_context); > > + ff_dnn_free_model_native(&model); > > + return NULL; > > + } > > + > > + for (layer = 1; layer < network->layers_num; ++layer){ > > + layer_type = (int32_t)avio_rl32(model_file_context); > > + dnn_size += 4; > > + switch (layer_type){ > > + case CONV: > > + conv_params = av_malloc(sizeof(ConvolutionalParams)); > > + if (!conv_params){ > > + avio_closep(&model_file_context); > > + ff_dnn_free_model_native(&model); > > + return NULL; > > + } > > + conv_params->dilation = (int32_t)avio_rl32(model_file_context); > > + conv_params->padding_method = (int32_t)avio_rl32(model_file_context); > > + conv_params->activation = (int32_t)avio_rl32(model_file_context); > > + conv_params->input_num = (int32_t)avio_rl32(model_file_context); > > + conv_params->output_num = (int32_t)avio_rl32(model_file_context); > > + conv_params->kernel_size = (int32_t)avio_rl32(model_file_context); > > + kernel_size = conv_params->input_num * conv_params->output_num * > > + conv_params->kernel_size * conv_params->kernel_size; > > + dnn_size += 24 + (kernel_size + conv_params->output_num << 2); > > + if (dnn_size > file_size || conv_params->input_num <= 0 || > > + conv_params->output_num <= 0 || conv_params->kernel_size <= 0){ > > + avio_closep(&model_file_context); > > + ff_dnn_free_model_native(&model); > > + return NULL; > > + } > > + conv_params->kernel = av_malloc(kernel_size * sizeof(float)); > > + conv_params->biases = av_malloc(conv_params->output_num * sizeof(float)); > > + if (!conv_params->kernel || !conv_params->biases){ > > + avio_closep(&model_file_context); > > + ff_dnn_free_model_native(&model); > > + return NULL; > > + } > > + for (i = 0; i < kernel_size; ++i){ > > + conv_params->kernel[i] = av_int2float(avio_rl32(model_file_context)); > > + } > > + for (i = 0; i < conv_params->output_num; ++i){ > > + conv_params->biases[i] = av_int2float(avio_rl32(model_file_context)); > > + } > > + network->layers[layer].type = CONV; > > + network->layers[layer].params = conv_params; > > + break; > > + case DEPTH_TO_SPACE: > > + depth_to_space_params = av_malloc(sizeof(DepthToSpaceParams)); > > + if (!depth_to_space_params){ > > + avio_closep(&model_file_context); > > + ff_dnn_free_model_native(&model); > > + return NULL; > > + } > > + depth_to_space_params->block_size = (int32_t)avio_rl32(model_file_context); > > + dnn_size += 4; > > + network->layers[layer].type = DEPTH_TO_SPACE; > > + network->layers[layer].params = depth_to_space_params; > > + break; > > + default: > > + avio_closep(&model_file_context); > > + ff_dnn_free_model_native(&model); > > + return NULL; > > + } > > + } > > + > > + avio_closep(&model_file_context); > > + > > + if (dnn_size != file_size){ > > + ff_dnn_free_model_native(&model); > > + return NULL; > > + } > > + > > + model->set_input_output = &set_input_output_native; > > + > > + return model; > > +} > > + > > +#define CLAMP_TO_EDGE(x, w) ((x) < 0 ? 0 : ((x) >= (w) ? (w - 1) : (x))) > > + > > +static void convolve(const float *input, float *output, const ConvolutionalParams *conv_params, int width, int height) > > +{ > > + int radius = conv_params->kernel_size >> 1; > > + int src_linesize = width * conv_params->input_num; > > + int filter_linesize = conv_params->kernel_size * conv_params->input_num; > > + int filter_size = conv_params->kernel_size * filter_linesize; > > + int pad_size = (conv_params->padding_method == VALID) ? (conv_params->kernel_size - 1) / 2 * conv_params->dilation : 0; > > + > > + for (int y = pad_size; y < height - pad_size; ++y) { > > + for (int x = pad_size; x < width - pad_size; ++x) { > > + for (int n_filter = 0; n_filter < conv_params->output_num; ++n_filter) { > > + output[n_filter] = conv_params->biases[n_filter]; > > + > > + for (int ch = 0; ch < conv_params->input_num; ++ch) { > > + for (int kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y) { > > + for (int kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x) { > > + float input_pel; > > + if (conv_params->padding_method == SAME_CLAMP_TO_EDGE) { > > + int y_pos = CLAMP_TO_EDGE(y + (kernel_y - radius) * conv_params->dilation, height); > > + int x_pos = CLAMP_TO_EDGE(x + (kernel_x - radius) * conv_params->dilation, width); > > + input_pel = input[y_pos * src_linesize + x_pos * conv_params->input_num + ch]; > > + } else { > > + int y_pos = y + (kernel_y - radius) * conv_params->dilation; > > + int x_pos = x + (kernel_x - radius) * conv_params->dilation; > > + input_pel = (x_pos < 0 || x_pos >= width || y_pos < 0 || y_pos >= height) ? 0.0 : > > + input[y_pos * src_linesize + x_pos * conv_params->input_num + ch]; > > + } > > + > > + > > + output[n_filter] += input_pel * conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize + > > + kernel_x * conv_params->input_num + ch]; > > + } > > + } > > + } > > + switch (conv_params->activation){ > > + case RELU: > > + output[n_filter] = FFMAX(output[n_filter], 0.0); > > + break; > > + case TANH: > > + output[n_filter] = 2.0f / (1.0f + exp(-2.0f * output[n_filter])) - 1.0f; > > + break; > > + case SIGMOID: > > + output[n_filter] = 1.0f / (1.0f + exp(-output[n_filter])); > > + break; > > + case NONE: > > + break; > > + case LEAKY_RELU: > > + output[n_filter] = FFMAX(output[n_filter], 0.0) + 0.2 * FFMIN(output[n_filter], 0.0); > > + } > > + } > > + output += conv_params->output_num; > > + } > > + } > > +} > > + > > +static void depth_to_space(const float *input, float *output, int block_size, int width, int height, int channels) > > +{ > > + int y, x, by, bx, ch; > > + int new_channels = channels / (block_size * block_size); > > + int output_linesize = width * channels; > > + int by_linesize = output_linesize / block_size; > > + int x_linesize = new_channels * block_size; > > + > > + for (y = 0; y < height; ++y){ > > + for (x = 0; x < width; ++x){ > > + for (by = 0; by < block_size; ++by){ > > + for (bx = 0; bx < block_size; ++bx){ > > + for (ch = 0; ch < new_channels; ++ch){ > > + output[by * by_linesize + x * x_linesize + bx * new_channels + ch] = input[ch]; > > + } > > + input += new_channels; > > + } > > + } > > + } > > + output += output_linesize; > > + } > > +} > > + > > +DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output) > > +{ > > + ConvolutionalNetwork *network = (ConvolutionalNetwork *)model->model; > > + int cur_width, cur_height, cur_channels; > > + int32_t layer; > > + InputParams *input_params; > > + ConvolutionalParams *conv_params; > > + DepthToSpaceParams *depth_to_space_params; > > + > > + if (network->layers_num <= 0 || network->layers[0].type != INPUT || !network->layers[0].output){ > > + return DNN_ERROR; > > + } > > + else{ > > + input_params = (InputParams *)network->layers[0].params; > > + cur_width = input_params->width; > > + cur_height = input_params->height; > > + cur_channels = input_params->channels; > > + } > > + > > + for (layer = 1; layer < network->layers_num; ++layer){ > > + if (!network->layers[layer].output){ > > + return DNN_ERROR; > > + } > > + switch (network->layers[layer].type){ > > + case CONV: > > + conv_params = (ConvolutionalParams *)network->layers[layer].params; > > + convolve(network->layers[layer - 1].output, network->layers[layer].output, conv_params, cur_width, cur_height); > > + cur_channels = conv_params->output_num; > > + if (conv_params->padding_method == VALID) { > > + int pad_size = (conv_params->kernel_size - 1) * conv_params->dilation; > > + cur_height -= pad_size; > > + cur_width -= pad_size; > > + } > > + break; > > + case DEPTH_TO_SPACE: > > + depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params; > > + depth_to_space(network->layers[layer - 1].output, network->layers[layer].output, > > + depth_to_space_params->block_size, cur_width, cur_height, cur_channels); > > + cur_height *= depth_to_space_params->block_size; > > + cur_width *= depth_to_space_params->block_size; > > + cur_channels /= depth_to_space_params->block_size * depth_to_space_params->block_size; > > + break; > > + case INPUT: > > + return DNN_ERROR; > > + } > > + } > > + > > + // native mode does not support multiple outputs yet > > + if (nb_output > 1) > > + return DNN_ERROR; > > + outputs[0].data = network->layers[network->layers_num - 1].output; > > + outputs[0].height = cur_height; > > + outputs[0].width = cur_width; > > + outputs[0].channels = cur_channels; > > + > > + return DNN_SUCCESS; > > +} > > + > > +void ff_dnn_free_model_native(DNNModel **model) > > +{ > > + ConvolutionalNetwork *network; > > + ConvolutionalParams *conv_params; > > + int32_t layer; > > + > > + if (*model) > > + { > > + network = (ConvolutionalNetwork *)(*model)->model; > > + for (layer = 0; layer < network->layers_num; ++layer){ > > + av_freep(&network->layers[layer].output); > > + if (network->layers[layer].type == CONV){ > > + conv_params = (ConvolutionalParams *)network->layers[layer].params; > > + av_freep(&conv_params->kernel); > > + av_freep(&conv_params->biases); > > + } > > + av_freep(&network->layers[layer].params); > > + } > > + av_freep(&network->layers); > > + av_freep(&network); > > + av_freep(model); > > + } > > +} > > diff --git a/libavfilter/dnn/dnn_backend_native.h b/libavfilter/dnn/dnn_backend_native.h > > new file mode 100644 > > index 0000000..532103c > > --- /dev/null > > +++ b/libavfilter/dnn/dnn_backend_native.h > > @@ -0,0 +1,74 @@ > > +/* > > + * Copyright (c) 2018 Sergey Lavrushkin > > + * > > + * This file is part of FFmpeg. > > + * > > + * FFmpeg is free software; you can redistribute it and/or > > + * modify it under the terms of the GNU Lesser General Public > > + * License as published by the Free Software Foundation; either > > + * version 2.1 of the License, or (at your option) any later version. > > + * > > + * FFmpeg is distributed in the hope that it will be useful, > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > + * Lesser General Public License for more details. > > + * > > + * You should have received a copy of the GNU Lesser General Public > > + * License along with FFmpeg; if not, write to the Free Software > > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > > + */ > > + > > +/** > > + * @file > > + * DNN inference functions interface for native backend. > > + */ > > + > > + > > +#ifndef AVFILTER_DNN_BACKEND_NATIVE_H > > +#define AVFILTER_DNN_BACKEND_NATIVE_H > > + > > +#include "../dnn_interface.h" > > +#include "libavformat/avio.h" > > + > > +typedef enum {INPUT, CONV, DEPTH_TO_SPACE} DNNLayerType; > > + > > +typedef enum {RELU, TANH, SIGMOID, NONE, LEAKY_RELU} DNNActivationFunc; > > + > > +typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} DNNConvPaddingParam; > > + > > +typedef struct Layer{ > > + DNNLayerType type; > > + float *output; > > + void *params; > > +} Layer; > > + > > +typedef struct ConvolutionalParams{ > > + int32_t input_num, output_num, kernel_size; > > + DNNActivationFunc activation; > > + DNNConvPaddingParam padding_method; > > + int32_t dilation; > > + float *kernel; > > + float *biases; > > +} ConvolutionalParams; > > + > > +typedef struct InputParams{ > > + int height, width, channels; > > +} InputParams; > > + > > +typedef struct DepthToSpaceParams{ > > + int block_size; > > +} DepthToSpaceParams; > > + > > +// Represents simple feed-forward convolutional network. > > +typedef struct ConvolutionalNetwork{ > > + Layer *layers; > > + int32_t layers_num; > > +} ConvolutionalNetwork; > > + > > +DNNModel *ff_dnn_load_model_native(const char *model_filename); > > + > > +DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output); > > + > > +void ff_dnn_free_model_native(DNNModel **model); > > + > > +#endif > > diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c > > new file mode 100644 > > index 0000000..ba959ae > > --- /dev/null > > +++ b/libavfilter/dnn/dnn_backend_tf.c > > @@ -0,0 +1,603 @@ > > +/* > > + * Copyright (c) 2018 Sergey Lavrushkin > > + * > > + * This file is part of FFmpeg. > > + * > > + * FFmpeg is free software; you can redistribute it and/or > > + * modify it under the terms of the GNU Lesser General Public > > + * License as published by the Free Software Foundation; either > > + * version 2.1 of the License, or (at your option) any later version. > > + * > > + * FFmpeg is distributed in the hope that it will be useful, > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > + * Lesser General Public License for more details. > > + * > > + * You should have received a copy of the GNU Lesser General Public > > + * License along with FFmpeg; if not, write to the Free Software > > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > > + */ > > + > > +/** > > + * @file > > + * DNN tensorflow backend implementation. > > + */ > > + > > +#include "dnn_backend_tf.h" > > +#include "dnn_backend_native.h" > > +#include "libavformat/avio.h" > > +#include "libavutil/avassert.h" > > + > > +#include <tensorflow/c/c_api.h> > > + > > +typedef struct TFModel{ > > + TF_Graph *graph; > > + TF_Session *session; > > + TF_Status *status; > > + TF_Output input; > > + TF_Tensor *input_tensor; > > + TF_Output *outputs; > > + TF_Tensor **output_tensors; > > + uint32_t nb_output; > > +} TFModel; > > + > > +static void free_buffer(void *data, size_t length) > > +{ > > + av_freep(&data); > > +} > > + > > +static TF_Buffer *read_graph(const char *model_filename) > > +{ > > + TF_Buffer *graph_buf; > > + unsigned char *graph_data = NULL; > > + AVIOContext *model_file_context; > > + long size, bytes_read; > > + > > + if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){ > > + return NULL; > > + } > > + > > + size = avio_size(model_file_context); > > + > > + graph_data = av_malloc(size); > > + if (!graph_data){ > > + avio_closep(&model_file_context); > > + return NULL; > > + } > > + bytes_read = avio_read(model_file_context, graph_data, size); > > + avio_closep(&model_file_context); > > + if (bytes_read != size){ > > + av_freep(&graph_data); > > + return NULL; > > + } > > + > > + graph_buf = TF_NewBuffer(); > > + graph_buf->data = (void *)graph_data; > > + graph_buf->length = size; > > + graph_buf->data_deallocator = free_buffer; > > + > > + return graph_buf; > > +} > > + > > +static TF_Tensor *allocate_input_tensor(const DNNInputData *input) > > +{ > > + TF_DataType dt; > > + size_t size; > > + int64_t input_dims[] = {1, input->height, input->width, input->channels}; > > + switch (input->dt) { > > + case DNN_FLOAT: > > + dt = TF_FLOAT; > > + size = sizeof(float); > > + break; > > + case DNN_UINT8: > > + dt = TF_UINT8; > > + size = sizeof(char); > > + break; > > + default: > > + av_assert0(!"should not reach here"); > > + } > > + > > + return TF_AllocateTensor(dt, input_dims, 4, > > + input_dims[1] * input_dims[2] * input_dims[3] * size); > > +} > > + > > +static DNNReturnType set_input_output_tf(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output) > > +{ > > + TFModel *tf_model = (TFModel *)model; > > + TF_SessionOptions *sess_opts; > > + const TF_Operation *init_op = TF_GraphOperationByName(tf_model->graph, "init"); > > + > > + // Input operation > > + tf_model->input.oper = TF_GraphOperationByName(tf_model->graph, input_name); > > + if (!tf_model->input.oper){ > > + return DNN_ERROR; > > + } > > + tf_model->input.index = 0; > > + if (tf_model->input_tensor){ > > + TF_DeleteTensor(tf_model->input_tensor); > > + } > > + tf_model->input_tensor = allocate_input_tensor(input); > > + if (!tf_model->input_tensor){ > > + return DNN_ERROR; > > + } > > + input->data = (float *)TF_TensorData(tf_model->input_tensor); > > + > > + // Output operation > > + if (nb_output == 0) > > + return DNN_ERROR; > > + > > + av_freep(&tf_model->outputs); > > + tf_model->outputs = av_malloc_array(nb_output, sizeof(*tf_model->outputs)); > > + if (!tf_model->outputs) > > + return DNN_ERROR; > > + for (int i = 0; i < nb_output; ++i) { > > + tf_model->outputs[i].oper = TF_GraphOperationByName(tf_model->graph, output_names[i]); > > + if (!tf_model->outputs[i].oper){ > > + av_freep(&tf_model->outputs); > > + return DNN_ERROR; > > + } > > + tf_model->outputs[i].index = 0; > > + } > > + > > + if (tf_model->output_tensors) { > > + for (uint32_t i = 0; i < tf_model->nb_output; ++i) { > > + if (tf_model->output_tensors[i]) { > > + TF_DeleteTensor(tf_model->output_tensors[i]); > > + tf_model->output_tensors[i] = NULL; > > + } > > + } > > + } > > + av_freep(&tf_model->output_tensors); > > + tf_model->output_tensors = av_mallocz_array(nb_output, sizeof(*tf_model->output_tensors)); > > + if (!tf_model->output_tensors) { > > + av_freep(&tf_model->outputs); > > + return DNN_ERROR; > > + } > > + > > + tf_model->nb_output = nb_output; > > + > > + if (tf_model->session){ > > + TF_CloseSession(tf_model->session, tf_model->status); > > + TF_DeleteSession(tf_model->session, tf_model->status); > > + } > > + > > + sess_opts = TF_NewSessionOptions(); > > + tf_model->session = TF_NewSession(tf_model->graph, sess_opts, tf_model->status); > > + TF_DeleteSessionOptions(sess_opts); > > + if (TF_GetCode(tf_model->status) != TF_OK) > > + { > > + return DNN_ERROR; > > + } > > + > > + // Run initialization operation with name "init" if it is present in graph > > + if (init_op){ > > + TF_SessionRun(tf_model->session, NULL, > > + NULL, NULL, 0, > > + NULL, NULL, 0, > > + &init_op, 1, NULL, tf_model->status); > > + if (TF_GetCode(tf_model->status) != TF_OK) > > + { > > + return DNN_ERROR; > > + } > > + } > > + > > + return DNN_SUCCESS; > > +} > > + > > +static DNNReturnType load_tf_model(TFModel *tf_model, const char *model_filename) > > +{ > > + TF_Buffer *graph_def; > > + TF_ImportGraphDefOptions *graph_opts; > > + > > + graph_def = read_graph(model_filename); > > + if (!graph_def){ > > + return DNN_ERROR; > > + } > > + tf_model->graph = TF_NewGraph(); > > + tf_model->status = TF_NewStatus(); > > + graph_opts = TF_NewImportGraphDefOptions(); > > + TF_GraphImportGraphDef(tf_model->graph, graph_def, graph_opts, tf_model->status); > > + TF_DeleteImportGraphDefOptions(graph_opts); > > + TF_DeleteBuffer(graph_def); > > + if (TF_GetCode(tf_model->status) != TF_OK){ > > + TF_DeleteGraph(tf_model->graph); > > + TF_DeleteStatus(tf_model->status); > > + return DNN_ERROR; > > + } > > + > > + return DNN_SUCCESS; > > +} > > + > > +#define NAME_BUFFER_SIZE 256 > > + > > +static DNNReturnType add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op, > > + ConvolutionalParams* params, const int layer) > > +{ > > + TF_Operation *op; > > + TF_OperationDescription *op_desc; > > + TF_Output input; > > + int64_t strides[] = {1, 1, 1, 1}; > > + TF_Tensor *tensor; > > + int64_t dims[4]; > > + int dims_len; > > + char name_buffer[NAME_BUFFER_SIZE]; > > + int32_t size; > > + > > + size = params->input_num * params->output_num * params->kernel_size * params->kernel_size; > > + input.index = 0; > > + > > + snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_kernel%d", layer); > > + op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer); > > + TF_SetAttrType(op_desc, "dtype", TF_FLOAT); > > + dims[0] = params->output_num; > > + dims[1] = params->kernel_size; > > + dims[2] = params->kernel_size; > > + dims[3] = params->input_num; > > + dims_len = 4; > > + tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, size * sizeof(float)); > > + memcpy(TF_TensorData(tensor), params->kernel, size * sizeof(float)); > > + TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); > > + if (TF_GetCode(tf_model->status) != TF_OK){ > > + return DNN_ERROR; > > + } > > + op = TF_FinishOperation(op_desc, tf_model->status); > > + if (TF_GetCode(tf_model->status) != TF_OK){ > > + return DNN_ERROR; > > + } > > + > > + snprintf(name_buffer, NAME_BUFFER_SIZE, "transpose%d", layer); > > + op_desc = TF_NewOperation(tf_model->graph, "Transpose", name_buffer); > > + input.oper = op; > > + TF_AddInput(op_desc, input); > > + input.oper = transpose_op; > > + TF_AddInput(op_desc, input); > > + TF_SetAttrType(op_desc, "T", TF_FLOAT); > > + TF_SetAttrType(op_desc, "Tperm", TF_INT32); > > + op = TF_FinishOperation(op_desc, tf_model->status); > > + if (TF_GetCode(tf_model->status) != TF_OK){ > > + return DNN_ERROR; > > + } > > + > > + snprintf(name_buffer, NAME_BUFFER_SIZE, "conv2d%d", layer); > > + op_desc = TF_NewOperation(tf_model->graph, "Conv2D", name_buffer); > > + input.oper = *cur_op; > > + TF_AddInput(op_desc, input); > > + input.oper = op; > > + TF_AddInput(op_desc, input); > > + TF_SetAttrType(op_desc, "T", TF_FLOAT); > > + TF_SetAttrIntList(op_desc, "strides", strides, 4); > > + TF_SetAttrString(op_desc, "padding", "VALID", 5); > > + *cur_op = TF_FinishOperation(op_desc, tf_model->status); > > + if (TF_GetCode(tf_model->status) != TF_OK){ > > + return DNN_ERROR; > > + } > > + > > + snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_biases%d", layer); > > + op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer); > > + TF_SetAttrType(op_desc, "dtype", TF_FLOAT); > > + dims[0] = params->output_num; > > + dims_len = 1; > > + tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, params->output_num * sizeof(float)); > > + memcpy(TF_TensorData(tensor), params->biases, params->output_num * sizeof(float)); > > + TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); > > + if (TF_GetCode(tf_model->status) != TF_OK){ > > + return DNN_ERROR; > > + } > > + op = TF_FinishOperation(op_desc, tf_model->status); > > + if (TF_GetCode(tf_model->status) != TF_OK){ > > + return DNN_ERROR; > > + } > > + > > + snprintf(name_buffer, NAME_BUFFER_SIZE, "bias_add%d", layer); > > + op_desc = TF_NewOperation(tf_model->graph, "BiasAdd", name_buffer); > > + input.oper = *cur_op; > > + TF_AddInput(op_desc, input); > > + input.oper = op; > > + TF_AddInput(op_desc, input); > > + TF_SetAttrType(op_desc, "T", TF_FLOAT); > > + *cur_op = TF_FinishOperation(op_desc, tf_model->status); > > + if (TF_GetCode(tf_model->status) != TF_OK){ > > + return DNN_ERROR; > > + } > > + > > + snprintf(name_buffer, NAME_BUFFER_SIZE, "activation%d", layer); > > + switch (params->activation){ > > + case RELU: > > + op_desc = TF_NewOperation(tf_model->graph, "Relu", name_buffer); > > + break; > > + case TANH: > > + op_desc = TF_NewOperation(tf_model->graph, "Tanh", name_buffer); > > + break; > > + case SIGMOID: > > + op_desc = TF_NewOperation(tf_model->graph, "Sigmoid", name_buffer); > > + break; > > + default: > > + return DNN_ERROR; > > + } > > + input.oper = *cur_op; > > + TF_AddInput(op_desc, input); > > + TF_SetAttrType(op_desc, "T", TF_FLOAT); > > + *cur_op = TF_FinishOperation(op_desc, tf_model->status); > > + if (TF_GetCode(tf_model->status) != TF_OK){ > > + return DNN_ERROR; > > + } > > + > > + return DNN_SUCCESS; > > +} > > + > > +static DNNReturnType add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op, > > + DepthToSpaceParams *params, const int layer) > > +{ > > + TF_OperationDescription *op_desc; > > + TF_Output input; > > + char name_buffer[NAME_BUFFER_SIZE]; > > + > > + snprintf(name_buffer, NAME_BUFFER_SIZE, "depth_to_space%d", layer); > > + op_desc = TF_NewOperation(tf_model->graph, "DepthToSpace", name_buffer); > > + input.oper = *cur_op; > > + input.index = 0; > > + TF_AddInput(op_desc, input); > > + TF_SetAttrType(op_desc, "T", TF_FLOAT); > > + TF_SetAttrInt(op_desc, "block_size", params->block_size); > > + *cur_op = TF_FinishOperation(op_desc, tf_model->status); > > + if (TF_GetCode(tf_model->status) != TF_OK){ > > + return DNN_ERROR; > > + } > > + > > + return DNN_SUCCESS; > > +} > > + > > +static int calculate_pad(const ConvolutionalNetwork *conv_network) > > +{ > > + ConvolutionalParams *params; > > + int32_t layer; > > + int pad = 0; > > + > > + for (layer = 0; layer < conv_network->layers_num; ++layer){ > > + if (conv_network->layers[layer].type == CONV){ > > + params = (ConvolutionalParams *)conv_network->layers[layer].params; > > + pad += params->kernel_size >> 1; > > + } > > + } > > + > > + return pad; > > +} > > + > > +static DNNReturnType add_pad_op(TFModel *tf_model, TF_Operation **cur_op, const int32_t pad) > > +{ > > + TF_Operation *op; > > + TF_Tensor *tensor; > > + TF_OperationDescription *op_desc; > > + TF_Output input; > > + int32_t *pads; > > + int64_t pads_shape[] = {4, 2}; > > + > > + input.index = 0; > > + > > + op_desc = TF_NewOperation(tf_model->graph, "Const", "pads"); > > + TF_SetAttrType(op_desc, "dtype", TF_INT32); > > + tensor = TF_AllocateTensor(TF_INT32, pads_shape, 2, 4 * 2 * sizeof(int32_t)); > > + pads = (int32_t *)TF_TensorData(tensor); > > + pads[0] = 0; pads[1] = 0; > > + pads[2] = pad; pads[3] = pad; > > + pads[4] = pad; pads[5] = pad; > > + pads[6] = 0; pads[7] = 0; > > + TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); > > + if (TF_GetCode(tf_model->status) != TF_OK){ > > + return DNN_ERROR; > > + } > > + op = TF_FinishOperation(op_desc, tf_model->status); > > + if (TF_GetCode(tf_model->status) != TF_OK){ > > + return DNN_ERROR; > > + } > > + > > + op_desc = TF_NewOperation(tf_model->graph, "MirrorPad", "mirror_pad"); > > + input.oper = *cur_op; > > + TF_AddInput(op_desc, input); > > + input.oper = op; > > + TF_AddInput(op_desc, input); > > + TF_SetAttrType(op_desc, "T", TF_FLOAT); > > + TF_SetAttrType(op_desc, "Tpaddings", TF_INT32); > > + TF_SetAttrString(op_desc, "mode", "SYMMETRIC", 9); > > + *cur_op = TF_FinishOperation(op_desc, tf_model->status); > > + if (TF_GetCode(tf_model->status) != TF_OK){ > > + return DNN_ERROR; > > + } > > + > > + return DNN_SUCCESS; > > +} > > + > > +static DNNReturnType load_native_model(TFModel *tf_model, const char *model_filename) > > +{ > > + int32_t layer; > > + TF_OperationDescription *op_desc; > > + TF_Operation *op; > > + TF_Operation *transpose_op; > > + TF_Tensor *tensor; > > + TF_Output input; > > + int32_t *transpose_perm; > > + int64_t transpose_perm_shape[] = {4}; > > + int64_t input_shape[] = {1, -1, -1, -1}; > > + int32_t pad; > > + DNNReturnType layer_add_res; > > + DNNModel *native_model = NULL; > > + ConvolutionalNetwork *conv_network; > > + > > + native_model = ff_dnn_load_model_native(model_filename); > > + if (!native_model){ > > + return DNN_ERROR; > > + } > > + > > + conv_network = (ConvolutionalNetwork *)native_model->model; > > + pad = calculate_pad(conv_network); > > + tf_model->graph = TF_NewGraph(); > > + tf_model->status = TF_NewStatus(); > > + > > +#define CLEANUP_ON_ERROR(tf_model) \ > > + { \ > > + TF_DeleteGraph(tf_model->graph); \ > > + TF_DeleteStatus(tf_model->status); \ > > + return DNN_ERROR; \ > > + } > > + > > + op_desc = TF_NewOperation(tf_model->graph, "Placeholder", "x"); > > + TF_SetAttrType(op_desc, "dtype", TF_FLOAT); > > + TF_SetAttrShape(op_desc, "shape", input_shape, 4); > > + op = TF_FinishOperation(op_desc, tf_model->status); > > + if (TF_GetCode(tf_model->status) != TF_OK){ > > + CLEANUP_ON_ERROR(tf_model); > > + } > > + > > + if (add_pad_op(tf_model, &op, pad) != DNN_SUCCESS){ > > + CLEANUP_ON_ERROR(tf_model); > > + } > > + > > + op_desc = TF_NewOperation(tf_model->graph, "Const", "transpose_perm"); > > + TF_SetAttrType(op_desc, "dtype", TF_INT32); > > + tensor = TF_AllocateTensor(TF_INT32, transpose_perm_shape, 1, 4 * sizeof(int32_t)); > > + transpose_perm = (int32_t *)TF_TensorData(tensor); > > + transpose_perm[0] = 1; > > + transpose_perm[1] = 2; > > + transpose_perm[2] = 3; > > + transpose_perm[3] = 0; > > + TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); > > + if (TF_GetCode(tf_model->status) != TF_OK){ > > + CLEANUP_ON_ERROR(tf_model); > > + } > > + transpose_op = TF_FinishOperation(op_desc, tf_model->status); > > + > > + for (layer = 0; layer < conv_network->layers_num; ++layer){ > > + switch (conv_network->layers[layer].type){ > > + case INPUT: > > + layer_add_res = DNN_SUCCESS; > > + break; > > + case CONV: > > + layer_add_res = add_conv_layer(tf_model, transpose_op, &op, > > + (ConvolutionalParams *)conv_network->layers[layer].params, layer); > > + break; > > + case DEPTH_TO_SPACE: > > + layer_add_res = add_depth_to_space_layer(tf_model, &op, > > + (DepthToSpaceParams *)conv_network->layers[layer].params, layer); > > + break; > > + default: > > + CLEANUP_ON_ERROR(tf_model); > > + } > > + > > + if (layer_add_res != DNN_SUCCESS){ > > + CLEANUP_ON_ERROR(tf_model); > > + } > > + } > > + > > + op_desc = TF_NewOperation(tf_model->graph, "Identity", "y"); > > + input.oper = op; > > + TF_AddInput(op_desc, input); > > + TF_FinishOperation(op_desc, tf_model->status); > > + if (TF_GetCode(tf_model->status) != TF_OK){ > > + CLEANUP_ON_ERROR(tf_model); > > + } > > + > > + ff_dnn_free_model_native(&native_model); > > + > > + return DNN_SUCCESS; > > +} > > + > > +DNNModel *ff_dnn_load_model_tf(const char *model_filename) > > +{ > > + DNNModel *model = NULL; > > + TFModel *tf_model = NULL; > > + > > + model = av_malloc(sizeof(DNNModel)); > > + if (!model){ > > + return NULL; > > + } > > + > > + tf_model = av_mallocz(sizeof(TFModel)); > > + if (!tf_model){ > > + av_freep(&model); > > + return NULL; > > + } > > + > > + if (load_tf_model(tf_model, model_filename) != DNN_SUCCESS){ > > + if (load_native_model(tf_model, model_filename) != DNN_SUCCESS){ > > + av_freep(&tf_model); > > + av_freep(&model); > > + > > + return NULL; > > + } > > + } > > + > > + model->model = (void *)tf_model; > > + model->set_input_output = &set_input_output_tf; > > + > > + return model; > > +} > > + > > + > > + > > +DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, uint32_t nb_output) > > +{ > > + TFModel *tf_model = (TFModel *)model->model; > > + uint32_t nb = FFMIN(nb_output, tf_model->nb_output); > > + if (nb == 0) > > + return DNN_ERROR; > > + > > + av_assert0(tf_model->output_tensors); > > + for (uint32_t i = 0; i < tf_model->nb_output; ++i) { > > + if (tf_model->output_tensors[i]) { > > + TF_DeleteTensor(tf_model->output_tensors[i]); > > + tf_model->output_tensors[i] = NULL; > > + } > > + } > > + > > + TF_SessionRun(tf_model->session, NULL, > > + &tf_model->input, &tf_model->input_tensor, 1, > > + tf_model->outputs, tf_model->output_tensors, nb, > > + NULL, 0, NULL, tf_model->status); > > + > > + if (TF_GetCode(tf_model->status) != TF_OK){ > > + return DNN_ERROR; > > + } > > + > > + for (uint32_t i = 0; i < nb; ++i) { > > + outputs[i].height = TF_Dim(tf_model->output_tensors[i], 1); > > + outputs[i].width = TF_Dim(tf_model->output_tensors[i], 2); > > + outputs[i].channels = TF_Dim(tf_model->output_tensors[i], 3); > > + outputs[i].data = TF_TensorData(tf_model->output_tensors[i]); > > + } > > + > > + return DNN_SUCCESS; > > +} > > + > > +void ff_dnn_free_model_tf(DNNModel **model) > > +{ > > + TFModel *tf_model; > > + > > + if (*model){ > > + tf_model = (TFModel *)(*model)->model; > > + if (tf_model->graph){ > > + TF_DeleteGraph(tf_model->graph); > > + } > > + if (tf_model->session){ > > + TF_CloseSession(tf_model->session, tf_model->status); > > + TF_DeleteSession(tf_model->session, tf_model->status); > > + } > > + if (tf_model->status){ > > + TF_DeleteStatus(tf_model->status); > > + } > > + if (tf_model->input_tensor){ > > + TF_DeleteTensor(tf_model->input_tensor); > > + } > > + if (tf_model->output_tensors) { > > + for (uint32_t i = 0; i < tf_model->nb_output; ++i) { > > + if (tf_model->output_tensors[i]) { > > + TF_DeleteTensor(tf_model->output_tensors[i]); > > + tf_model->output_tensors[i] = NULL; > > + } > > + } > > + } > > + av_freep(&tf_model->outputs); > > + av_freep(&tf_model->output_tensors); > > + av_freep(&tf_model); > > + av_freep(model); > > + } > > +} > > diff --git a/libavfilter/dnn/dnn_backend_tf.h b/libavfilter/dnn/dnn_backend_tf.h > > new file mode 100644 > > index 0000000..bb1c85f > > --- /dev/null > > +++ b/libavfilter/dnn/dnn_backend_tf.h > > @@ -0,0 +1,38 @@ > > +/* > > + * Copyright (c) 2018 Sergey Lavrushkin > > + * > > + * This file is part of FFmpeg. > > + * > > + * FFmpeg is free software; you can redistribute it and/or > > + * modify it under the terms of the GNU Lesser General Public > > + * License as published by the Free Software Foundation; either > > + * version 2.1 of the License, or (at your option) any later version. > > + * > > + * FFmpeg is distributed in the hope that it will be useful, > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > + * Lesser General Public License for more details. > > + * > > + * You should have received a copy of the GNU Lesser General Public > > + * License along with FFmpeg; if not, write to the Free Software > > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > > + */ > > + > > +/** > > + * @file > > + * DNN inference functions interface for TensorFlow backend. > > + */ > > + > > + > > +#ifndef AVFILTER_DNN_BACKEND_TF_H > > +#define AVFILTER_DNN_BACKEND_TF_H > > + > > +#include "../dnn_interface.h" > > + > > +DNNModel *ff_dnn_load_model_tf(const char *model_filename); > > + > > +DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, uint32_t nb_output); > > + > > +void ff_dnn_free_model_tf(DNNModel **model); > > + > > +#endif > > diff --git a/libavfilter/dnn/dnn_interface.c b/libavfilter/dnn/dnn_interface.c > > new file mode 100644 > > index 0000000..62da55f > > --- /dev/null > > +++ b/libavfilter/dnn/dnn_interface.c > > @@ -0,0 +1,63 @@ > > +/* > > + * Copyright (c) 2018 Sergey Lavrushkin > > + * > > + * This file is part of FFmpeg. > > + * > > + * FFmpeg is free software; you can redistribute it and/or > > + * modify it under the terms of the GNU Lesser General Public > > + * License as published by the Free Software Foundation; either > > + * version 2.1 of the License, or (at your option) any later version. > > + * > > + * FFmpeg is distributed in the hope that it will be useful, > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > + * Lesser General Public License for more details. > > + * > > + * You should have received a copy of the GNU Lesser General Public > > + * License along with FFmpeg; if not, write to the Free Software > > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > > + */ > > + > > +/** > > + * @file > > + * Implements DNN module initialization with specified backend. > > + */ > > + > > +#include "../dnn_interface.h" > > +#include "dnn_backend_native.h" > > +#include "dnn_backend_tf.h" > > +#include "libavutil/mem.h" > > + > > +DNNModule *ff_get_dnn_module(DNNBackendType backend_type) > > +{ > > + DNNModule *dnn_module; > > + > > + dnn_module = av_malloc(sizeof(DNNModule)); > > + if(!dnn_module){ > > + return NULL; > > + } > > + > > + switch(backend_type){ > > + case DNN_NATIVE: > > + dnn_module->load_model = &ff_dnn_load_model_native; > > + dnn_module->execute_model = &ff_dnn_execute_model_native; > > + dnn_module->free_model = &ff_dnn_free_model_native; > > + break; > > + case DNN_TF: > > + #if (CONFIG_LIBTENSORFLOW == 1) > > + dnn_module->load_model = &ff_dnn_load_model_tf; > > + dnn_module->execute_model = &ff_dnn_execute_model_tf; > > + dnn_module->free_model = &ff_dnn_free_model_tf; > > + #else > > + av_freep(&dnn_module); > > + return NULL; > > + #endif > > + break; > > + default: > > + av_log(NULL, AV_LOG_ERROR, "Module backend_type is not native or tensorflow\n"); > > + av_freep(&dnn_module); > > + return NULL; > > + } > > + > > + return dnn_module; > > +} > > diff --git a/libavfilter/dnn_backend_native.c b/libavfilter/dnn_backend_native.c > > deleted file mode 100644 > > index 82e900b..0000000 > > --- a/libavfilter/dnn_backend_native.c > > +++ /dev/null > > @@ -1,389 +0,0 @@ > > -/* > > - * Copyright (c) 2018 Sergey Lavrushkin > > - * > > - * This file is part of FFmpeg. > > - * > > - * FFmpeg is free software; you can redistribute it and/or > > - * modify it under the terms of the GNU Lesser General Public > > - * License as published by the Free Software Foundation; either > > - * version 2.1 of the License, or (at your option) any later version. > > - * > > - * FFmpeg is distributed in the hope that it will be useful, > > - * but WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * Lesser General Public License for more details. > > - * > > - * You should have received a copy of the GNU Lesser General Public > > - * License along with FFmpeg; if not, write to the Free Software > > - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > > - */ > > - > > -/** > > - * @file > > - * DNN native backend implementation. > > - */ > > - > > -#include "dnn_backend_native.h" > > -#include "libavutil/avassert.h" > > - > > -static DNNReturnType set_input_output_native(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output) > > -{ > > - ConvolutionalNetwork *network = (ConvolutionalNetwork *)model; > > - InputParams *input_params; > > - ConvolutionalParams *conv_params; > > - DepthToSpaceParams *depth_to_space_params; > > - int cur_width, cur_height, cur_channels; > > - int32_t layer; > > - > > - if (network->layers_num <= 0 || network->layers[0].type != INPUT){ > > - return DNN_ERROR; > > - } > > - else{ > > - input_params = (InputParams *)network->layers[0].params; > > - input_params->width = cur_width = input->width; > > - input_params->height = cur_height = input->height; > > - input_params->channels = cur_channels = input->channels; > > - if (input->data){ > > - av_freep(&input->data); > > - } > > - av_assert0(input->dt == DNN_FLOAT); > > - network->layers[0].output = input->data = av_malloc(cur_height * cur_width * cur_channels * sizeof(float)); > > - if (!network->layers[0].output){ > > - return DNN_ERROR; > > - } > > - } > > - > > - for (layer = 1; layer < network->layers_num; ++layer){ > > - switch (network->layers[layer].type){ > > - case CONV: > > - conv_params = (ConvolutionalParams *)network->layers[layer].params; > > - if (conv_params->input_num != cur_channels){ > > - return DNN_ERROR; > > - } > > - cur_channels = conv_params->output_num; > > - > > - if (conv_params->padding_method == VALID) { > > - int pad_size = (conv_params->kernel_size - 1) * conv_params->dilation; > > - cur_height -= pad_size; > > - cur_width -= pad_size; > > - } > > - break; > > - case DEPTH_TO_SPACE: > > - depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params; > > - if (cur_channels % (depth_to_space_params->block_size * depth_to_space_params->block_size) != 0){ > > - return DNN_ERROR; > > - } > > - cur_channels = cur_channels / (depth_to_space_params->block_size * depth_to_space_params->block_size); > > - cur_height *= depth_to_space_params->block_size; > > - cur_width *= depth_to_space_params->block_size; > > - break; > > - default: > > - return DNN_ERROR; > > - } > > - if (network->layers[layer].output){ > > - av_freep(&network->layers[layer].output); > > - } > > - > > - if (cur_height <= 0 || cur_width <= 0) > > - return DNN_ERROR; > > - > > - network->layers[layer].output = av_malloc(cur_height * cur_width * cur_channels * sizeof(float)); > > - if (!network->layers[layer].output){ > > - return DNN_ERROR; > > - } > > - } > > - > > - return DNN_SUCCESS; > > -} > > - > > -// Loads model and its parameters that are stored in a binary file with following structure: > > -// layers_num,layer_type,layer_parameterss,layer_type,layer_parameters... > > -// For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases > > -// For DEPTH_TO_SPACE layer: block_size > > -DNNModel *ff_dnn_load_model_native(const char *model_filename) > > -{ > > - DNNModel *model = NULL; > > - ConvolutionalNetwork *network = NULL; > > - AVIOContext *model_file_context; > > - int file_size, dnn_size, kernel_size, i; > > - int32_t layer; > > - DNNLayerType layer_type; > > - ConvolutionalParams *conv_params; > > - DepthToSpaceParams *depth_to_space_params; > > - > > - model = av_malloc(sizeof(DNNModel)); > > - if (!model){ > > - return NULL; > > - } > > - > > - if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){ > > - av_freep(&model); > > - return NULL; > > - } > > - file_size = avio_size(model_file_context); > > - > > - network = av_malloc(sizeof(ConvolutionalNetwork)); > > - if (!network){ > > - avio_closep(&model_file_context); > > - av_freep(&model); > > - return NULL; > > - } > > - model->model = (void *)network; > > - > > - network->layers_num = 1 + (int32_t)avio_rl32(model_file_context); > > - dnn_size = 4; > > - > > - network->layers = av_malloc(network->layers_num * sizeof(Layer)); > > - if (!network->layers){ > > - av_freep(&network); > > - avio_closep(&model_file_context); > > - av_freep(&model); > > - return NULL; > > - } > > - > > - for (layer = 0; layer < network->layers_num; ++layer){ > > - network->layers[layer].output = NULL; > > - network->layers[layer].params = NULL; > > - } > > - network->layers[0].type = INPUT; > > - network->layers[0].params = av_malloc(sizeof(InputParams)); > > - if (!network->layers[0].params){ > > - avio_closep(&model_file_context); > > - ff_dnn_free_model_native(&model); > > - return NULL; > > - } > > - > > - for (layer = 1; layer < network->layers_num; ++layer){ > > - layer_type = (int32_t)avio_rl32(model_file_context); > > - dnn_size += 4; > > - switch (layer_type){ > > - case CONV: > > - conv_params = av_malloc(sizeof(ConvolutionalParams)); > > - if (!conv_params){ > > - avio_closep(&model_file_context); > > - ff_dnn_free_model_native(&model); > > - return NULL; > > - } > > - conv_params->dilation = (int32_t)avio_rl32(model_file_context); > > - conv_params->padding_method = (int32_t)avio_rl32(model_file_context); > > - conv_params->activation = (int32_t)avio_rl32(model_file_context); > > - conv_params->input_num = (int32_t)avio_rl32(model_file_context); > > - conv_params->output_num = (int32_t)avio_rl32(model_file_context); > > - conv_params->kernel_size = (int32_t)avio_rl32(model_file_context); > > - kernel_size = conv_params->input_num * conv_params->output_num * > > - conv_params->kernel_size * conv_params->kernel_size; > > - dnn_size += 24 + (kernel_size + conv_params->output_num << 2); > > - if (dnn_size > file_size || conv_params->input_num <= 0 || > > - conv_params->output_num <= 0 || conv_params->kernel_size <= 0){ > > - avio_closep(&model_file_context); > > - ff_dnn_free_model_native(&model); > > - return NULL; > > - } > > - conv_params->kernel = av_malloc(kernel_size * sizeof(float)); > > - conv_params->biases = av_malloc(conv_params->output_num * sizeof(float)); > > - if (!conv_params->kernel || !conv_params->biases){ > > - avio_closep(&model_file_context); > > - ff_dnn_free_model_native(&model); > > - return NULL; > > - } > > - for (i = 0; i < kernel_size; ++i){ > > - conv_params->kernel[i] = av_int2float(avio_rl32(model_file_context)); > > - } > > - for (i = 0; i < conv_params->output_num; ++i){ > > - conv_params->biases[i] = av_int2float(avio_rl32(model_file_context)); > > - } > > - network->layers[layer].type = CONV; > > - network->layers[layer].params = conv_params; > > - break; > > - case DEPTH_TO_SPACE: > > - depth_to_space_params = av_malloc(sizeof(DepthToSpaceParams)); > > - if (!depth_to_space_params){ > > - avio_closep(&model_file_context); > > - ff_dnn_free_model_native(&model); > > - return NULL; > > - } > > - depth_to_space_params->block_size = (int32_t)avio_rl32(model_file_context); > > - dnn_size += 4; > > - network->layers[layer].type = DEPTH_TO_SPACE; > > - network->layers[layer].params = depth_to_space_params; > > - break; > > - default: > > - avio_closep(&model_file_context); > > - ff_dnn_free_model_native(&model); > > - return NULL; > > - } > > - } > > - > > - avio_closep(&model_file_context); > > - > > - if (dnn_size != file_size){ > > - ff_dnn_free_model_native(&model); > > - return NULL; > > - } > > - > > - model->set_input_output = &set_input_output_native; > > - > > - return model; > > -} > > - > > -#define CLAMP_TO_EDGE(x, w) ((x) < 0 ? 0 : ((x) >= (w) ? (w - 1) : (x))) > > - > > -static void convolve(const float *input, float *output, const ConvolutionalParams *conv_params, int width, int height) > > -{ > > - int radius = conv_params->kernel_size >> 1; > > - int src_linesize = width * conv_params->input_num; > > - int filter_linesize = conv_params->kernel_size * conv_params->input_num; > > - int filter_size = conv_params->kernel_size * filter_linesize; > > - int pad_size = (conv_params->padding_method == VALID) ? (conv_params->kernel_size - 1) / 2 * conv_params->dilation : 0; > > - > > - for (int y = pad_size; y < height - pad_size; ++y) { > > - for (int x = pad_size; x < width - pad_size; ++x) { > > - for (int n_filter = 0; n_filter < conv_params->output_num; ++n_filter) { > > - output[n_filter] = conv_params->biases[n_filter]; > > - > > - for (int ch = 0; ch < conv_params->input_num; ++ch) { > > - for (int kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y) { > > - for (int kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x) { > > - float input_pel; > > - if (conv_params->padding_method == SAME_CLAMP_TO_EDGE) { > > - int y_pos = CLAMP_TO_EDGE(y + (kernel_y - radius) * conv_params->dilation, height); > > - int x_pos = CLAMP_TO_EDGE(x + (kernel_x - radius) * conv_params->dilation, width); > > - input_pel = input[y_pos * src_linesize + x_pos * conv_params->input_num + ch]; > > - } else { > > - int y_pos = y + (kernel_y - radius) * conv_params->dilation; > > - int x_pos = x + (kernel_x - radius) * conv_params->dilation; > > - input_pel = (x_pos < 0 || x_pos >= width || y_pos < 0 || y_pos >= height) ? 0.0 : > > - input[y_pos * src_linesize + x_pos * conv_params->input_num + ch]; > > - } > > - > > - > > - output[n_filter] += input_pel * conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize + > > - kernel_x * conv_params->input_num + ch]; > > - } > > - } > > - } > > - switch (conv_params->activation){ > > - case RELU: > > - output[n_filter] = FFMAX(output[n_filter], 0.0); > > - break; > > - case TANH: > > - output[n_filter] = 2.0f / (1.0f + exp(-2.0f * output[n_filter])) - 1.0f; > > - break; > > - case SIGMOID: > > - output[n_filter] = 1.0f / (1.0f + exp(-output[n_filter])); > > - break; > > - case NONE: > > - break; > > - case LEAKY_RELU: > > - output[n_filter] = FFMAX(output[n_filter], 0.0) + 0.2 * FFMIN(output[n_filter], 0.0); > > - } > > - } > > - output += conv_params->output_num; > > - } > > - } > > -} > > - > > -static void depth_to_space(const float *input, float *output, int block_size, int width, int height, int channels) > > -{ > > - int y, x, by, bx, ch; > > - int new_channels = channels / (block_size * block_size); > > - int output_linesize = width * channels; > > - int by_linesize = output_linesize / block_size; > > - int x_linesize = new_channels * block_size; > > - > > - for (y = 0; y < height; ++y){ > > - for (x = 0; x < width; ++x){ > > - for (by = 0; by < block_size; ++by){ > > - for (bx = 0; bx < block_size; ++bx){ > > - for (ch = 0; ch < new_channels; ++ch){ > > - output[by * by_linesize + x * x_linesize + bx * new_channels + ch] = input[ch]; > > - } > > - input += new_channels; > > - } > > - } > > - } > > - output += output_linesize; > > - } > > -} > > - > > -DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output) > > -{ > > - ConvolutionalNetwork *network = (ConvolutionalNetwork *)model->model; > > - int cur_width, cur_height, cur_channels; > > - int32_t layer; > > - InputParams *input_params; > > - ConvolutionalParams *conv_params; > > - DepthToSpaceParams *depth_to_space_params; > > - > > - if (network->layers_num <= 0 || network->layers[0].type != INPUT || !network->layers[0].output){ > > - return DNN_ERROR; > > - } > > - else{ > > - input_params = (InputParams *)network->layers[0].params; > > - cur_width = input_params->width; > > - cur_height = input_params->height; > > - cur_channels = input_params->channels; > > - } > > - > > - for (layer = 1; layer < network->layers_num; ++layer){ > > - if (!network->layers[layer].output){ > > - return DNN_ERROR; > > - } > > - switch (network->layers[layer].type){ > > - case CONV: > > - conv_params = (ConvolutionalParams *)network->layers[layer].params; > > - convolve(network->layers[layer - 1].output, network->layers[layer].output, conv_params, cur_width, cur_height); > > - cur_channels = conv_params->output_num; > > - if (conv_params->padding_method == VALID) { > > - int pad_size = (conv_params->kernel_size - 1) * conv_params->dilation; > > - cur_height -= pad_size; > > - cur_width -= pad_size; > > - } > > - break; > > - case DEPTH_TO_SPACE: > > - depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params; > > - depth_to_space(network->layers[layer - 1].output, network->layers[layer].output, > > - depth_to_space_params->block_size, cur_width, cur_height, cur_channels); > > - cur_height *= depth_to_space_params->block_size; > > - cur_width *= depth_to_space_params->block_size; > > - cur_channels /= depth_to_space_params->block_size * depth_to_space_params->block_size; > > - break; > > - case INPUT: > > - return DNN_ERROR; > > - } > > - } > > - > > - // native mode does not support multiple outputs yet > > - if (nb_output > 1) > > - return DNN_ERROR; > > - outputs[0].data = network->layers[network->layers_num - 1].output; > > - outputs[0].height = cur_height; > > - outputs[0].width = cur_width; > > - outputs[0].channels = cur_channels; > > - > > - return DNN_SUCCESS; > > -} > > - > > -void ff_dnn_free_model_native(DNNModel **model) > > -{ > > - ConvolutionalNetwork *network; > > - ConvolutionalParams *conv_params; > > - int32_t layer; > > - > > - if (*model) > > - { > > - network = (ConvolutionalNetwork *)(*model)->model; > > - for (layer = 0; layer < network->layers_num; ++layer){ > > - av_freep(&network->layers[layer].output); > > - if (network->layers[layer].type == CONV){ > > - conv_params = (ConvolutionalParams *)network->layers[layer].params; > > - av_freep(&conv_params->kernel); > > - av_freep(&conv_params->biases); > > - } > > - av_freep(&network->layers[layer].params); > > - } > > - av_freep(&network->layers); > > - av_freep(&network); > > - av_freep(model); > > - } > > -} > > diff --git a/libavfilter/dnn_backend_native.h b/libavfilter/dnn_backend_native.h > > deleted file mode 100644 > > index 5917955..0000000 > > --- a/libavfilter/dnn_backend_native.h > > +++ /dev/null > > @@ -1,74 +0,0 @@ > > -/* > > - * Copyright (c) 2018 Sergey Lavrushkin > > - * > > - * This file is part of FFmpeg. > > - * > > - * FFmpeg is free software; you can redistribute it and/or > > - * modify it under the terms of the GNU Lesser General Public > > - * License as published by the Free Software Foundation; either > > - * version 2.1 of the License, or (at your option) any later version. > > - * > > - * FFmpeg is distributed in the hope that it will be useful, > > - * but WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * Lesser General Public License for more details. > > - * > > - * You should have received a copy of the GNU Lesser General Public > > - * License along with FFmpeg; if not, write to the Free Software > > - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > > - */ > > - > > -/** > > - * @file > > - * DNN inference functions interface for native backend. > > - */ > > - > > - > > -#ifndef AVFILTER_DNN_BACKEND_NATIVE_H > > -#define AVFILTER_DNN_BACKEND_NATIVE_H > > - > > -#include "dnn_interface.h" > > -#include "libavformat/avio.h" > > - > > -typedef enum {INPUT, CONV, DEPTH_TO_SPACE} DNNLayerType; > > - > > -typedef enum {RELU, TANH, SIGMOID, NONE, LEAKY_RELU} DNNActivationFunc; > > - > > -typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} DNNConvPaddingParam; > > - > > -typedef struct Layer{ > > - DNNLayerType type; > > - float *output; > > - void *params; > > -} Layer; > > - > > -typedef struct ConvolutionalParams{ > > - int32_t input_num, output_num, kernel_size; > > - DNNActivationFunc activation; > > - DNNConvPaddingParam padding_method; > > - int32_t dilation; > > - float *kernel; > > - float *biases; > > -} ConvolutionalParams; > > - > > -typedef struct InputParams{ > > - int height, width, channels; > > -} InputParams; > > - > > -typedef struct DepthToSpaceParams{ > > - int block_size; > > -} DepthToSpaceParams; > > - > > -// Represents simple feed-forward convolutional network. > > -typedef struct ConvolutionalNetwork{ > > - Layer *layers; > > - int32_t layers_num; > > -} ConvolutionalNetwork; > > - > > -DNNModel *ff_dnn_load_model_native(const char *model_filename); > > - > > -DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output); > > - > > -void ff_dnn_free_model_native(DNNModel **model); > > - > > -#endif > > diff --git a/libavfilter/dnn_backend_tf.c b/libavfilter/dnn_backend_tf.c > > deleted file mode 100644 > > index ba959ae..0000000 > > --- a/libavfilter/dnn_backend_tf.c > > +++ /dev/null > > @@ -1,603 +0,0 @@ > > -/* > > - * Copyright (c) 2018 Sergey Lavrushkin > > - * > > - * This file is part of FFmpeg. > > - * > > - * FFmpeg is free software; you can redistribute it and/or > > - * modify it under the terms of the GNU Lesser General Public > > - * License as published by the Free Software Foundation; either > > - * version 2.1 of the License, or (at your option) any later version. > > - * > > - * FFmpeg is distributed in the hope that it will be useful, > > - * but WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * Lesser General Public License for more details. > > - * > > - * You should have received a copy of the GNU Lesser General Public > > - * License along with FFmpeg; if not, write to the Free Software > > - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > > - */ > > - > > -/** > > - * @file > > - * DNN tensorflow backend implementation. > > - */ > > - > > -#include "dnn_backend_tf.h" > > -#include "dnn_backend_native.h" > > -#include "libavformat/avio.h" > > -#include "libavutil/avassert.h" > > - > > -#include <tensorflow/c/c_api.h> > > - > > -typedef struct TFModel{ > > - TF_Graph *graph; > > - TF_Session *session; > > - TF_Status *status; > > - TF_Output input; > > - TF_Tensor *input_tensor; > > - TF_Output *outputs; > > - TF_Tensor **output_tensors; > > - uint32_t nb_output; > > -} TFModel; > > - > > -static void free_buffer(void *data, size_t length) > > -{ > > - av_freep(&data); > > -} > > - > > -static TF_Buffer *read_graph(const char *model_filename) > > -{ > > - TF_Buffer *graph_buf; > > - unsigned char *graph_data = NULL; > > - AVIOContext *model_file_context; > > - long size, bytes_read; > > - > > - if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){ > > - return NULL; > > - } > > - > > - size = avio_size(model_file_context); > > - > > - graph_data = av_malloc(size); > > - if (!graph_data){ > > - avio_closep(&model_file_context); > > - return NULL; > > - } > > - bytes_read = avio_read(model_file_context, graph_data, size); > > - avio_closep(&model_file_context); > > - if (bytes_read != size){ > > - av_freep(&graph_data); > > - return NULL; > > - } > > - > > - graph_buf = TF_NewBuffer(); > > - graph_buf->data = (void *)graph_data; > > - graph_buf->length = size; > > - graph_buf->data_deallocator = free_buffer; > > - > > - return graph_buf; > > -} > > - > > -static TF_Tensor *allocate_input_tensor(const DNNInputData *input) > > -{ > > - TF_DataType dt; > > - size_t size; > > - int64_t input_dims[] = {1, input->height, input->width, input->channels}; > > - switch (input->dt) { > > - case DNN_FLOAT: > > - dt = TF_FLOAT; > > - size = sizeof(float); > > - break; > > - case DNN_UINT8: > > - dt = TF_UINT8; > > - size = sizeof(char); > > - break; > > - default: > > - av_assert0(!"should not reach here"); > > - } > > - > > - return TF_AllocateTensor(dt, input_dims, 4, > > - input_dims[1] * input_dims[2] * input_dims[3] * size); > > -} > > - > > -static DNNReturnType set_input_output_tf(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output) > > -{ > > - TFModel *tf_model = (TFModel *)model; > > - TF_SessionOptions *sess_opts; > > - const TF_Operation *init_op = TF_GraphOperationByName(tf_model->graph, "init"); > > - > > - // Input operation > > - tf_model->input.oper = TF_GraphOperationByName(tf_model->graph, input_name); > > - if (!tf_model->input.oper){ > > - return DNN_ERROR; > > - } > > - tf_model->input.index = 0; > > - if (tf_model->input_tensor){ > > - TF_DeleteTensor(tf_model->input_tensor); > > - } > > - tf_model->input_tensor = allocate_input_tensor(input); > > - if (!tf_model->input_tensor){ > > - return DNN_ERROR; > > - } > > - input->data = (float *)TF_TensorData(tf_model->input_tensor); > > - > > - // Output operation > > - if (nb_output == 0) > > - return DNN_ERROR; > > - > > - av_freep(&tf_model->outputs); > > - tf_model->outputs = av_malloc_array(nb_output, sizeof(*tf_model->outputs)); > > - if (!tf_model->outputs) > > - return DNN_ERROR; > > - for (int i = 0; i < nb_output; ++i) { > > - tf_model->outputs[i].oper = TF_GraphOperationByName(tf_model->graph, output_names[i]); > > - if (!tf_model->outputs[i].oper){ > > - av_freep(&tf_model->outputs); > > - return DNN_ERROR; > > - } > > - tf_model->outputs[i].index = 0; > > - } > > - > > - if (tf_model->output_tensors) { > > - for (uint32_t i = 0; i < tf_model->nb_output; ++i) { > > - if (tf_model->output_tensors[i]) { > > - TF_DeleteTensor(tf_model->output_tensors[i]); > > - tf_model->output_tensors[i] = NULL; > > - } > > - } > > - } > > - av_freep(&tf_model->output_tensors); > > - tf_model->output_tensors = av_mallocz_array(nb_output, sizeof(*tf_model->output_tensors)); > > - if (!tf_model->output_tensors) { > > - av_freep(&tf_model->outputs); > > - return DNN_ERROR; > > - } > > - > > - tf_model->nb_output = nb_output; > > - > > - if (tf_model->session){ > > - TF_CloseSession(tf_model->session, tf_model->status); > > - TF_DeleteSession(tf_model->session, tf_model->status); > > - } > > - > > - sess_opts = TF_NewSessionOptions(); > > - tf_model->session = TF_NewSession(tf_model->graph, sess_opts, tf_model->status); > > - TF_DeleteSessionOptions(sess_opts); > > - if (TF_GetCode(tf_model->status) != TF_OK) > > - { > > - return DNN_ERROR; > > - } > > - > > - // Run initialization operation with name "init" if it is present in graph > > - if (init_op){ > > - TF_SessionRun(tf_model->session, NULL, > > - NULL, NULL, 0, > > - NULL, NULL, 0, > > - &init_op, 1, NULL, tf_model->status); > > - if (TF_GetCode(tf_model->status) != TF_OK) > > - { > > - return DNN_ERROR; > > - } > > - } > > - > > - return DNN_SUCCESS; > > -} > > - > > -static DNNReturnType load_tf_model(TFModel *tf_model, const char *model_filename) > > -{ > > - TF_Buffer *graph_def; > > - TF_ImportGraphDefOptions *graph_opts; > > - > > - graph_def = read_graph(model_filename); > > - if (!graph_def){ > > - return DNN_ERROR; > > - } > > - tf_model->graph = TF_NewGraph(); > > - tf_model->status = TF_NewStatus(); > > - graph_opts = TF_NewImportGraphDefOptions(); > > - TF_GraphImportGraphDef(tf_model->graph, graph_def, graph_opts, tf_model->status); > > - TF_DeleteImportGraphDefOptions(graph_opts); > > - TF_DeleteBuffer(graph_def); > > - if (TF_GetCode(tf_model->status) != TF_OK){ > > - TF_DeleteGraph(tf_model->graph); > > - TF_DeleteStatus(tf_model->status); > > - return DNN_ERROR; > > - } > > - > > - return DNN_SUCCESS; > > -} > > - > > -#define NAME_BUFFER_SIZE 256 > > - > > -static DNNReturnType add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op, > > - ConvolutionalParams* params, const int layer) > > -{ > > - TF_Operation *op; > > - TF_OperationDescription *op_desc; > > - TF_Output input; > > - int64_t strides[] = {1, 1, 1, 1}; > > - TF_Tensor *tensor; > > - int64_t dims[4]; > > - int dims_len; > > - char name_buffer[NAME_BUFFER_SIZE]; > > - int32_t size; > > - > > - size = params->input_num * params->output_num * params->kernel_size * params->kernel_size; > > - input.index = 0; > > - > > - snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_kernel%d", layer); > > - op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer); > > - TF_SetAttrType(op_desc, "dtype", TF_FLOAT); > > - dims[0] = params->output_num; > > - dims[1] = params->kernel_size; > > - dims[2] = params->kernel_size; > > - dims[3] = params->input_num; > > - dims_len = 4; > > - tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, size * sizeof(float)); > > - memcpy(TF_TensorData(tensor), params->kernel, size * sizeof(float)); > > - TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); > > - if (TF_GetCode(tf_model->status) != TF_OK){ > > - return DNN_ERROR; > > - } > > - op = TF_FinishOperation(op_desc, tf_model->status); > > - if (TF_GetCode(tf_model->status) != TF_OK){ > > - return DNN_ERROR; > > - } > > - > > - snprintf(name_buffer, NAME_BUFFER_SIZE, "transpose%d", layer); > > - op_desc = TF_NewOperation(tf_model->graph, "Transpose", name_buffer); > > - input.oper = op; > > - TF_AddInput(op_desc, input); > > - input.oper = transpose_op; > > - TF_AddInput(op_desc, input); > > - TF_SetAttrType(op_desc, "T", TF_FLOAT); > > - TF_SetAttrType(op_desc, "Tperm", TF_INT32); > > - op = TF_FinishOperation(op_desc, tf_model->status); > > - if (TF_GetCode(tf_model->status) != TF_OK){ > > - return DNN_ERROR; > > - } > > - > > - snprintf(name_buffer, NAME_BUFFER_SIZE, "conv2d%d", layer); > > - op_desc = TF_NewOperation(tf_model->graph, "Conv2D", name_buffer); > > - input.oper = *cur_op; > > - TF_AddInput(op_desc, input); > > - input.oper = op; > > - TF_AddInput(op_desc, input); > > - TF_SetAttrType(op_desc, "T", TF_FLOAT); > > - TF_SetAttrIntList(op_desc, "strides", strides, 4); > > - TF_SetAttrString(op_desc, "padding", "VALID", 5); > > - *cur_op = TF_FinishOperation(op_desc, tf_model->status); > > - if (TF_GetCode(tf_model->status) != TF_OK){ > > - return DNN_ERROR; > > - } > > - > > - snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_biases%d", layer); > > - op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer); > > - TF_SetAttrType(op_desc, "dtype", TF_FLOAT); > > - dims[0] = params->output_num; > > - dims_len = 1; > > - tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, params->output_num * sizeof(float)); > > - memcpy(TF_TensorData(tensor), params->biases, params->output_num * sizeof(float)); > > - TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); > > - if (TF_GetCode(tf_model->status) != TF_OK){ > > - return DNN_ERROR; > > - } > > - op = TF_FinishOperation(op_desc, tf_model->status); > > - if (TF_GetCode(tf_model->status) != TF_OK){ > > - return DNN_ERROR; > > - } > > - > > - snprintf(name_buffer, NAME_BUFFER_SIZE, "bias_add%d", layer); > > - op_desc = TF_NewOperation(tf_model->graph, "BiasAdd", name_buffer); > > - input.oper = *cur_op; > > - TF_AddInput(op_desc, input); > > - input.oper = op; > > - TF_AddInput(op_desc, input); > > - TF_SetAttrType(op_desc, "T", TF_FLOAT); > > - *cur_op = TF_FinishOperation(op_desc, tf_model->status); > > - if (TF_GetCode(tf_model->status) != TF_OK){ > > - return DNN_ERROR; > > - } > > - > > - snprintf(name_buffer, NAME_BUFFER_SIZE, "activation%d", layer); > > - switch (params->activation){ > > - case RELU: > > - op_desc = TF_NewOperation(tf_model->graph, "Relu", name_buffer); > > - break; > > - case TANH: > > - op_desc = TF_NewOperation(tf_model->graph, "Tanh", name_buffer); > > - break; > > - case SIGMOID: > > - op_desc = TF_NewOperation(tf_model->graph, "Sigmoid", name_buffer); > > - break; > > - default: > > - return DNN_ERROR; > > - } > > - input.oper = *cur_op; > > - TF_AddInput(op_desc, input); > > - TF_SetAttrType(op_desc, "T", TF_FLOAT); > > - *cur_op = TF_FinishOperation(op_desc, tf_model->status); > > - if (TF_GetCode(tf_model->status) != TF_OK){ > > - return DNN_ERROR; > > - } > > - > > - return DNN_SUCCESS; > > -} > > - > > -static DNNReturnType add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op, > > - DepthToSpaceParams *params, const int layer) > > -{ > > - TF_OperationDescription *op_desc; > > - TF_Output input; > > - char name_buffer[NAME_BUFFER_SIZE]; > > - > > - snprintf(name_buffer, NAME_BUFFER_SIZE, "depth_to_space%d", layer); > > - op_desc = TF_NewOperation(tf_model->graph, "DepthToSpace", name_buffer); > > - input.oper = *cur_op; > > - input.index = 0; > > - TF_AddInput(op_desc, input); > > - TF_SetAttrType(op_desc, "T", TF_FLOAT); > > - TF_SetAttrInt(op_desc, "block_size", params->block_size); > > - *cur_op = TF_FinishOperation(op_desc, tf_model->status); > > - if (TF_GetCode(tf_model->status) != TF_OK){ > > - return DNN_ERROR; > > - } > > - > > - return DNN_SUCCESS; > > -} > > - > > -static int calculate_pad(const ConvolutionalNetwork *conv_network) > > -{ > > - ConvolutionalParams *params; > > - int32_t layer; > > - int pad = 0; > > - > > - for (layer = 0; layer < conv_network->layers_num; ++layer){ > > - if (conv_network->layers[layer].type == CONV){ > > - params = (ConvolutionalParams *)conv_network->layers[layer].params; > > - pad += params->kernel_size >> 1; > > - } > > - } > > - > > - return pad; > > -} > > - > > -static DNNReturnType add_pad_op(TFModel *tf_model, TF_Operation **cur_op, const int32_t pad) > > -{ > > - TF_Operation *op; > > - TF_Tensor *tensor; > > - TF_OperationDescription *op_desc; > > - TF_Output input; > > - int32_t *pads; > > - int64_t pads_shape[] = {4, 2}; > > - > > - input.index = 0; > > - > > - op_desc = TF_NewOperation(tf_model->graph, "Const", "pads"); > > - TF_SetAttrType(op_desc, "dtype", TF_INT32); > > - tensor = TF_AllocateTensor(TF_INT32, pads_shape, 2, 4 * 2 * sizeof(int32_t)); > > - pads = (int32_t *)TF_TensorData(tensor); > > - pads[0] = 0; pads[1] = 0; > > - pads[2] = pad; pads[3] = pad; > > - pads[4] = pad; pads[5] = pad; > > - pads[6] = 0; pads[7] = 0; > > - TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); > > - if (TF_GetCode(tf_model->status) != TF_OK){ > > - return DNN_ERROR; > > - } > > - op = TF_FinishOperation(op_desc, tf_model->status); > > - if (TF_GetCode(tf_model->status) != TF_OK){ > > - return DNN_ERROR; > > - } > > - > > - op_desc = TF_NewOperation(tf_model->graph, "MirrorPad", "mirror_pad"); > > - input.oper = *cur_op; > > - TF_AddInput(op_desc, input); > > - input.oper = op; > > - TF_AddInput(op_desc, input); > > - TF_SetAttrType(op_desc, "T", TF_FLOAT); > > - TF_SetAttrType(op_desc, "Tpaddings", TF_INT32); > > - TF_SetAttrString(op_desc, "mode", "SYMMETRIC", 9); > > - *cur_op = TF_FinishOperation(op_desc, tf_model->status); > > - if (TF_GetCode(tf_model->status) != TF_OK){ > > - return DNN_ERROR; > > - } > > - > > - return DNN_SUCCESS; > > -} > > - > > -static DNNReturnType load_native_model(TFModel *tf_model, const char *model_filename) > > -{ > > - int32_t layer; > > - TF_OperationDescription *op_desc; > > - TF_Operation *op; > > - TF_Operation *transpose_op; > > - TF_Tensor *tensor; > > - TF_Output input; > > - int32_t *transpose_perm; > > - int64_t transpose_perm_shape[] = {4}; > > - int64_t input_shape[] = {1, -1, -1, -1}; > > - int32_t pad; > > - DNNReturnType layer_add_res; > > - DNNModel *native_model = NULL; > > - ConvolutionalNetwork *conv_network; > > - > > - native_model = ff_dnn_load_model_native(model_filename); > > - if (!native_model){ > > - return DNN_ERROR; > > - } > > - > > - conv_network = (ConvolutionalNetwork *)native_model->model; > > - pad = calculate_pad(conv_network); > > - tf_model->graph = TF_NewGraph(); > > - tf_model->status = TF_NewStatus(); > > - > > -#define CLEANUP_ON_ERROR(tf_model) \ > > - { \ > > - TF_DeleteGraph(tf_model->graph); \ > > - TF_DeleteStatus(tf_model->status); \ > > - return DNN_ERROR; \ > > - } > > - > > - op_desc = TF_NewOperation(tf_model->graph, "Placeholder", "x"); > > - TF_SetAttrType(op_desc, "dtype", TF_FLOAT); > > - TF_SetAttrShape(op_desc, "shape", input_shape, 4); > > - op = TF_FinishOperation(op_desc, tf_model->status); > > - if (TF_GetCode(tf_model->status) != TF_OK){ > > - CLEANUP_ON_ERROR(tf_model); > > - } > > - > > - if (add_pad_op(tf_model, &op, pad) != DNN_SUCCESS){ > > - CLEANUP_ON_ERROR(tf_model); > > - } > > - > > - op_desc = TF_NewOperation(tf_model->graph, "Const", "transpose_perm"); > > - TF_SetAttrType(op_desc, "dtype", TF_INT32); > > - tensor = TF_AllocateTensor(TF_INT32, transpose_perm_shape, 1, 4 * sizeof(int32_t)); > > - transpose_perm = (int32_t *)TF_TensorData(tensor); > > - transpose_perm[0] = 1; > > - transpose_perm[1] = 2; > > - transpose_perm[2] = 3; > > - transpose_perm[3] = 0; > > - TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); > > - if (TF_GetCode(tf_model->status) != TF_OK){ > > - CLEANUP_ON_ERROR(tf_model); > > - } > > - transpose_op = TF_FinishOperation(op_desc, tf_model->status); > > - > > - for (layer = 0; layer < conv_network->layers_num; ++layer){ > > - switch (conv_network->layers[layer].type){ > > - case INPUT: > > - layer_add_res = DNN_SUCCESS; > > - break; > > - case CONV: > > - layer_add_res = add_conv_layer(tf_model, transpose_op, &op, > > - (ConvolutionalParams *)conv_network->layers[layer].params, layer); > > - break; > > - case DEPTH_TO_SPACE: > > - layer_add_res = add_depth_to_space_layer(tf_model, &op, > > - (DepthToSpaceParams *)conv_network->layers[layer].params, layer); > > - break; > > - default: > > - CLEANUP_ON_ERROR(tf_model); > > - } > > - > > - if (layer_add_res != DNN_SUCCESS){ > > - CLEANUP_ON_ERROR(tf_model); > > - } > > - } > > - > > - op_desc = TF_NewOperation(tf_model->graph, "Identity", "y"); > > - input.oper = op; > > - TF_AddInput(op_desc, input); > > - TF_FinishOperation(op_desc, tf_model->status); > > - if (TF_GetCode(tf_model->status) != TF_OK){ > > - CLEANUP_ON_ERROR(tf_model); > > - } > > - > > - ff_dnn_free_model_native(&native_model); > > - > > - return DNN_SUCCESS; > > -} > > - > > -DNNModel *ff_dnn_load_model_tf(const char *model_filename) > > -{ > > - DNNModel *model = NULL; > > - TFModel *tf_model = NULL; > > - > > - model = av_malloc(sizeof(DNNModel)); > > - if (!model){ > > - return NULL; > > - } > > - > > - tf_model = av_mallocz(sizeof(TFModel)); > > - if (!tf_model){ > > - av_freep(&model); > > - return NULL; > > - } > > - > > - if (load_tf_model(tf_model, model_filename) != DNN_SUCCESS){ > > - if (load_native_model(tf_model, model_filename) != DNN_SUCCESS){ > > - av_freep(&tf_model); > > - av_freep(&model); > > - > > - return NULL; > > - } > > - } > > - > > - model->model = (void *)tf_model; > > - model->set_input_output = &set_input_output_tf; > > - > > - return model; > > -} > > - > > - > > - > > -DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, uint32_t nb_output) > > -{ > > - TFModel *tf_model = (TFModel *)model->model; > > - uint32_t nb = FFMIN(nb_output, tf_model->nb_output); > > - if (nb == 0) > > - return DNN_ERROR; > > - > > - av_assert0(tf_model->output_tensors); > > - for (uint32_t i = 0; i < tf_model->nb_output; ++i) { > > - if (tf_model->output_tensors[i]) { > > - TF_DeleteTensor(tf_model->output_tensors[i]); > > - tf_model->output_tensors[i] = NULL; > > - } > > - } > > - > > - TF_SessionRun(tf_model->session, NULL, > > - &tf_model->input, &tf_model->input_tensor, 1, > > - tf_model->outputs, tf_model->output_tensors, nb, > > - NULL, 0, NULL, tf_model->status); > > - > > - if (TF_GetCode(tf_model->status) != TF_OK){ > > - return DNN_ERROR; > > - } > > - > > - for (uint32_t i = 0; i < nb; ++i) { > > - outputs[i].height = TF_Dim(tf_model->output_tensors[i], 1); > > - outputs[i].width = TF_Dim(tf_model->output_tensors[i], 2); > > - outputs[i].channels = TF_Dim(tf_model->output_tensors[i], 3); > > - outputs[i].data = TF_TensorData(tf_model->output_tensors[i]); > > - } > > - > > - return DNN_SUCCESS; > > -} > > - > > -void ff_dnn_free_model_tf(DNNModel **model) > > -{ > > - TFModel *tf_model; > > - > > - if (*model){ > > - tf_model = (TFModel *)(*model)->model; > > - if (tf_model->graph){ > > - TF_DeleteGraph(tf_model->graph); > > - } > > - if (tf_model->session){ > > - TF_CloseSession(tf_model->session, tf_model->status); > > - TF_DeleteSession(tf_model->session, tf_model->status); > > - } > > - if (tf_model->status){ > > - TF_DeleteStatus(tf_model->status); > > - } > > - if (tf_model->input_tensor){ > > - TF_DeleteTensor(tf_model->input_tensor); > > - } > > - if (tf_model->output_tensors) { > > - for (uint32_t i = 0; i < tf_model->nb_output; ++i) { > > - if (tf_model->output_tensors[i]) { > > - TF_DeleteTensor(tf_model->output_tensors[i]); > > - tf_model->output_tensors[i] = NULL; > > - } > > - } > > - } > > - av_freep(&tf_model->outputs); > > - av_freep(&tf_model->output_tensors); > > - av_freep(&tf_model); > > - av_freep(model); > > - } > > -} > > diff --git a/libavfilter/dnn_backend_tf.h b/libavfilter/dnn_backend_tf.h > > deleted file mode 100644 > > index 07877b1..0000000 > > --- a/libavfilter/dnn_backend_tf.h > > +++ /dev/null > > @@ -1,38 +0,0 @@ > > -/* > > - * Copyright (c) 2018 Sergey Lavrushkin > > - * > > - * This file is part of FFmpeg. > > - * > > - * FFmpeg is free software; you can redistribute it and/or > > - * modify it under the terms of the GNU Lesser General Public > > - * License as published by the Free Software Foundation; either > > - * version 2.1 of the License, or (at your option) any later version. > > - * > > - * FFmpeg is distributed in the hope that it will be useful, > > - * but WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * Lesser General Public License for more details. > > - * > > - * You should have received a copy of the GNU Lesser General Public > > - * License along with FFmpeg; if not, write to the Free Software > > - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > > - */ > > - > > -/** > > - * @file > > - * DNN inference functions interface for TensorFlow backend. > > - */ > > - > > - > > -#ifndef AVFILTER_DNN_BACKEND_TF_H > > -#define AVFILTER_DNN_BACKEND_TF_H > > - > > -#include "dnn_interface.h" > > - > > -DNNModel *ff_dnn_load_model_tf(const char *model_filename); > > - > > -DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, uint32_t nb_output); > > - > > -void ff_dnn_free_model_tf(DNNModel **model); > > - > > -#endif > > diff --git a/libavfilter/dnn_interface.c b/libavfilter/dnn_interface.c > > deleted file mode 100644 > > index 86fc283..0000000 > > --- a/libavfilter/dnn_interface.c > > +++ /dev/null > > @@ -1,63 +0,0 @@ > > -/* > > - * Copyright (c) 2018 Sergey Lavrushkin > > - * > > - * This file is part of FFmpeg. > > - * > > - * FFmpeg is free software; you can redistribute it and/or > > - * modify it under the terms of the GNU Lesser General Public > > - * License as published by the Free Software Foundation; either > > - * version 2.1 of the License, or (at your option) any later version. > > - * > > - * FFmpeg is distributed in the hope that it will be useful, > > - * but WITHOUT ANY WARRANTY; without even the implied warranty of > > - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > - * Lesser General Public License for more details. > > - * > > - * You should have received a copy of the GNU Lesser General Public > > - * License along with FFmpeg; if not, write to the Free Software > > - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA > > - */ > > - > > -/** > > - * @file > > - * Implements DNN module initialization with specified backend. > > - */ > > - > > -#include "dnn_interface.h" > > -#include "dnn_backend_native.h" > > -#include "dnn_backend_tf.h" > > -#include "libavutil/mem.h" > > - > > -DNNModule *ff_get_dnn_module(DNNBackendType backend_type) > > -{ > > - DNNModule *dnn_module; > > - > > - dnn_module = av_malloc(sizeof(DNNModule)); > > - if(!dnn_module){ > > - return NULL; > > - } > > - > > - switch(backend_type){ > > - case DNN_NATIVE: > > - dnn_module->load_model = &ff_dnn_load_model_native; > > - dnn_module->execute_model = &ff_dnn_execute_model_native; > > - dnn_module->free_model = &ff_dnn_free_model_native; > > - break; > > - case DNN_TF: > > - #if (CONFIG_LIBTENSORFLOW == 1) > > - dnn_module->load_model = &ff_dnn_load_model_tf; > > - dnn_module->execute_model = &ff_dnn_execute_model_tf; > > - dnn_module->free_model = &ff_dnn_free_model_tf; > > - #else > > - av_freep(&dnn_module); > > - return NULL; > > - #endif > > - break; > > - default: > > - av_log(NULL, AV_LOG_ERROR, "Module backend_type is not native or tensorflow\n"); > > - av_freep(&dnn_module); > > - return NULL; > > - } > > - > > - return dnn_module; > > -} > > -- > > 2.7.4 > > > > _______________________________________________ > > ffmpeg-devel mailing list > > ffmpeg-devel@ffmpeg.org > > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > > > To unsubscribe, visit link above, or email > > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
> -----Original Message----- > From: Pedro Arthur [mailto:bygrandao@gmail.com] > Sent: Saturday, July 27, 2019 12:10 AM > To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org> > Cc: Guo, Yejun <yejun.guo@intel.com> > Subject: Re: [FFmpeg-devel] [PATCH 1/4] libavfilter/dnn: move dnn files from > libavfilter to libavfilter/dnn > > Em sex, 26 de jul de 2019 às 13:02, Pedro Arthur <bygrandao@gmail.com> > escreveu: > > > > Hi, > > It fails fate source guard header tests, > > The headers should be changed from AVFILTER_DNN_BACKEND_xxx to > > AVFILTER_DNN_DNN_BACKEND_xxx. > Changed locally and pushed. thanks!
diff --git a/libavfilter/Makefile b/libavfilter/Makefile index 455c809..450d781 100644 --- a/libavfilter/Makefile +++ b/libavfilter/Makefile @@ -26,9 +26,8 @@ OBJS-$(HAVE_THREADS) += pthread.o # subsystems OBJS-$(CONFIG_QSVVPP) += qsvvpp.o -DNN-OBJS-$(CONFIG_LIBTENSORFLOW) += dnn_backend_tf.o -OBJS-$(CONFIG_DNN) += dnn_interface.o dnn_backend_native.o $(DNN-OBJS-yes) OBJS-$(CONFIG_SCENE_SAD) += scene_sad.o +include $(SRC_PATH)/libavfilter/dnn/Makefile # audio filters OBJS-$(CONFIG_ABENCH_FILTER) += f_bench.o diff --git a/libavfilter/dnn/Makefile b/libavfilter/dnn/Makefile new file mode 100644 index 0000000..1d12ade --- /dev/null +++ b/libavfilter/dnn/Makefile @@ -0,0 +1,6 @@ +OBJS-$(CONFIG_DNN) += dnn/dnn_interface.o +OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native.o + +DNN-OBJS-$(CONFIG_LIBTENSORFLOW) += dnn/dnn_backend_tf.o + +OBJS-$(CONFIG_DNN) += $(DNN-OBJS-yes) diff --git a/libavfilter/dnn/dnn_backend_native.c b/libavfilter/dnn/dnn_backend_native.c new file mode 100644 index 0000000..82e900b --- /dev/null +++ b/libavfilter/dnn/dnn_backend_native.c @@ -0,0 +1,389 @@ +/* + * Copyright (c) 2018 Sergey Lavrushkin + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * DNN native backend implementation. + */ + +#include "dnn_backend_native.h" +#include "libavutil/avassert.h" + +static DNNReturnType set_input_output_native(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output) +{ + ConvolutionalNetwork *network = (ConvolutionalNetwork *)model; + InputParams *input_params; + ConvolutionalParams *conv_params; + DepthToSpaceParams *depth_to_space_params; + int cur_width, cur_height, cur_channels; + int32_t layer; + + if (network->layers_num <= 0 || network->layers[0].type != INPUT){ + return DNN_ERROR; + } + else{ + input_params = (InputParams *)network->layers[0].params; + input_params->width = cur_width = input->width; + input_params->height = cur_height = input->height; + input_params->channels = cur_channels = input->channels; + if (input->data){ + av_freep(&input->data); + } + av_assert0(input->dt == DNN_FLOAT); + network->layers[0].output = input->data = av_malloc(cur_height * cur_width * cur_channels * sizeof(float)); + if (!network->layers[0].output){ + return DNN_ERROR; + } + } + + for (layer = 1; layer < network->layers_num; ++layer){ + switch (network->layers[layer].type){ + case CONV: + conv_params = (ConvolutionalParams *)network->layers[layer].params; + if (conv_params->input_num != cur_channels){ + return DNN_ERROR; + } + cur_channels = conv_params->output_num; + + if (conv_params->padding_method == VALID) { + int pad_size = (conv_params->kernel_size - 1) * conv_params->dilation; + cur_height -= pad_size; + cur_width -= pad_size; + } + break; + case DEPTH_TO_SPACE: + depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params; + if (cur_channels % (depth_to_space_params->block_size * depth_to_space_params->block_size) != 0){ + return DNN_ERROR; + } + cur_channels = cur_channels / (depth_to_space_params->block_size * depth_to_space_params->block_size); + cur_height *= depth_to_space_params->block_size; + cur_width *= depth_to_space_params->block_size; + break; + default: + return DNN_ERROR; + } + if (network->layers[layer].output){ + av_freep(&network->layers[layer].output); + } + + if (cur_height <= 0 || cur_width <= 0) + return DNN_ERROR; + + network->layers[layer].output = av_malloc(cur_height * cur_width * cur_channels * sizeof(float)); + if (!network->layers[layer].output){ + return DNN_ERROR; + } + } + + return DNN_SUCCESS; +} + +// Loads model and its parameters that are stored in a binary file with following structure: +// layers_num,layer_type,layer_parameterss,layer_type,layer_parameters... +// For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases +// For DEPTH_TO_SPACE layer: block_size +DNNModel *ff_dnn_load_model_native(const char *model_filename) +{ + DNNModel *model = NULL; + ConvolutionalNetwork *network = NULL; + AVIOContext *model_file_context; + int file_size, dnn_size, kernel_size, i; + int32_t layer; + DNNLayerType layer_type; + ConvolutionalParams *conv_params; + DepthToSpaceParams *depth_to_space_params; + + model = av_malloc(sizeof(DNNModel)); + if (!model){ + return NULL; + } + + if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){ + av_freep(&model); + return NULL; + } + file_size = avio_size(model_file_context); + + network = av_malloc(sizeof(ConvolutionalNetwork)); + if (!network){ + avio_closep(&model_file_context); + av_freep(&model); + return NULL; + } + model->model = (void *)network; + + network->layers_num = 1 + (int32_t)avio_rl32(model_file_context); + dnn_size = 4; + + network->layers = av_malloc(network->layers_num * sizeof(Layer)); + if (!network->layers){ + av_freep(&network); + avio_closep(&model_file_context); + av_freep(&model); + return NULL; + } + + for (layer = 0; layer < network->layers_num; ++layer){ + network->layers[layer].output = NULL; + network->layers[layer].params = NULL; + } + network->layers[0].type = INPUT; + network->layers[0].params = av_malloc(sizeof(InputParams)); + if (!network->layers[0].params){ + avio_closep(&model_file_context); + ff_dnn_free_model_native(&model); + return NULL; + } + + for (layer = 1; layer < network->layers_num; ++layer){ + layer_type = (int32_t)avio_rl32(model_file_context); + dnn_size += 4; + switch (layer_type){ + case CONV: + conv_params = av_malloc(sizeof(ConvolutionalParams)); + if (!conv_params){ + avio_closep(&model_file_context); + ff_dnn_free_model_native(&model); + return NULL; + } + conv_params->dilation = (int32_t)avio_rl32(model_file_context); + conv_params->padding_method = (int32_t)avio_rl32(model_file_context); + conv_params->activation = (int32_t)avio_rl32(model_file_context); + conv_params->input_num = (int32_t)avio_rl32(model_file_context); + conv_params->output_num = (int32_t)avio_rl32(model_file_context); + conv_params->kernel_size = (int32_t)avio_rl32(model_file_context); + kernel_size = conv_params->input_num * conv_params->output_num * + conv_params->kernel_size * conv_params->kernel_size; + dnn_size += 24 + (kernel_size + conv_params->output_num << 2); + if (dnn_size > file_size || conv_params->input_num <= 0 || + conv_params->output_num <= 0 || conv_params->kernel_size <= 0){ + avio_closep(&model_file_context); + ff_dnn_free_model_native(&model); + return NULL; + } + conv_params->kernel = av_malloc(kernel_size * sizeof(float)); + conv_params->biases = av_malloc(conv_params->output_num * sizeof(float)); + if (!conv_params->kernel || !conv_params->biases){ + avio_closep(&model_file_context); + ff_dnn_free_model_native(&model); + return NULL; + } + for (i = 0; i < kernel_size; ++i){ + conv_params->kernel[i] = av_int2float(avio_rl32(model_file_context)); + } + for (i = 0; i < conv_params->output_num; ++i){ + conv_params->biases[i] = av_int2float(avio_rl32(model_file_context)); + } + network->layers[layer].type = CONV; + network->layers[layer].params = conv_params; + break; + case DEPTH_TO_SPACE: + depth_to_space_params = av_malloc(sizeof(DepthToSpaceParams)); + if (!depth_to_space_params){ + avio_closep(&model_file_context); + ff_dnn_free_model_native(&model); + return NULL; + } + depth_to_space_params->block_size = (int32_t)avio_rl32(model_file_context); + dnn_size += 4; + network->layers[layer].type = DEPTH_TO_SPACE; + network->layers[layer].params = depth_to_space_params; + break; + default: + avio_closep(&model_file_context); + ff_dnn_free_model_native(&model); + return NULL; + } + } + + avio_closep(&model_file_context); + + if (dnn_size != file_size){ + ff_dnn_free_model_native(&model); + return NULL; + } + + model->set_input_output = &set_input_output_native; + + return model; +} + +#define CLAMP_TO_EDGE(x, w) ((x) < 0 ? 0 : ((x) >= (w) ? (w - 1) : (x))) + +static void convolve(const float *input, float *output, const ConvolutionalParams *conv_params, int width, int height) +{ + int radius = conv_params->kernel_size >> 1; + int src_linesize = width * conv_params->input_num; + int filter_linesize = conv_params->kernel_size * conv_params->input_num; + int filter_size = conv_params->kernel_size * filter_linesize; + int pad_size = (conv_params->padding_method == VALID) ? (conv_params->kernel_size - 1) / 2 * conv_params->dilation : 0; + + for (int y = pad_size; y < height - pad_size; ++y) { + for (int x = pad_size; x < width - pad_size; ++x) { + for (int n_filter = 0; n_filter < conv_params->output_num; ++n_filter) { + output[n_filter] = conv_params->biases[n_filter]; + + for (int ch = 0; ch < conv_params->input_num; ++ch) { + for (int kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y) { + for (int kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x) { + float input_pel; + if (conv_params->padding_method == SAME_CLAMP_TO_EDGE) { + int y_pos = CLAMP_TO_EDGE(y + (kernel_y - radius) * conv_params->dilation, height); + int x_pos = CLAMP_TO_EDGE(x + (kernel_x - radius) * conv_params->dilation, width); + input_pel = input[y_pos * src_linesize + x_pos * conv_params->input_num + ch]; + } else { + int y_pos = y + (kernel_y - radius) * conv_params->dilation; + int x_pos = x + (kernel_x - radius) * conv_params->dilation; + input_pel = (x_pos < 0 || x_pos >= width || y_pos < 0 || y_pos >= height) ? 0.0 : + input[y_pos * src_linesize + x_pos * conv_params->input_num + ch]; + } + + + output[n_filter] += input_pel * conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize + + kernel_x * conv_params->input_num + ch]; + } + } + } + switch (conv_params->activation){ + case RELU: + output[n_filter] = FFMAX(output[n_filter], 0.0); + break; + case TANH: + output[n_filter] = 2.0f / (1.0f + exp(-2.0f * output[n_filter])) - 1.0f; + break; + case SIGMOID: + output[n_filter] = 1.0f / (1.0f + exp(-output[n_filter])); + break; + case NONE: + break; + case LEAKY_RELU: + output[n_filter] = FFMAX(output[n_filter], 0.0) + 0.2 * FFMIN(output[n_filter], 0.0); + } + } + output += conv_params->output_num; + } + } +} + +static void depth_to_space(const float *input, float *output, int block_size, int width, int height, int channels) +{ + int y, x, by, bx, ch; + int new_channels = channels / (block_size * block_size); + int output_linesize = width * channels; + int by_linesize = output_linesize / block_size; + int x_linesize = new_channels * block_size; + + for (y = 0; y < height; ++y){ + for (x = 0; x < width; ++x){ + for (by = 0; by < block_size; ++by){ + for (bx = 0; bx < block_size; ++bx){ + for (ch = 0; ch < new_channels; ++ch){ + output[by * by_linesize + x * x_linesize + bx * new_channels + ch] = input[ch]; + } + input += new_channels; + } + } + } + output += output_linesize; + } +} + +DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output) +{ + ConvolutionalNetwork *network = (ConvolutionalNetwork *)model->model; + int cur_width, cur_height, cur_channels; + int32_t layer; + InputParams *input_params; + ConvolutionalParams *conv_params; + DepthToSpaceParams *depth_to_space_params; + + if (network->layers_num <= 0 || network->layers[0].type != INPUT || !network->layers[0].output){ + return DNN_ERROR; + } + else{ + input_params = (InputParams *)network->layers[0].params; + cur_width = input_params->width; + cur_height = input_params->height; + cur_channels = input_params->channels; + } + + for (layer = 1; layer < network->layers_num; ++layer){ + if (!network->layers[layer].output){ + return DNN_ERROR; + } + switch (network->layers[layer].type){ + case CONV: + conv_params = (ConvolutionalParams *)network->layers[layer].params; + convolve(network->layers[layer - 1].output, network->layers[layer].output, conv_params, cur_width, cur_height); + cur_channels = conv_params->output_num; + if (conv_params->padding_method == VALID) { + int pad_size = (conv_params->kernel_size - 1) * conv_params->dilation; + cur_height -= pad_size; + cur_width -= pad_size; + } + break; + case DEPTH_TO_SPACE: + depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params; + depth_to_space(network->layers[layer - 1].output, network->layers[layer].output, + depth_to_space_params->block_size, cur_width, cur_height, cur_channels); + cur_height *= depth_to_space_params->block_size; + cur_width *= depth_to_space_params->block_size; + cur_channels /= depth_to_space_params->block_size * depth_to_space_params->block_size; + break; + case INPUT: + return DNN_ERROR; + } + } + + // native mode does not support multiple outputs yet + if (nb_output > 1) + return DNN_ERROR; + outputs[0].data = network->layers[network->layers_num - 1].output; + outputs[0].height = cur_height; + outputs[0].width = cur_width; + outputs[0].channels = cur_channels; + + return DNN_SUCCESS; +} + +void ff_dnn_free_model_native(DNNModel **model) +{ + ConvolutionalNetwork *network; + ConvolutionalParams *conv_params; + int32_t layer; + + if (*model) + { + network = (ConvolutionalNetwork *)(*model)->model; + for (layer = 0; layer < network->layers_num; ++layer){ + av_freep(&network->layers[layer].output); + if (network->layers[layer].type == CONV){ + conv_params = (ConvolutionalParams *)network->layers[layer].params; + av_freep(&conv_params->kernel); + av_freep(&conv_params->biases); + } + av_freep(&network->layers[layer].params); + } + av_freep(&network->layers); + av_freep(&network); + av_freep(model); + } +} diff --git a/libavfilter/dnn/dnn_backend_native.h b/libavfilter/dnn/dnn_backend_native.h new file mode 100644 index 0000000..532103c --- /dev/null +++ b/libavfilter/dnn/dnn_backend_native.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2018 Sergey Lavrushkin + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * DNN inference functions interface for native backend. + */ + + +#ifndef AVFILTER_DNN_BACKEND_NATIVE_H +#define AVFILTER_DNN_BACKEND_NATIVE_H + +#include "../dnn_interface.h" +#include "libavformat/avio.h" + +typedef enum {INPUT, CONV, DEPTH_TO_SPACE} DNNLayerType; + +typedef enum {RELU, TANH, SIGMOID, NONE, LEAKY_RELU} DNNActivationFunc; + +typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} DNNConvPaddingParam; + +typedef struct Layer{ + DNNLayerType type; + float *output; + void *params; +} Layer; + +typedef struct ConvolutionalParams{ + int32_t input_num, output_num, kernel_size; + DNNActivationFunc activation; + DNNConvPaddingParam padding_method; + int32_t dilation; + float *kernel; + float *biases; +} ConvolutionalParams; + +typedef struct InputParams{ + int height, width, channels; +} InputParams; + +typedef struct DepthToSpaceParams{ + int block_size; +} DepthToSpaceParams; + +// Represents simple feed-forward convolutional network. +typedef struct ConvolutionalNetwork{ + Layer *layers; + int32_t layers_num; +} ConvolutionalNetwork; + +DNNModel *ff_dnn_load_model_native(const char *model_filename); + +DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output); + +void ff_dnn_free_model_native(DNNModel **model); + +#endif diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c new file mode 100644 index 0000000..ba959ae --- /dev/null +++ b/libavfilter/dnn/dnn_backend_tf.c @@ -0,0 +1,603 @@ +/* + * Copyright (c) 2018 Sergey Lavrushkin + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * DNN tensorflow backend implementation. + */ + +#include "dnn_backend_tf.h" +#include "dnn_backend_native.h" +#include "libavformat/avio.h" +#include "libavutil/avassert.h" + +#include <tensorflow/c/c_api.h> + +typedef struct TFModel{ + TF_Graph *graph; + TF_Session *session; + TF_Status *status; + TF_Output input; + TF_Tensor *input_tensor; + TF_Output *outputs; + TF_Tensor **output_tensors; + uint32_t nb_output; +} TFModel; + +static void free_buffer(void *data, size_t length) +{ + av_freep(&data); +} + +static TF_Buffer *read_graph(const char *model_filename) +{ + TF_Buffer *graph_buf; + unsigned char *graph_data = NULL; + AVIOContext *model_file_context; + long size, bytes_read; + + if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){ + return NULL; + } + + size = avio_size(model_file_context); + + graph_data = av_malloc(size); + if (!graph_data){ + avio_closep(&model_file_context); + return NULL; + } + bytes_read = avio_read(model_file_context, graph_data, size); + avio_closep(&model_file_context); + if (bytes_read != size){ + av_freep(&graph_data); + return NULL; + } + + graph_buf = TF_NewBuffer(); + graph_buf->data = (void *)graph_data; + graph_buf->length = size; + graph_buf->data_deallocator = free_buffer; + + return graph_buf; +} + +static TF_Tensor *allocate_input_tensor(const DNNInputData *input) +{ + TF_DataType dt; + size_t size; + int64_t input_dims[] = {1, input->height, input->width, input->channels}; + switch (input->dt) { + case DNN_FLOAT: + dt = TF_FLOAT; + size = sizeof(float); + break; + case DNN_UINT8: + dt = TF_UINT8; + size = sizeof(char); + break; + default: + av_assert0(!"should not reach here"); + } + + return TF_AllocateTensor(dt, input_dims, 4, + input_dims[1] * input_dims[2] * input_dims[3] * size); +} + +static DNNReturnType set_input_output_tf(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output) +{ + TFModel *tf_model = (TFModel *)model; + TF_SessionOptions *sess_opts; + const TF_Operation *init_op = TF_GraphOperationByName(tf_model->graph, "init"); + + // Input operation + tf_model->input.oper = TF_GraphOperationByName(tf_model->graph, input_name); + if (!tf_model->input.oper){ + return DNN_ERROR; + } + tf_model->input.index = 0; + if (tf_model->input_tensor){ + TF_DeleteTensor(tf_model->input_tensor); + } + tf_model->input_tensor = allocate_input_tensor(input); + if (!tf_model->input_tensor){ + return DNN_ERROR; + } + input->data = (float *)TF_TensorData(tf_model->input_tensor); + + // Output operation + if (nb_output == 0) + return DNN_ERROR; + + av_freep(&tf_model->outputs); + tf_model->outputs = av_malloc_array(nb_output, sizeof(*tf_model->outputs)); + if (!tf_model->outputs) + return DNN_ERROR; + for (int i = 0; i < nb_output; ++i) { + tf_model->outputs[i].oper = TF_GraphOperationByName(tf_model->graph, output_names[i]); + if (!tf_model->outputs[i].oper){ + av_freep(&tf_model->outputs); + return DNN_ERROR; + } + tf_model->outputs[i].index = 0; + } + + if (tf_model->output_tensors) { + for (uint32_t i = 0; i < tf_model->nb_output; ++i) { + if (tf_model->output_tensors[i]) { + TF_DeleteTensor(tf_model->output_tensors[i]); + tf_model->output_tensors[i] = NULL; + } + } + } + av_freep(&tf_model->output_tensors); + tf_model->output_tensors = av_mallocz_array(nb_output, sizeof(*tf_model->output_tensors)); + if (!tf_model->output_tensors) { + av_freep(&tf_model->outputs); + return DNN_ERROR; + } + + tf_model->nb_output = nb_output; + + if (tf_model->session){ + TF_CloseSession(tf_model->session, tf_model->status); + TF_DeleteSession(tf_model->session, tf_model->status); + } + + sess_opts = TF_NewSessionOptions(); + tf_model->session = TF_NewSession(tf_model->graph, sess_opts, tf_model->status); + TF_DeleteSessionOptions(sess_opts); + if (TF_GetCode(tf_model->status) != TF_OK) + { + return DNN_ERROR; + } + + // Run initialization operation with name "init" if it is present in graph + if (init_op){ + TF_SessionRun(tf_model->session, NULL, + NULL, NULL, 0, + NULL, NULL, 0, + &init_op, 1, NULL, tf_model->status); + if (TF_GetCode(tf_model->status) != TF_OK) + { + return DNN_ERROR; + } + } + + return DNN_SUCCESS; +} + +static DNNReturnType load_tf_model(TFModel *tf_model, const char *model_filename) +{ + TF_Buffer *graph_def; + TF_ImportGraphDefOptions *graph_opts; + + graph_def = read_graph(model_filename); + if (!graph_def){ + return DNN_ERROR; + } + tf_model->graph = TF_NewGraph(); + tf_model->status = TF_NewStatus(); + graph_opts = TF_NewImportGraphDefOptions(); + TF_GraphImportGraphDef(tf_model->graph, graph_def, graph_opts, tf_model->status); + TF_DeleteImportGraphDefOptions(graph_opts); + TF_DeleteBuffer(graph_def); + if (TF_GetCode(tf_model->status) != TF_OK){ + TF_DeleteGraph(tf_model->graph); + TF_DeleteStatus(tf_model->status); + return DNN_ERROR; + } + + return DNN_SUCCESS; +} + +#define NAME_BUFFER_SIZE 256 + +static DNNReturnType add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op, + ConvolutionalParams* params, const int layer) +{ + TF_Operation *op; + TF_OperationDescription *op_desc; + TF_Output input; + int64_t strides[] = {1, 1, 1, 1}; + TF_Tensor *tensor; + int64_t dims[4]; + int dims_len; + char name_buffer[NAME_BUFFER_SIZE]; + int32_t size; + + size = params->input_num * params->output_num * params->kernel_size * params->kernel_size; + input.index = 0; + + snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_kernel%d", layer); + op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer); + TF_SetAttrType(op_desc, "dtype", TF_FLOAT); + dims[0] = params->output_num; + dims[1] = params->kernel_size; + dims[2] = params->kernel_size; + dims[3] = params->input_num; + dims_len = 4; + tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, size * sizeof(float)); + memcpy(TF_TensorData(tensor), params->kernel, size * sizeof(float)); + TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); + if (TF_GetCode(tf_model->status) != TF_OK){ + return DNN_ERROR; + } + op = TF_FinishOperation(op_desc, tf_model->status); + if (TF_GetCode(tf_model->status) != TF_OK){ + return DNN_ERROR; + } + + snprintf(name_buffer, NAME_BUFFER_SIZE, "transpose%d", layer); + op_desc = TF_NewOperation(tf_model->graph, "Transpose", name_buffer); + input.oper = op; + TF_AddInput(op_desc, input); + input.oper = transpose_op; + TF_AddInput(op_desc, input); + TF_SetAttrType(op_desc, "T", TF_FLOAT); + TF_SetAttrType(op_desc, "Tperm", TF_INT32); + op = TF_FinishOperation(op_desc, tf_model->status); + if (TF_GetCode(tf_model->status) != TF_OK){ + return DNN_ERROR; + } + + snprintf(name_buffer, NAME_BUFFER_SIZE, "conv2d%d", layer); + op_desc = TF_NewOperation(tf_model->graph, "Conv2D", name_buffer); + input.oper = *cur_op; + TF_AddInput(op_desc, input); + input.oper = op; + TF_AddInput(op_desc, input); + TF_SetAttrType(op_desc, "T", TF_FLOAT); + TF_SetAttrIntList(op_desc, "strides", strides, 4); + TF_SetAttrString(op_desc, "padding", "VALID", 5); + *cur_op = TF_FinishOperation(op_desc, tf_model->status); + if (TF_GetCode(tf_model->status) != TF_OK){ + return DNN_ERROR; + } + + snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_biases%d", layer); + op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer); + TF_SetAttrType(op_desc, "dtype", TF_FLOAT); + dims[0] = params->output_num; + dims_len = 1; + tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, params->output_num * sizeof(float)); + memcpy(TF_TensorData(tensor), params->biases, params->output_num * sizeof(float)); + TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); + if (TF_GetCode(tf_model->status) != TF_OK){ + return DNN_ERROR; + } + op = TF_FinishOperation(op_desc, tf_model->status); + if (TF_GetCode(tf_model->status) != TF_OK){ + return DNN_ERROR; + } + + snprintf(name_buffer, NAME_BUFFER_SIZE, "bias_add%d", layer); + op_desc = TF_NewOperation(tf_model->graph, "BiasAdd", name_buffer); + input.oper = *cur_op; + TF_AddInput(op_desc, input); + input.oper = op; + TF_AddInput(op_desc, input); + TF_SetAttrType(op_desc, "T", TF_FLOAT); + *cur_op = TF_FinishOperation(op_desc, tf_model->status); + if (TF_GetCode(tf_model->status) != TF_OK){ + return DNN_ERROR; + } + + snprintf(name_buffer, NAME_BUFFER_SIZE, "activation%d", layer); + switch (params->activation){ + case RELU: + op_desc = TF_NewOperation(tf_model->graph, "Relu", name_buffer); + break; + case TANH: + op_desc = TF_NewOperation(tf_model->graph, "Tanh", name_buffer); + break; + case SIGMOID: + op_desc = TF_NewOperation(tf_model->graph, "Sigmoid", name_buffer); + break; + default: + return DNN_ERROR; + } + input.oper = *cur_op; + TF_AddInput(op_desc, input); + TF_SetAttrType(op_desc, "T", TF_FLOAT); + *cur_op = TF_FinishOperation(op_desc, tf_model->status); + if (TF_GetCode(tf_model->status) != TF_OK){ + return DNN_ERROR; + } + + return DNN_SUCCESS; +} + +static DNNReturnType add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op, + DepthToSpaceParams *params, const int layer) +{ + TF_OperationDescription *op_desc; + TF_Output input; + char name_buffer[NAME_BUFFER_SIZE]; + + snprintf(name_buffer, NAME_BUFFER_SIZE, "depth_to_space%d", layer); + op_desc = TF_NewOperation(tf_model->graph, "DepthToSpace", name_buffer); + input.oper = *cur_op; + input.index = 0; + TF_AddInput(op_desc, input); + TF_SetAttrType(op_desc, "T", TF_FLOAT); + TF_SetAttrInt(op_desc, "block_size", params->block_size); + *cur_op = TF_FinishOperation(op_desc, tf_model->status); + if (TF_GetCode(tf_model->status) != TF_OK){ + return DNN_ERROR; + } + + return DNN_SUCCESS; +} + +static int calculate_pad(const ConvolutionalNetwork *conv_network) +{ + ConvolutionalParams *params; + int32_t layer; + int pad = 0; + + for (layer = 0; layer < conv_network->layers_num; ++layer){ + if (conv_network->layers[layer].type == CONV){ + params = (ConvolutionalParams *)conv_network->layers[layer].params; + pad += params->kernel_size >> 1; + } + } + + return pad; +} + +static DNNReturnType add_pad_op(TFModel *tf_model, TF_Operation **cur_op, const int32_t pad) +{ + TF_Operation *op; + TF_Tensor *tensor; + TF_OperationDescription *op_desc; + TF_Output input; + int32_t *pads; + int64_t pads_shape[] = {4, 2}; + + input.index = 0; + + op_desc = TF_NewOperation(tf_model->graph, "Const", "pads"); + TF_SetAttrType(op_desc, "dtype", TF_INT32); + tensor = TF_AllocateTensor(TF_INT32, pads_shape, 2, 4 * 2 * sizeof(int32_t)); + pads = (int32_t *)TF_TensorData(tensor); + pads[0] = 0; pads[1] = 0; + pads[2] = pad; pads[3] = pad; + pads[4] = pad; pads[5] = pad; + pads[6] = 0; pads[7] = 0; + TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); + if (TF_GetCode(tf_model->status) != TF_OK){ + return DNN_ERROR; + } + op = TF_FinishOperation(op_desc, tf_model->status); + if (TF_GetCode(tf_model->status) != TF_OK){ + return DNN_ERROR; + } + + op_desc = TF_NewOperation(tf_model->graph, "MirrorPad", "mirror_pad"); + input.oper = *cur_op; + TF_AddInput(op_desc, input); + input.oper = op; + TF_AddInput(op_desc, input); + TF_SetAttrType(op_desc, "T", TF_FLOAT); + TF_SetAttrType(op_desc, "Tpaddings", TF_INT32); + TF_SetAttrString(op_desc, "mode", "SYMMETRIC", 9); + *cur_op = TF_FinishOperation(op_desc, tf_model->status); + if (TF_GetCode(tf_model->status) != TF_OK){ + return DNN_ERROR; + } + + return DNN_SUCCESS; +} + +static DNNReturnType load_native_model(TFModel *tf_model, const char *model_filename) +{ + int32_t layer; + TF_OperationDescription *op_desc; + TF_Operation *op; + TF_Operation *transpose_op; + TF_Tensor *tensor; + TF_Output input; + int32_t *transpose_perm; + int64_t transpose_perm_shape[] = {4}; + int64_t input_shape[] = {1, -1, -1, -1}; + int32_t pad; + DNNReturnType layer_add_res; + DNNModel *native_model = NULL; + ConvolutionalNetwork *conv_network; + + native_model = ff_dnn_load_model_native(model_filename); + if (!native_model){ + return DNN_ERROR; + } + + conv_network = (ConvolutionalNetwork *)native_model->model; + pad = calculate_pad(conv_network); + tf_model->graph = TF_NewGraph(); + tf_model->status = TF_NewStatus(); + +#define CLEANUP_ON_ERROR(tf_model) \ + { \ + TF_DeleteGraph(tf_model->graph); \ + TF_DeleteStatus(tf_model->status); \ + return DNN_ERROR; \ + } + + op_desc = TF_NewOperation(tf_model->graph, "Placeholder", "x"); + TF_SetAttrType(op_desc, "dtype", TF_FLOAT); + TF_SetAttrShape(op_desc, "shape", input_shape, 4); + op = TF_FinishOperation(op_desc, tf_model->status); + if (TF_GetCode(tf_model->status) != TF_OK){ + CLEANUP_ON_ERROR(tf_model); + } + + if (add_pad_op(tf_model, &op, pad) != DNN_SUCCESS){ + CLEANUP_ON_ERROR(tf_model); + } + + op_desc = TF_NewOperation(tf_model->graph, "Const", "transpose_perm"); + TF_SetAttrType(op_desc, "dtype", TF_INT32); + tensor = TF_AllocateTensor(TF_INT32, transpose_perm_shape, 1, 4 * sizeof(int32_t)); + transpose_perm = (int32_t *)TF_TensorData(tensor); + transpose_perm[0] = 1; + transpose_perm[1] = 2; + transpose_perm[2] = 3; + transpose_perm[3] = 0; + TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); + if (TF_GetCode(tf_model->status) != TF_OK){ + CLEANUP_ON_ERROR(tf_model); + } + transpose_op = TF_FinishOperation(op_desc, tf_model->status); + + for (layer = 0; layer < conv_network->layers_num; ++layer){ + switch (conv_network->layers[layer].type){ + case INPUT: + layer_add_res = DNN_SUCCESS; + break; + case CONV: + layer_add_res = add_conv_layer(tf_model, transpose_op, &op, + (ConvolutionalParams *)conv_network->layers[layer].params, layer); + break; + case DEPTH_TO_SPACE: + layer_add_res = add_depth_to_space_layer(tf_model, &op, + (DepthToSpaceParams *)conv_network->layers[layer].params, layer); + break; + default: + CLEANUP_ON_ERROR(tf_model); + } + + if (layer_add_res != DNN_SUCCESS){ + CLEANUP_ON_ERROR(tf_model); + } + } + + op_desc = TF_NewOperation(tf_model->graph, "Identity", "y"); + input.oper = op; + TF_AddInput(op_desc, input); + TF_FinishOperation(op_desc, tf_model->status); + if (TF_GetCode(tf_model->status) != TF_OK){ + CLEANUP_ON_ERROR(tf_model); + } + + ff_dnn_free_model_native(&native_model); + + return DNN_SUCCESS; +} + +DNNModel *ff_dnn_load_model_tf(const char *model_filename) +{ + DNNModel *model = NULL; + TFModel *tf_model = NULL; + + model = av_malloc(sizeof(DNNModel)); + if (!model){ + return NULL; + } + + tf_model = av_mallocz(sizeof(TFModel)); + if (!tf_model){ + av_freep(&model); + return NULL; + } + + if (load_tf_model(tf_model, model_filename) != DNN_SUCCESS){ + if (load_native_model(tf_model, model_filename) != DNN_SUCCESS){ + av_freep(&tf_model); + av_freep(&model); + + return NULL; + } + } + + model->model = (void *)tf_model; + model->set_input_output = &set_input_output_tf; + + return model; +} + + + +DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, uint32_t nb_output) +{ + TFModel *tf_model = (TFModel *)model->model; + uint32_t nb = FFMIN(nb_output, tf_model->nb_output); + if (nb == 0) + return DNN_ERROR; + + av_assert0(tf_model->output_tensors); + for (uint32_t i = 0; i < tf_model->nb_output; ++i) { + if (tf_model->output_tensors[i]) { + TF_DeleteTensor(tf_model->output_tensors[i]); + tf_model->output_tensors[i] = NULL; + } + } + + TF_SessionRun(tf_model->session, NULL, + &tf_model->input, &tf_model->input_tensor, 1, + tf_model->outputs, tf_model->output_tensors, nb, + NULL, 0, NULL, tf_model->status); + + if (TF_GetCode(tf_model->status) != TF_OK){ + return DNN_ERROR; + } + + for (uint32_t i = 0; i < nb; ++i) { + outputs[i].height = TF_Dim(tf_model->output_tensors[i], 1); + outputs[i].width = TF_Dim(tf_model->output_tensors[i], 2); + outputs[i].channels = TF_Dim(tf_model->output_tensors[i], 3); + outputs[i].data = TF_TensorData(tf_model->output_tensors[i]); + } + + return DNN_SUCCESS; +} + +void ff_dnn_free_model_tf(DNNModel **model) +{ + TFModel *tf_model; + + if (*model){ + tf_model = (TFModel *)(*model)->model; + if (tf_model->graph){ + TF_DeleteGraph(tf_model->graph); + } + if (tf_model->session){ + TF_CloseSession(tf_model->session, tf_model->status); + TF_DeleteSession(tf_model->session, tf_model->status); + } + if (tf_model->status){ + TF_DeleteStatus(tf_model->status); + } + if (tf_model->input_tensor){ + TF_DeleteTensor(tf_model->input_tensor); + } + if (tf_model->output_tensors) { + for (uint32_t i = 0; i < tf_model->nb_output; ++i) { + if (tf_model->output_tensors[i]) { + TF_DeleteTensor(tf_model->output_tensors[i]); + tf_model->output_tensors[i] = NULL; + } + } + } + av_freep(&tf_model->outputs); + av_freep(&tf_model->output_tensors); + av_freep(&tf_model); + av_freep(model); + } +} diff --git a/libavfilter/dnn/dnn_backend_tf.h b/libavfilter/dnn/dnn_backend_tf.h new file mode 100644 index 0000000..bb1c85f --- /dev/null +++ b/libavfilter/dnn/dnn_backend_tf.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2018 Sergey Lavrushkin + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * DNN inference functions interface for TensorFlow backend. + */ + + +#ifndef AVFILTER_DNN_BACKEND_TF_H +#define AVFILTER_DNN_BACKEND_TF_H + +#include "../dnn_interface.h" + +DNNModel *ff_dnn_load_model_tf(const char *model_filename); + +DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, uint32_t nb_output); + +void ff_dnn_free_model_tf(DNNModel **model); + +#endif diff --git a/libavfilter/dnn/dnn_interface.c b/libavfilter/dnn/dnn_interface.c new file mode 100644 index 0000000..62da55f --- /dev/null +++ b/libavfilter/dnn/dnn_interface.c @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2018 Sergey Lavrushkin + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file + * Implements DNN module initialization with specified backend. + */ + +#include "../dnn_interface.h" +#include "dnn_backend_native.h" +#include "dnn_backend_tf.h" +#include "libavutil/mem.h" + +DNNModule *ff_get_dnn_module(DNNBackendType backend_type) +{ + DNNModule *dnn_module; + + dnn_module = av_malloc(sizeof(DNNModule)); + if(!dnn_module){ + return NULL; + } + + switch(backend_type){ + case DNN_NATIVE: + dnn_module->load_model = &ff_dnn_load_model_native; + dnn_module->execute_model = &ff_dnn_execute_model_native; + dnn_module->free_model = &ff_dnn_free_model_native; + break; + case DNN_TF: + #if (CONFIG_LIBTENSORFLOW == 1) + dnn_module->load_model = &ff_dnn_load_model_tf; + dnn_module->execute_model = &ff_dnn_execute_model_tf; + dnn_module->free_model = &ff_dnn_free_model_tf; + #else + av_freep(&dnn_module); + return NULL; + #endif + break; + default: + av_log(NULL, AV_LOG_ERROR, "Module backend_type is not native or tensorflow\n"); + av_freep(&dnn_module); + return NULL; + } + + return dnn_module; +} diff --git a/libavfilter/dnn_backend_native.c b/libavfilter/dnn_backend_native.c deleted file mode 100644 index 82e900b..0000000 --- a/libavfilter/dnn_backend_native.c +++ /dev/null @@ -1,389 +0,0 @@ -/* - * Copyright (c) 2018 Sergey Lavrushkin - * - * This file is part of FFmpeg. - * - * FFmpeg is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * FFmpeg is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with FFmpeg; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ - -/** - * @file - * DNN native backend implementation. - */ - -#include "dnn_backend_native.h" -#include "libavutil/avassert.h" - -static DNNReturnType set_input_output_native(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output) -{ - ConvolutionalNetwork *network = (ConvolutionalNetwork *)model; - InputParams *input_params; - ConvolutionalParams *conv_params; - DepthToSpaceParams *depth_to_space_params; - int cur_width, cur_height, cur_channels; - int32_t layer; - - if (network->layers_num <= 0 || network->layers[0].type != INPUT){ - return DNN_ERROR; - } - else{ - input_params = (InputParams *)network->layers[0].params; - input_params->width = cur_width = input->width; - input_params->height = cur_height = input->height; - input_params->channels = cur_channels = input->channels; - if (input->data){ - av_freep(&input->data); - } - av_assert0(input->dt == DNN_FLOAT); - network->layers[0].output = input->data = av_malloc(cur_height * cur_width * cur_channels * sizeof(float)); - if (!network->layers[0].output){ - return DNN_ERROR; - } - } - - for (layer = 1; layer < network->layers_num; ++layer){ - switch (network->layers[layer].type){ - case CONV: - conv_params = (ConvolutionalParams *)network->layers[layer].params; - if (conv_params->input_num != cur_channels){ - return DNN_ERROR; - } - cur_channels = conv_params->output_num; - - if (conv_params->padding_method == VALID) { - int pad_size = (conv_params->kernel_size - 1) * conv_params->dilation; - cur_height -= pad_size; - cur_width -= pad_size; - } - break; - case DEPTH_TO_SPACE: - depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params; - if (cur_channels % (depth_to_space_params->block_size * depth_to_space_params->block_size) != 0){ - return DNN_ERROR; - } - cur_channels = cur_channels / (depth_to_space_params->block_size * depth_to_space_params->block_size); - cur_height *= depth_to_space_params->block_size; - cur_width *= depth_to_space_params->block_size; - break; - default: - return DNN_ERROR; - } - if (network->layers[layer].output){ - av_freep(&network->layers[layer].output); - } - - if (cur_height <= 0 || cur_width <= 0) - return DNN_ERROR; - - network->layers[layer].output = av_malloc(cur_height * cur_width * cur_channels * sizeof(float)); - if (!network->layers[layer].output){ - return DNN_ERROR; - } - } - - return DNN_SUCCESS; -} - -// Loads model and its parameters that are stored in a binary file with following structure: -// layers_num,layer_type,layer_parameterss,layer_type,layer_parameters... -// For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases -// For DEPTH_TO_SPACE layer: block_size -DNNModel *ff_dnn_load_model_native(const char *model_filename) -{ - DNNModel *model = NULL; - ConvolutionalNetwork *network = NULL; - AVIOContext *model_file_context; - int file_size, dnn_size, kernel_size, i; - int32_t layer; - DNNLayerType layer_type; - ConvolutionalParams *conv_params; - DepthToSpaceParams *depth_to_space_params; - - model = av_malloc(sizeof(DNNModel)); - if (!model){ - return NULL; - } - - if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){ - av_freep(&model); - return NULL; - } - file_size = avio_size(model_file_context); - - network = av_malloc(sizeof(ConvolutionalNetwork)); - if (!network){ - avio_closep(&model_file_context); - av_freep(&model); - return NULL; - } - model->model = (void *)network; - - network->layers_num = 1 + (int32_t)avio_rl32(model_file_context); - dnn_size = 4; - - network->layers = av_malloc(network->layers_num * sizeof(Layer)); - if (!network->layers){ - av_freep(&network); - avio_closep(&model_file_context); - av_freep(&model); - return NULL; - } - - for (layer = 0; layer < network->layers_num; ++layer){ - network->layers[layer].output = NULL; - network->layers[layer].params = NULL; - } - network->layers[0].type = INPUT; - network->layers[0].params = av_malloc(sizeof(InputParams)); - if (!network->layers[0].params){ - avio_closep(&model_file_context); - ff_dnn_free_model_native(&model); - return NULL; - } - - for (layer = 1; layer < network->layers_num; ++layer){ - layer_type = (int32_t)avio_rl32(model_file_context); - dnn_size += 4; - switch (layer_type){ - case CONV: - conv_params = av_malloc(sizeof(ConvolutionalParams)); - if (!conv_params){ - avio_closep(&model_file_context); - ff_dnn_free_model_native(&model); - return NULL; - } - conv_params->dilation = (int32_t)avio_rl32(model_file_context); - conv_params->padding_method = (int32_t)avio_rl32(model_file_context); - conv_params->activation = (int32_t)avio_rl32(model_file_context); - conv_params->input_num = (int32_t)avio_rl32(model_file_context); - conv_params->output_num = (int32_t)avio_rl32(model_file_context); - conv_params->kernel_size = (int32_t)avio_rl32(model_file_context); - kernel_size = conv_params->input_num * conv_params->output_num * - conv_params->kernel_size * conv_params->kernel_size; - dnn_size += 24 + (kernel_size + conv_params->output_num << 2); - if (dnn_size > file_size || conv_params->input_num <= 0 || - conv_params->output_num <= 0 || conv_params->kernel_size <= 0){ - avio_closep(&model_file_context); - ff_dnn_free_model_native(&model); - return NULL; - } - conv_params->kernel = av_malloc(kernel_size * sizeof(float)); - conv_params->biases = av_malloc(conv_params->output_num * sizeof(float)); - if (!conv_params->kernel || !conv_params->biases){ - avio_closep(&model_file_context); - ff_dnn_free_model_native(&model); - return NULL; - } - for (i = 0; i < kernel_size; ++i){ - conv_params->kernel[i] = av_int2float(avio_rl32(model_file_context)); - } - for (i = 0; i < conv_params->output_num; ++i){ - conv_params->biases[i] = av_int2float(avio_rl32(model_file_context)); - } - network->layers[layer].type = CONV; - network->layers[layer].params = conv_params; - break; - case DEPTH_TO_SPACE: - depth_to_space_params = av_malloc(sizeof(DepthToSpaceParams)); - if (!depth_to_space_params){ - avio_closep(&model_file_context); - ff_dnn_free_model_native(&model); - return NULL; - } - depth_to_space_params->block_size = (int32_t)avio_rl32(model_file_context); - dnn_size += 4; - network->layers[layer].type = DEPTH_TO_SPACE; - network->layers[layer].params = depth_to_space_params; - break; - default: - avio_closep(&model_file_context); - ff_dnn_free_model_native(&model); - return NULL; - } - } - - avio_closep(&model_file_context); - - if (dnn_size != file_size){ - ff_dnn_free_model_native(&model); - return NULL; - } - - model->set_input_output = &set_input_output_native; - - return model; -} - -#define CLAMP_TO_EDGE(x, w) ((x) < 0 ? 0 : ((x) >= (w) ? (w - 1) : (x))) - -static void convolve(const float *input, float *output, const ConvolutionalParams *conv_params, int width, int height) -{ - int radius = conv_params->kernel_size >> 1; - int src_linesize = width * conv_params->input_num; - int filter_linesize = conv_params->kernel_size * conv_params->input_num; - int filter_size = conv_params->kernel_size * filter_linesize; - int pad_size = (conv_params->padding_method == VALID) ? (conv_params->kernel_size - 1) / 2 * conv_params->dilation : 0; - - for (int y = pad_size; y < height - pad_size; ++y) { - for (int x = pad_size; x < width - pad_size; ++x) { - for (int n_filter = 0; n_filter < conv_params->output_num; ++n_filter) { - output[n_filter] = conv_params->biases[n_filter]; - - for (int ch = 0; ch < conv_params->input_num; ++ch) { - for (int kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y) { - for (int kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x) { - float input_pel; - if (conv_params->padding_method == SAME_CLAMP_TO_EDGE) { - int y_pos = CLAMP_TO_EDGE(y + (kernel_y - radius) * conv_params->dilation, height); - int x_pos = CLAMP_TO_EDGE(x + (kernel_x - radius) * conv_params->dilation, width); - input_pel = input[y_pos * src_linesize + x_pos * conv_params->input_num + ch]; - } else { - int y_pos = y + (kernel_y - radius) * conv_params->dilation; - int x_pos = x + (kernel_x - radius) * conv_params->dilation; - input_pel = (x_pos < 0 || x_pos >= width || y_pos < 0 || y_pos >= height) ? 0.0 : - input[y_pos * src_linesize + x_pos * conv_params->input_num + ch]; - } - - - output[n_filter] += input_pel * conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize + - kernel_x * conv_params->input_num + ch]; - } - } - } - switch (conv_params->activation){ - case RELU: - output[n_filter] = FFMAX(output[n_filter], 0.0); - break; - case TANH: - output[n_filter] = 2.0f / (1.0f + exp(-2.0f * output[n_filter])) - 1.0f; - break; - case SIGMOID: - output[n_filter] = 1.0f / (1.0f + exp(-output[n_filter])); - break; - case NONE: - break; - case LEAKY_RELU: - output[n_filter] = FFMAX(output[n_filter], 0.0) + 0.2 * FFMIN(output[n_filter], 0.0); - } - } - output += conv_params->output_num; - } - } -} - -static void depth_to_space(const float *input, float *output, int block_size, int width, int height, int channels) -{ - int y, x, by, bx, ch; - int new_channels = channels / (block_size * block_size); - int output_linesize = width * channels; - int by_linesize = output_linesize / block_size; - int x_linesize = new_channels * block_size; - - for (y = 0; y < height; ++y){ - for (x = 0; x < width; ++x){ - for (by = 0; by < block_size; ++by){ - for (bx = 0; bx < block_size; ++bx){ - for (ch = 0; ch < new_channels; ++ch){ - output[by * by_linesize + x * x_linesize + bx * new_channels + ch] = input[ch]; - } - input += new_channels; - } - } - } - output += output_linesize; - } -} - -DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output) -{ - ConvolutionalNetwork *network = (ConvolutionalNetwork *)model->model; - int cur_width, cur_height, cur_channels; - int32_t layer; - InputParams *input_params; - ConvolutionalParams *conv_params; - DepthToSpaceParams *depth_to_space_params; - - if (network->layers_num <= 0 || network->layers[0].type != INPUT || !network->layers[0].output){ - return DNN_ERROR; - } - else{ - input_params = (InputParams *)network->layers[0].params; - cur_width = input_params->width; - cur_height = input_params->height; - cur_channels = input_params->channels; - } - - for (layer = 1; layer < network->layers_num; ++layer){ - if (!network->layers[layer].output){ - return DNN_ERROR; - } - switch (network->layers[layer].type){ - case CONV: - conv_params = (ConvolutionalParams *)network->layers[layer].params; - convolve(network->layers[layer - 1].output, network->layers[layer].output, conv_params, cur_width, cur_height); - cur_channels = conv_params->output_num; - if (conv_params->padding_method == VALID) { - int pad_size = (conv_params->kernel_size - 1) * conv_params->dilation; - cur_height -= pad_size; - cur_width -= pad_size; - } - break; - case DEPTH_TO_SPACE: - depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params; - depth_to_space(network->layers[layer - 1].output, network->layers[layer].output, - depth_to_space_params->block_size, cur_width, cur_height, cur_channels); - cur_height *= depth_to_space_params->block_size; - cur_width *= depth_to_space_params->block_size; - cur_channels /= depth_to_space_params->block_size * depth_to_space_params->block_size; - break; - case INPUT: - return DNN_ERROR; - } - } - - // native mode does not support multiple outputs yet - if (nb_output > 1) - return DNN_ERROR; - outputs[0].data = network->layers[network->layers_num - 1].output; - outputs[0].height = cur_height; - outputs[0].width = cur_width; - outputs[0].channels = cur_channels; - - return DNN_SUCCESS; -} - -void ff_dnn_free_model_native(DNNModel **model) -{ - ConvolutionalNetwork *network; - ConvolutionalParams *conv_params; - int32_t layer; - - if (*model) - { - network = (ConvolutionalNetwork *)(*model)->model; - for (layer = 0; layer < network->layers_num; ++layer){ - av_freep(&network->layers[layer].output); - if (network->layers[layer].type == CONV){ - conv_params = (ConvolutionalParams *)network->layers[layer].params; - av_freep(&conv_params->kernel); - av_freep(&conv_params->biases); - } - av_freep(&network->layers[layer].params); - } - av_freep(&network->layers); - av_freep(&network); - av_freep(model); - } -} diff --git a/libavfilter/dnn_backend_native.h b/libavfilter/dnn_backend_native.h deleted file mode 100644 index 5917955..0000000 --- a/libavfilter/dnn_backend_native.h +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2018 Sergey Lavrushkin - * - * This file is part of FFmpeg. - * - * FFmpeg is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * FFmpeg is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with FFmpeg; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ - -/** - * @file - * DNN inference functions interface for native backend. - */ - - -#ifndef AVFILTER_DNN_BACKEND_NATIVE_H -#define AVFILTER_DNN_BACKEND_NATIVE_H - -#include "dnn_interface.h" -#include "libavformat/avio.h" - -typedef enum {INPUT, CONV, DEPTH_TO_SPACE} DNNLayerType; - -typedef enum {RELU, TANH, SIGMOID, NONE, LEAKY_RELU} DNNActivationFunc; - -typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} DNNConvPaddingParam; - -typedef struct Layer{ - DNNLayerType type; - float *output; - void *params; -} Layer; - -typedef struct ConvolutionalParams{ - int32_t input_num, output_num, kernel_size; - DNNActivationFunc activation; - DNNConvPaddingParam padding_method; - int32_t dilation; - float *kernel; - float *biases; -} ConvolutionalParams; - -typedef struct InputParams{ - int height, width, channels; -} InputParams; - -typedef struct DepthToSpaceParams{ - int block_size; -} DepthToSpaceParams; - -// Represents simple feed-forward convolutional network. -typedef struct ConvolutionalNetwork{ - Layer *layers; - int32_t layers_num; -} ConvolutionalNetwork; - -DNNModel *ff_dnn_load_model_native(const char *model_filename); - -DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output); - -void ff_dnn_free_model_native(DNNModel **model); - -#endif diff --git a/libavfilter/dnn_backend_tf.c b/libavfilter/dnn_backend_tf.c deleted file mode 100644 index ba959ae..0000000 --- a/libavfilter/dnn_backend_tf.c +++ /dev/null @@ -1,603 +0,0 @@ -/* - * Copyright (c) 2018 Sergey Lavrushkin - * - * This file is part of FFmpeg. - * - * FFmpeg is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * FFmpeg is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with FFmpeg; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ - -/** - * @file - * DNN tensorflow backend implementation. - */ - -#include "dnn_backend_tf.h" -#include "dnn_backend_native.h" -#include "libavformat/avio.h" -#include "libavutil/avassert.h" - -#include <tensorflow/c/c_api.h> - -typedef struct TFModel{ - TF_Graph *graph; - TF_Session *session; - TF_Status *status; - TF_Output input; - TF_Tensor *input_tensor; - TF_Output *outputs; - TF_Tensor **output_tensors; - uint32_t nb_output; -} TFModel; - -static void free_buffer(void *data, size_t length) -{ - av_freep(&data); -} - -static TF_Buffer *read_graph(const char *model_filename) -{ - TF_Buffer *graph_buf; - unsigned char *graph_data = NULL; - AVIOContext *model_file_context; - long size, bytes_read; - - if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){ - return NULL; - } - - size = avio_size(model_file_context); - - graph_data = av_malloc(size); - if (!graph_data){ - avio_closep(&model_file_context); - return NULL; - } - bytes_read = avio_read(model_file_context, graph_data, size); - avio_closep(&model_file_context); - if (bytes_read != size){ - av_freep(&graph_data); - return NULL; - } - - graph_buf = TF_NewBuffer(); - graph_buf->data = (void *)graph_data; - graph_buf->length = size; - graph_buf->data_deallocator = free_buffer; - - return graph_buf; -} - -static TF_Tensor *allocate_input_tensor(const DNNInputData *input) -{ - TF_DataType dt; - size_t size; - int64_t input_dims[] = {1, input->height, input->width, input->channels}; - switch (input->dt) { - case DNN_FLOAT: - dt = TF_FLOAT; - size = sizeof(float); - break; - case DNN_UINT8: - dt = TF_UINT8; - size = sizeof(char); - break; - default: - av_assert0(!"should not reach here"); - } - - return TF_AllocateTensor(dt, input_dims, 4, - input_dims[1] * input_dims[2] * input_dims[3] * size); -} - -static DNNReturnType set_input_output_tf(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output) -{ - TFModel *tf_model = (TFModel *)model; - TF_SessionOptions *sess_opts; - const TF_Operation *init_op = TF_GraphOperationByName(tf_model->graph, "init"); - - // Input operation - tf_model->input.oper = TF_GraphOperationByName(tf_model->graph, input_name); - if (!tf_model->input.oper){ - return DNN_ERROR; - } - tf_model->input.index = 0; - if (tf_model->input_tensor){ - TF_DeleteTensor(tf_model->input_tensor); - } - tf_model->input_tensor = allocate_input_tensor(input); - if (!tf_model->input_tensor){ - return DNN_ERROR; - } - input->data = (float *)TF_TensorData(tf_model->input_tensor); - - // Output operation - if (nb_output == 0) - return DNN_ERROR; - - av_freep(&tf_model->outputs); - tf_model->outputs = av_malloc_array(nb_output, sizeof(*tf_model->outputs)); - if (!tf_model->outputs) - return DNN_ERROR; - for (int i = 0; i < nb_output; ++i) { - tf_model->outputs[i].oper = TF_GraphOperationByName(tf_model->graph, output_names[i]); - if (!tf_model->outputs[i].oper){ - av_freep(&tf_model->outputs); - return DNN_ERROR; - } - tf_model->outputs[i].index = 0; - } - - if (tf_model->output_tensors) { - for (uint32_t i = 0; i < tf_model->nb_output; ++i) { - if (tf_model->output_tensors[i]) { - TF_DeleteTensor(tf_model->output_tensors[i]); - tf_model->output_tensors[i] = NULL; - } - } - } - av_freep(&tf_model->output_tensors); - tf_model->output_tensors = av_mallocz_array(nb_output, sizeof(*tf_model->output_tensors)); - if (!tf_model->output_tensors) { - av_freep(&tf_model->outputs); - return DNN_ERROR; - } - - tf_model->nb_output = nb_output; - - if (tf_model->session){ - TF_CloseSession(tf_model->session, tf_model->status); - TF_DeleteSession(tf_model->session, tf_model->status); - } - - sess_opts = TF_NewSessionOptions(); - tf_model->session = TF_NewSession(tf_model->graph, sess_opts, tf_model->status); - TF_DeleteSessionOptions(sess_opts); - if (TF_GetCode(tf_model->status) != TF_OK) - { - return DNN_ERROR; - } - - // Run initialization operation with name "init" if it is present in graph - if (init_op){ - TF_SessionRun(tf_model->session, NULL, - NULL, NULL, 0, - NULL, NULL, 0, - &init_op, 1, NULL, tf_model->status); - if (TF_GetCode(tf_model->status) != TF_OK) - { - return DNN_ERROR; - } - } - - return DNN_SUCCESS; -} - -static DNNReturnType load_tf_model(TFModel *tf_model, const char *model_filename) -{ - TF_Buffer *graph_def; - TF_ImportGraphDefOptions *graph_opts; - - graph_def = read_graph(model_filename); - if (!graph_def){ - return DNN_ERROR; - } - tf_model->graph = TF_NewGraph(); - tf_model->status = TF_NewStatus(); - graph_opts = TF_NewImportGraphDefOptions(); - TF_GraphImportGraphDef(tf_model->graph, graph_def, graph_opts, tf_model->status); - TF_DeleteImportGraphDefOptions(graph_opts); - TF_DeleteBuffer(graph_def); - if (TF_GetCode(tf_model->status) != TF_OK){ - TF_DeleteGraph(tf_model->graph); - TF_DeleteStatus(tf_model->status); - return DNN_ERROR; - } - - return DNN_SUCCESS; -} - -#define NAME_BUFFER_SIZE 256 - -static DNNReturnType add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op, - ConvolutionalParams* params, const int layer) -{ - TF_Operation *op; - TF_OperationDescription *op_desc; - TF_Output input; - int64_t strides[] = {1, 1, 1, 1}; - TF_Tensor *tensor; - int64_t dims[4]; - int dims_len; - char name_buffer[NAME_BUFFER_SIZE]; - int32_t size; - - size = params->input_num * params->output_num * params->kernel_size * params->kernel_size; - input.index = 0; - - snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_kernel%d", layer); - op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer); - TF_SetAttrType(op_desc, "dtype", TF_FLOAT); - dims[0] = params->output_num; - dims[1] = params->kernel_size; - dims[2] = params->kernel_size; - dims[3] = params->input_num; - dims_len = 4; - tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, size * sizeof(float)); - memcpy(TF_TensorData(tensor), params->kernel, size * sizeof(float)); - TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); - if (TF_GetCode(tf_model->status) != TF_OK){ - return DNN_ERROR; - } - op = TF_FinishOperation(op_desc, tf_model->status); - if (TF_GetCode(tf_model->status) != TF_OK){ - return DNN_ERROR; - } - - snprintf(name_buffer, NAME_BUFFER_SIZE, "transpose%d", layer); - op_desc = TF_NewOperation(tf_model->graph, "Transpose", name_buffer); - input.oper = op; - TF_AddInput(op_desc, input); - input.oper = transpose_op; - TF_AddInput(op_desc, input); - TF_SetAttrType(op_desc, "T", TF_FLOAT); - TF_SetAttrType(op_desc, "Tperm", TF_INT32); - op = TF_FinishOperation(op_desc, tf_model->status); - if (TF_GetCode(tf_model->status) != TF_OK){ - return DNN_ERROR; - } - - snprintf(name_buffer, NAME_BUFFER_SIZE, "conv2d%d", layer); - op_desc = TF_NewOperation(tf_model->graph, "Conv2D", name_buffer); - input.oper = *cur_op; - TF_AddInput(op_desc, input); - input.oper = op; - TF_AddInput(op_desc, input); - TF_SetAttrType(op_desc, "T", TF_FLOAT); - TF_SetAttrIntList(op_desc, "strides", strides, 4); - TF_SetAttrString(op_desc, "padding", "VALID", 5); - *cur_op = TF_FinishOperation(op_desc, tf_model->status); - if (TF_GetCode(tf_model->status) != TF_OK){ - return DNN_ERROR; - } - - snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_biases%d", layer); - op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer); - TF_SetAttrType(op_desc, "dtype", TF_FLOAT); - dims[0] = params->output_num; - dims_len = 1; - tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, params->output_num * sizeof(float)); - memcpy(TF_TensorData(tensor), params->biases, params->output_num * sizeof(float)); - TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); - if (TF_GetCode(tf_model->status) != TF_OK){ - return DNN_ERROR; - } - op = TF_FinishOperation(op_desc, tf_model->status); - if (TF_GetCode(tf_model->status) != TF_OK){ - return DNN_ERROR; - } - - snprintf(name_buffer, NAME_BUFFER_SIZE, "bias_add%d", layer); - op_desc = TF_NewOperation(tf_model->graph, "BiasAdd", name_buffer); - input.oper = *cur_op; - TF_AddInput(op_desc, input); - input.oper = op; - TF_AddInput(op_desc, input); - TF_SetAttrType(op_desc, "T", TF_FLOAT); - *cur_op = TF_FinishOperation(op_desc, tf_model->status); - if (TF_GetCode(tf_model->status) != TF_OK){ - return DNN_ERROR; - } - - snprintf(name_buffer, NAME_BUFFER_SIZE, "activation%d", layer); - switch (params->activation){ - case RELU: - op_desc = TF_NewOperation(tf_model->graph, "Relu", name_buffer); - break; - case TANH: - op_desc = TF_NewOperation(tf_model->graph, "Tanh", name_buffer); - break; - case SIGMOID: - op_desc = TF_NewOperation(tf_model->graph, "Sigmoid", name_buffer); - break; - default: - return DNN_ERROR; - } - input.oper = *cur_op; - TF_AddInput(op_desc, input); - TF_SetAttrType(op_desc, "T", TF_FLOAT); - *cur_op = TF_FinishOperation(op_desc, tf_model->status); - if (TF_GetCode(tf_model->status) != TF_OK){ - return DNN_ERROR; - } - - return DNN_SUCCESS; -} - -static DNNReturnType add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op, - DepthToSpaceParams *params, const int layer) -{ - TF_OperationDescription *op_desc; - TF_Output input; - char name_buffer[NAME_BUFFER_SIZE]; - - snprintf(name_buffer, NAME_BUFFER_SIZE, "depth_to_space%d", layer); - op_desc = TF_NewOperation(tf_model->graph, "DepthToSpace", name_buffer); - input.oper = *cur_op; - input.index = 0; - TF_AddInput(op_desc, input); - TF_SetAttrType(op_desc, "T", TF_FLOAT); - TF_SetAttrInt(op_desc, "block_size", params->block_size); - *cur_op = TF_FinishOperation(op_desc, tf_model->status); - if (TF_GetCode(tf_model->status) != TF_OK){ - return DNN_ERROR; - } - - return DNN_SUCCESS; -} - -static int calculate_pad(const ConvolutionalNetwork *conv_network) -{ - ConvolutionalParams *params; - int32_t layer; - int pad = 0; - - for (layer = 0; layer < conv_network->layers_num; ++layer){ - if (conv_network->layers[layer].type == CONV){ - params = (ConvolutionalParams *)conv_network->layers[layer].params; - pad += params->kernel_size >> 1; - } - } - - return pad; -} - -static DNNReturnType add_pad_op(TFModel *tf_model, TF_Operation **cur_op, const int32_t pad) -{ - TF_Operation *op; - TF_Tensor *tensor; - TF_OperationDescription *op_desc; - TF_Output input; - int32_t *pads; - int64_t pads_shape[] = {4, 2}; - - input.index = 0; - - op_desc = TF_NewOperation(tf_model->graph, "Const", "pads"); - TF_SetAttrType(op_desc, "dtype", TF_INT32); - tensor = TF_AllocateTensor(TF_INT32, pads_shape, 2, 4 * 2 * sizeof(int32_t)); - pads = (int32_t *)TF_TensorData(tensor); - pads[0] = 0; pads[1] = 0; - pads[2] = pad; pads[3] = pad; - pads[4] = pad; pads[5] = pad; - pads[6] = 0; pads[7] = 0; - TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); - if (TF_GetCode(tf_model->status) != TF_OK){ - return DNN_ERROR; - } - op = TF_FinishOperation(op_desc, tf_model->status); - if (TF_GetCode(tf_model->status) != TF_OK){ - return DNN_ERROR; - } - - op_desc = TF_NewOperation(tf_model->graph, "MirrorPad", "mirror_pad"); - input.oper = *cur_op; - TF_AddInput(op_desc, input); - input.oper = op; - TF_AddInput(op_desc, input); - TF_SetAttrType(op_desc, "T", TF_FLOAT); - TF_SetAttrType(op_desc, "Tpaddings", TF_INT32); - TF_SetAttrString(op_desc, "mode", "SYMMETRIC", 9); - *cur_op = TF_FinishOperation(op_desc, tf_model->status); - if (TF_GetCode(tf_model->status) != TF_OK){ - return DNN_ERROR; - } - - return DNN_SUCCESS; -} - -static DNNReturnType load_native_model(TFModel *tf_model, const char *model_filename) -{ - int32_t layer; - TF_OperationDescription *op_desc; - TF_Operation *op; - TF_Operation *transpose_op; - TF_Tensor *tensor; - TF_Output input; - int32_t *transpose_perm; - int64_t transpose_perm_shape[] = {4}; - int64_t input_shape[] = {1, -1, -1, -1}; - int32_t pad; - DNNReturnType layer_add_res; - DNNModel *native_model = NULL; - ConvolutionalNetwork *conv_network; - - native_model = ff_dnn_load_model_native(model_filename); - if (!native_model){ - return DNN_ERROR; - } - - conv_network = (ConvolutionalNetwork *)native_model->model; - pad = calculate_pad(conv_network); - tf_model->graph = TF_NewGraph(); - tf_model->status = TF_NewStatus(); - -#define CLEANUP_ON_ERROR(tf_model) \ - { \ - TF_DeleteGraph(tf_model->graph); \ - TF_DeleteStatus(tf_model->status); \ - return DNN_ERROR; \ - } - - op_desc = TF_NewOperation(tf_model->graph, "Placeholder", "x"); - TF_SetAttrType(op_desc, "dtype", TF_FLOAT); - TF_SetAttrShape(op_desc, "shape", input_shape, 4); - op = TF_FinishOperation(op_desc, tf_model->status); - if (TF_GetCode(tf_model->status) != TF_OK){ - CLEANUP_ON_ERROR(tf_model); - } - - if (add_pad_op(tf_model, &op, pad) != DNN_SUCCESS){ - CLEANUP_ON_ERROR(tf_model); - } - - op_desc = TF_NewOperation(tf_model->graph, "Const", "transpose_perm"); - TF_SetAttrType(op_desc, "dtype", TF_INT32); - tensor = TF_AllocateTensor(TF_INT32, transpose_perm_shape, 1, 4 * sizeof(int32_t)); - transpose_perm = (int32_t *)TF_TensorData(tensor); - transpose_perm[0] = 1; - transpose_perm[1] = 2; - transpose_perm[2] = 3; - transpose_perm[3] = 0; - TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status); - if (TF_GetCode(tf_model->status) != TF_OK){ - CLEANUP_ON_ERROR(tf_model); - } - transpose_op = TF_FinishOperation(op_desc, tf_model->status); - - for (layer = 0; layer < conv_network->layers_num; ++layer){ - switch (conv_network->layers[layer].type){ - case INPUT: - layer_add_res = DNN_SUCCESS; - break; - case CONV: - layer_add_res = add_conv_layer(tf_model, transpose_op, &op, - (ConvolutionalParams *)conv_network->layers[layer].params, layer); - break; - case DEPTH_TO_SPACE: - layer_add_res = add_depth_to_space_layer(tf_model, &op, - (DepthToSpaceParams *)conv_network->layers[layer].params, layer); - break; - default: - CLEANUP_ON_ERROR(tf_model); - } - - if (layer_add_res != DNN_SUCCESS){ - CLEANUP_ON_ERROR(tf_model); - } - } - - op_desc = TF_NewOperation(tf_model->graph, "Identity", "y"); - input.oper = op; - TF_AddInput(op_desc, input); - TF_FinishOperation(op_desc, tf_model->status); - if (TF_GetCode(tf_model->status) != TF_OK){ - CLEANUP_ON_ERROR(tf_model); - } - - ff_dnn_free_model_native(&native_model); - - return DNN_SUCCESS; -} - -DNNModel *ff_dnn_load_model_tf(const char *model_filename) -{ - DNNModel *model = NULL; - TFModel *tf_model = NULL; - - model = av_malloc(sizeof(DNNModel)); - if (!model){ - return NULL; - } - - tf_model = av_mallocz(sizeof(TFModel)); - if (!tf_model){ - av_freep(&model); - return NULL; - } - - if (load_tf_model(tf_model, model_filename) != DNN_SUCCESS){ - if (load_native_model(tf_model, model_filename) != DNN_SUCCESS){ - av_freep(&tf_model); - av_freep(&model); - - return NULL; - } - } - - model->model = (void *)tf_model; - model->set_input_output = &set_input_output_tf; - - return model; -} - - - -DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, uint32_t nb_output) -{ - TFModel *tf_model = (TFModel *)model->model; - uint32_t nb = FFMIN(nb_output, tf_model->nb_output); - if (nb == 0) - return DNN_ERROR; - - av_assert0(tf_model->output_tensors); - for (uint32_t i = 0; i < tf_model->nb_output; ++i) { - if (tf_model->output_tensors[i]) { - TF_DeleteTensor(tf_model->output_tensors[i]); - tf_model->output_tensors[i] = NULL; - } - } - - TF_SessionRun(tf_model->session, NULL, - &tf_model->input, &tf_model->input_tensor, 1, - tf_model->outputs, tf_model->output_tensors, nb, - NULL, 0, NULL, tf_model->status); - - if (TF_GetCode(tf_model->status) != TF_OK){ - return DNN_ERROR; - } - - for (uint32_t i = 0; i < nb; ++i) { - outputs[i].height = TF_Dim(tf_model->output_tensors[i], 1); - outputs[i].width = TF_Dim(tf_model->output_tensors[i], 2); - outputs[i].channels = TF_Dim(tf_model->output_tensors[i], 3); - outputs[i].data = TF_TensorData(tf_model->output_tensors[i]); - } - - return DNN_SUCCESS; -} - -void ff_dnn_free_model_tf(DNNModel **model) -{ - TFModel *tf_model; - - if (*model){ - tf_model = (TFModel *)(*model)->model; - if (tf_model->graph){ - TF_DeleteGraph(tf_model->graph); - } - if (tf_model->session){ - TF_CloseSession(tf_model->session, tf_model->status); - TF_DeleteSession(tf_model->session, tf_model->status); - } - if (tf_model->status){ - TF_DeleteStatus(tf_model->status); - } - if (tf_model->input_tensor){ - TF_DeleteTensor(tf_model->input_tensor); - } - if (tf_model->output_tensors) { - for (uint32_t i = 0; i < tf_model->nb_output; ++i) { - if (tf_model->output_tensors[i]) { - TF_DeleteTensor(tf_model->output_tensors[i]); - tf_model->output_tensors[i] = NULL; - } - } - } - av_freep(&tf_model->outputs); - av_freep(&tf_model->output_tensors); - av_freep(&tf_model); - av_freep(model); - } -} diff --git a/libavfilter/dnn_backend_tf.h b/libavfilter/dnn_backend_tf.h deleted file mode 100644 index 07877b1..0000000 --- a/libavfilter/dnn_backend_tf.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2018 Sergey Lavrushkin - * - * This file is part of FFmpeg. - * - * FFmpeg is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * FFmpeg is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with FFmpeg; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ - -/** - * @file - * DNN inference functions interface for TensorFlow backend. - */ - - -#ifndef AVFILTER_DNN_BACKEND_TF_H -#define AVFILTER_DNN_BACKEND_TF_H - -#include "dnn_interface.h" - -DNNModel *ff_dnn_load_model_tf(const char *model_filename); - -DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, uint32_t nb_output); - -void ff_dnn_free_model_tf(DNNModel **model); - -#endif diff --git a/libavfilter/dnn_interface.c b/libavfilter/dnn_interface.c deleted file mode 100644 index 86fc283..0000000 --- a/libavfilter/dnn_interface.c +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2018 Sergey Lavrushkin - * - * This file is part of FFmpeg. - * - * FFmpeg is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * FFmpeg is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with FFmpeg; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ - -/** - * @file - * Implements DNN module initialization with specified backend. - */ - -#include "dnn_interface.h" -#include "dnn_backend_native.h" -#include "dnn_backend_tf.h" -#include "libavutil/mem.h" - -DNNModule *ff_get_dnn_module(DNNBackendType backend_type) -{ - DNNModule *dnn_module; - - dnn_module = av_malloc(sizeof(DNNModule)); - if(!dnn_module){ - return NULL; - } - - switch(backend_type){ - case DNN_NATIVE: - dnn_module->load_model = &ff_dnn_load_model_native; - dnn_module->execute_model = &ff_dnn_execute_model_native; - dnn_module->free_model = &ff_dnn_free_model_native; - break; - case DNN_TF: - #if (CONFIG_LIBTENSORFLOW == 1) - dnn_module->load_model = &ff_dnn_load_model_tf; - dnn_module->execute_model = &ff_dnn_execute_model_tf; - dnn_module->free_model = &ff_dnn_free_model_tf; - #else - av_freep(&dnn_module); - return NULL; - #endif - break; - default: - av_log(NULL, AV_LOG_ERROR, "Module backend_type is not native or tensorflow\n"); - av_freep(&dnn_module); - return NULL; - } - - return dnn_module; -}
it is expected that there will be more files to support native mode, so put all the dnn codes under libavfilter/dnn The main change of this patch is to move the file location, see below: modified: libavfilter/Makefile new file: libavfilter/dnn/Makefile renamed: libavfilter/dnn_backend_native.c -> libavfilter/dnn/dnn_backend_native.c renamed: libavfilter/dnn_backend_native.h -> libavfilter/dnn/dnn_backend_native.h renamed: libavfilter/dnn_backend_tf.c -> libavfilter/dnn/dnn_backend_tf.c renamed: libavfilter/dnn_backend_tf.h -> libavfilter/dnn/dnn_backend_tf.h renamed: libavfilter/dnn_interface.c -> libavfilter/dnn/dnn_interface.c Signed-off-by: Guo, Yejun <yejun.guo@intel.com> --- libavfilter/Makefile | 3 +- libavfilter/dnn/Makefile | 6 + libavfilter/dnn/dnn_backend_native.c | 389 ++++++++++++++++++++++ libavfilter/dnn/dnn_backend_native.h | 74 +++++ libavfilter/dnn/dnn_backend_tf.c | 603 +++++++++++++++++++++++++++++++++++ libavfilter/dnn/dnn_backend_tf.h | 38 +++ libavfilter/dnn/dnn_interface.c | 63 ++++ libavfilter/dnn_backend_native.c | 389 ---------------------- libavfilter/dnn_backend_native.h | 74 ----- libavfilter/dnn_backend_tf.c | 603 ----------------------------------- libavfilter/dnn_backend_tf.h | 38 --- libavfilter/dnn_interface.c | 63 ---- 12 files changed, 1174 insertions(+), 1169 deletions(-) create mode 100644 libavfilter/dnn/Makefile create mode 100644 libavfilter/dnn/dnn_backend_native.c create mode 100644 libavfilter/dnn/dnn_backend_native.h create mode 100644 libavfilter/dnn/dnn_backend_tf.c create mode 100644 libavfilter/dnn/dnn_backend_tf.h create mode 100644 libavfilter/dnn/dnn_interface.c delete mode 100644 libavfilter/dnn_backend_native.c delete mode 100644 libavfilter/dnn_backend_native.h delete mode 100644 libavfilter/dnn_backend_tf.c delete mode 100644 libavfilter/dnn_backend_tf.h delete mode 100644 libavfilter/dnn_interface.c