[FFmpeg-devel] [PATCH 1/4] libavfilter/dnn: move dnn files from libavfilter to libavfilter/dnn
Guo, Yejun
yejun.guo at intel.com
Tue Jul 16 08:55:45 EEST 2019
it is expected that there will be more files to support native mode,
so put all the dnn codes under libavfilter/dnn
The main change of this patch is to move the file location, see below:
modified: libavfilter/Makefile
new file: libavfilter/dnn/Makefile
renamed: libavfilter/dnn_backend_native.c -> libavfilter/dnn/dnn_backend_native.c
renamed: libavfilter/dnn_backend_native.h -> libavfilter/dnn/dnn_backend_native.h
renamed: libavfilter/dnn_backend_tf.c -> libavfilter/dnn/dnn_backend_tf.c
renamed: libavfilter/dnn_backend_tf.h -> libavfilter/dnn/dnn_backend_tf.h
renamed: libavfilter/dnn_interface.c -> libavfilter/dnn/dnn_interface.c
Signed-off-by: Guo, Yejun <yejun.guo at intel.com>
---
libavfilter/Makefile | 3 +-
libavfilter/dnn/Makefile | 6 +
libavfilter/dnn/dnn_backend_native.c | 389 ++++++++++++++++++++++
libavfilter/dnn/dnn_backend_native.h | 74 +++++
libavfilter/dnn/dnn_backend_tf.c | 603 +++++++++++++++++++++++++++++++++++
libavfilter/dnn/dnn_backend_tf.h | 38 +++
libavfilter/dnn/dnn_interface.c | 63 ++++
libavfilter/dnn_backend_native.c | 389 ----------------------
libavfilter/dnn_backend_native.h | 74 -----
libavfilter/dnn_backend_tf.c | 603 -----------------------------------
libavfilter/dnn_backend_tf.h | 38 ---
libavfilter/dnn_interface.c | 63 ----
12 files changed, 1174 insertions(+), 1169 deletions(-)
create mode 100644 libavfilter/dnn/Makefile
create mode 100644 libavfilter/dnn/dnn_backend_native.c
create mode 100644 libavfilter/dnn/dnn_backend_native.h
create mode 100644 libavfilter/dnn/dnn_backend_tf.c
create mode 100644 libavfilter/dnn/dnn_backend_tf.h
create mode 100644 libavfilter/dnn/dnn_interface.c
delete mode 100644 libavfilter/dnn_backend_native.c
delete mode 100644 libavfilter/dnn_backend_native.h
delete mode 100644 libavfilter/dnn_backend_tf.c
delete mode 100644 libavfilter/dnn_backend_tf.h
delete mode 100644 libavfilter/dnn_interface.c
diff --git a/libavfilter/Makefile b/libavfilter/Makefile
index 455c809..450d781 100644
--- a/libavfilter/Makefile
+++ b/libavfilter/Makefile
@@ -26,9 +26,8 @@ OBJS-$(HAVE_THREADS) += pthread.o
# subsystems
OBJS-$(CONFIG_QSVVPP) += qsvvpp.o
-DNN-OBJS-$(CONFIG_LIBTENSORFLOW) += dnn_backend_tf.o
-OBJS-$(CONFIG_DNN) += dnn_interface.o dnn_backend_native.o $(DNN-OBJS-yes)
OBJS-$(CONFIG_SCENE_SAD) += scene_sad.o
+include $(SRC_PATH)/libavfilter/dnn/Makefile
# audio filters
OBJS-$(CONFIG_ABENCH_FILTER) += f_bench.o
diff --git a/libavfilter/dnn/Makefile b/libavfilter/dnn/Makefile
new file mode 100644
index 0000000..1d12ade
--- /dev/null
+++ b/libavfilter/dnn/Makefile
@@ -0,0 +1,6 @@
+OBJS-$(CONFIG_DNN) += dnn/dnn_interface.o
+OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native.o
+
+DNN-OBJS-$(CONFIG_LIBTENSORFLOW) += dnn/dnn_backend_tf.o
+
+OBJS-$(CONFIG_DNN) += $(DNN-OBJS-yes)
diff --git a/libavfilter/dnn/dnn_backend_native.c b/libavfilter/dnn/dnn_backend_native.c
new file mode 100644
index 0000000..82e900b
--- /dev/null
+++ b/libavfilter/dnn/dnn_backend_native.c
@@ -0,0 +1,389 @@
+/*
+ * Copyright (c) 2018 Sergey Lavrushkin
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file
+ * DNN native backend implementation.
+ */
+
+#include "dnn_backend_native.h"
+#include "libavutil/avassert.h"
+
+static DNNReturnType set_input_output_native(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output)
+{
+ ConvolutionalNetwork *network = (ConvolutionalNetwork *)model;
+ InputParams *input_params;
+ ConvolutionalParams *conv_params;
+ DepthToSpaceParams *depth_to_space_params;
+ int cur_width, cur_height, cur_channels;
+ int32_t layer;
+
+ if (network->layers_num <= 0 || network->layers[0].type != INPUT){
+ return DNN_ERROR;
+ }
+ else{
+ input_params = (InputParams *)network->layers[0].params;
+ input_params->width = cur_width = input->width;
+ input_params->height = cur_height = input->height;
+ input_params->channels = cur_channels = input->channels;
+ if (input->data){
+ av_freep(&input->data);
+ }
+ av_assert0(input->dt == DNN_FLOAT);
+ network->layers[0].output = input->data = av_malloc(cur_height * cur_width * cur_channels * sizeof(float));
+ if (!network->layers[0].output){
+ return DNN_ERROR;
+ }
+ }
+
+ for (layer = 1; layer < network->layers_num; ++layer){
+ switch (network->layers[layer].type){
+ case CONV:
+ conv_params = (ConvolutionalParams *)network->layers[layer].params;
+ if (conv_params->input_num != cur_channels){
+ return DNN_ERROR;
+ }
+ cur_channels = conv_params->output_num;
+
+ if (conv_params->padding_method == VALID) {
+ int pad_size = (conv_params->kernel_size - 1) * conv_params->dilation;
+ cur_height -= pad_size;
+ cur_width -= pad_size;
+ }
+ break;
+ case DEPTH_TO_SPACE:
+ depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params;
+ if (cur_channels % (depth_to_space_params->block_size * depth_to_space_params->block_size) != 0){
+ return DNN_ERROR;
+ }
+ cur_channels = cur_channels / (depth_to_space_params->block_size * depth_to_space_params->block_size);
+ cur_height *= depth_to_space_params->block_size;
+ cur_width *= depth_to_space_params->block_size;
+ break;
+ default:
+ return DNN_ERROR;
+ }
+ if (network->layers[layer].output){
+ av_freep(&network->layers[layer].output);
+ }
+
+ if (cur_height <= 0 || cur_width <= 0)
+ return DNN_ERROR;
+
+ network->layers[layer].output = av_malloc(cur_height * cur_width * cur_channels * sizeof(float));
+ if (!network->layers[layer].output){
+ return DNN_ERROR;
+ }
+ }
+
+ return DNN_SUCCESS;
+}
+
+// Loads model and its parameters that are stored in a binary file with following structure:
+// layers_num,layer_type,layer_parameterss,layer_type,layer_parameters...
+// For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases
+// For DEPTH_TO_SPACE layer: block_size
+DNNModel *ff_dnn_load_model_native(const char *model_filename)
+{
+ DNNModel *model = NULL;
+ ConvolutionalNetwork *network = NULL;
+ AVIOContext *model_file_context;
+ int file_size, dnn_size, kernel_size, i;
+ int32_t layer;
+ DNNLayerType layer_type;
+ ConvolutionalParams *conv_params;
+ DepthToSpaceParams *depth_to_space_params;
+
+ model = av_malloc(sizeof(DNNModel));
+ if (!model){
+ return NULL;
+ }
+
+ if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
+ av_freep(&model);
+ return NULL;
+ }
+ file_size = avio_size(model_file_context);
+
+ network = av_malloc(sizeof(ConvolutionalNetwork));
+ if (!network){
+ avio_closep(&model_file_context);
+ av_freep(&model);
+ return NULL;
+ }
+ model->model = (void *)network;
+
+ network->layers_num = 1 + (int32_t)avio_rl32(model_file_context);
+ dnn_size = 4;
+
+ network->layers = av_malloc(network->layers_num * sizeof(Layer));
+ if (!network->layers){
+ av_freep(&network);
+ avio_closep(&model_file_context);
+ av_freep(&model);
+ return NULL;
+ }
+
+ for (layer = 0; layer < network->layers_num; ++layer){
+ network->layers[layer].output = NULL;
+ network->layers[layer].params = NULL;
+ }
+ network->layers[0].type = INPUT;
+ network->layers[0].params = av_malloc(sizeof(InputParams));
+ if (!network->layers[0].params){
+ avio_closep(&model_file_context);
+ ff_dnn_free_model_native(&model);
+ return NULL;
+ }
+
+ for (layer = 1; layer < network->layers_num; ++layer){
+ layer_type = (int32_t)avio_rl32(model_file_context);
+ dnn_size += 4;
+ switch (layer_type){
+ case CONV:
+ conv_params = av_malloc(sizeof(ConvolutionalParams));
+ if (!conv_params){
+ avio_closep(&model_file_context);
+ ff_dnn_free_model_native(&model);
+ return NULL;
+ }
+ conv_params->dilation = (int32_t)avio_rl32(model_file_context);
+ conv_params->padding_method = (int32_t)avio_rl32(model_file_context);
+ conv_params->activation = (int32_t)avio_rl32(model_file_context);
+ conv_params->input_num = (int32_t)avio_rl32(model_file_context);
+ conv_params->output_num = (int32_t)avio_rl32(model_file_context);
+ conv_params->kernel_size = (int32_t)avio_rl32(model_file_context);
+ kernel_size = conv_params->input_num * conv_params->output_num *
+ conv_params->kernel_size * conv_params->kernel_size;
+ dnn_size += 24 + (kernel_size + conv_params->output_num << 2);
+ if (dnn_size > file_size || conv_params->input_num <= 0 ||
+ conv_params->output_num <= 0 || conv_params->kernel_size <= 0){
+ avio_closep(&model_file_context);
+ ff_dnn_free_model_native(&model);
+ return NULL;
+ }
+ conv_params->kernel = av_malloc(kernel_size * sizeof(float));
+ conv_params->biases = av_malloc(conv_params->output_num * sizeof(float));
+ if (!conv_params->kernel || !conv_params->biases){
+ avio_closep(&model_file_context);
+ ff_dnn_free_model_native(&model);
+ return NULL;
+ }
+ for (i = 0; i < kernel_size; ++i){
+ conv_params->kernel[i] = av_int2float(avio_rl32(model_file_context));
+ }
+ for (i = 0; i < conv_params->output_num; ++i){
+ conv_params->biases[i] = av_int2float(avio_rl32(model_file_context));
+ }
+ network->layers[layer].type = CONV;
+ network->layers[layer].params = conv_params;
+ break;
+ case DEPTH_TO_SPACE:
+ depth_to_space_params = av_malloc(sizeof(DepthToSpaceParams));
+ if (!depth_to_space_params){
+ avio_closep(&model_file_context);
+ ff_dnn_free_model_native(&model);
+ return NULL;
+ }
+ depth_to_space_params->block_size = (int32_t)avio_rl32(model_file_context);
+ dnn_size += 4;
+ network->layers[layer].type = DEPTH_TO_SPACE;
+ network->layers[layer].params = depth_to_space_params;
+ break;
+ default:
+ avio_closep(&model_file_context);
+ ff_dnn_free_model_native(&model);
+ return NULL;
+ }
+ }
+
+ avio_closep(&model_file_context);
+
+ if (dnn_size != file_size){
+ ff_dnn_free_model_native(&model);
+ return NULL;
+ }
+
+ model->set_input_output = &set_input_output_native;
+
+ return model;
+}
+
+#define CLAMP_TO_EDGE(x, w) ((x) < 0 ? 0 : ((x) >= (w) ? (w - 1) : (x)))
+
+static void convolve(const float *input, float *output, const ConvolutionalParams *conv_params, int width, int height)
+{
+ int radius = conv_params->kernel_size >> 1;
+ int src_linesize = width * conv_params->input_num;
+ int filter_linesize = conv_params->kernel_size * conv_params->input_num;
+ int filter_size = conv_params->kernel_size * filter_linesize;
+ int pad_size = (conv_params->padding_method == VALID) ? (conv_params->kernel_size - 1) / 2 * conv_params->dilation : 0;
+
+ for (int y = pad_size; y < height - pad_size; ++y) {
+ for (int x = pad_size; x < width - pad_size; ++x) {
+ for (int n_filter = 0; n_filter < conv_params->output_num; ++n_filter) {
+ output[n_filter] = conv_params->biases[n_filter];
+
+ for (int ch = 0; ch < conv_params->input_num; ++ch) {
+ for (int kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y) {
+ for (int kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x) {
+ float input_pel;
+ if (conv_params->padding_method == SAME_CLAMP_TO_EDGE) {
+ int y_pos = CLAMP_TO_EDGE(y + (kernel_y - radius) * conv_params->dilation, height);
+ int x_pos = CLAMP_TO_EDGE(x + (kernel_x - radius) * conv_params->dilation, width);
+ input_pel = input[y_pos * src_linesize + x_pos * conv_params->input_num + ch];
+ } else {
+ int y_pos = y + (kernel_y - radius) * conv_params->dilation;
+ int x_pos = x + (kernel_x - radius) * conv_params->dilation;
+ input_pel = (x_pos < 0 || x_pos >= width || y_pos < 0 || y_pos >= height) ? 0.0 :
+ input[y_pos * src_linesize + x_pos * conv_params->input_num + ch];
+ }
+
+
+ output[n_filter] += input_pel * conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize +
+ kernel_x * conv_params->input_num + ch];
+ }
+ }
+ }
+ switch (conv_params->activation){
+ case RELU:
+ output[n_filter] = FFMAX(output[n_filter], 0.0);
+ break;
+ case TANH:
+ output[n_filter] = 2.0f / (1.0f + exp(-2.0f * output[n_filter])) - 1.0f;
+ break;
+ case SIGMOID:
+ output[n_filter] = 1.0f / (1.0f + exp(-output[n_filter]));
+ break;
+ case NONE:
+ break;
+ case LEAKY_RELU:
+ output[n_filter] = FFMAX(output[n_filter], 0.0) + 0.2 * FFMIN(output[n_filter], 0.0);
+ }
+ }
+ output += conv_params->output_num;
+ }
+ }
+}
+
+static void depth_to_space(const float *input, float *output, int block_size, int width, int height, int channels)
+{
+ int y, x, by, bx, ch;
+ int new_channels = channels / (block_size * block_size);
+ int output_linesize = width * channels;
+ int by_linesize = output_linesize / block_size;
+ int x_linesize = new_channels * block_size;
+
+ for (y = 0; y < height; ++y){
+ for (x = 0; x < width; ++x){
+ for (by = 0; by < block_size; ++by){
+ for (bx = 0; bx < block_size; ++bx){
+ for (ch = 0; ch < new_channels; ++ch){
+ output[by * by_linesize + x * x_linesize + bx * new_channels + ch] = input[ch];
+ }
+ input += new_channels;
+ }
+ }
+ }
+ output += output_linesize;
+ }
+}
+
+DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output)
+{
+ ConvolutionalNetwork *network = (ConvolutionalNetwork *)model->model;
+ int cur_width, cur_height, cur_channels;
+ int32_t layer;
+ InputParams *input_params;
+ ConvolutionalParams *conv_params;
+ DepthToSpaceParams *depth_to_space_params;
+
+ if (network->layers_num <= 0 || network->layers[0].type != INPUT || !network->layers[0].output){
+ return DNN_ERROR;
+ }
+ else{
+ input_params = (InputParams *)network->layers[0].params;
+ cur_width = input_params->width;
+ cur_height = input_params->height;
+ cur_channels = input_params->channels;
+ }
+
+ for (layer = 1; layer < network->layers_num; ++layer){
+ if (!network->layers[layer].output){
+ return DNN_ERROR;
+ }
+ switch (network->layers[layer].type){
+ case CONV:
+ conv_params = (ConvolutionalParams *)network->layers[layer].params;
+ convolve(network->layers[layer - 1].output, network->layers[layer].output, conv_params, cur_width, cur_height);
+ cur_channels = conv_params->output_num;
+ if (conv_params->padding_method == VALID) {
+ int pad_size = (conv_params->kernel_size - 1) * conv_params->dilation;
+ cur_height -= pad_size;
+ cur_width -= pad_size;
+ }
+ break;
+ case DEPTH_TO_SPACE:
+ depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params;
+ depth_to_space(network->layers[layer - 1].output, network->layers[layer].output,
+ depth_to_space_params->block_size, cur_width, cur_height, cur_channels);
+ cur_height *= depth_to_space_params->block_size;
+ cur_width *= depth_to_space_params->block_size;
+ cur_channels /= depth_to_space_params->block_size * depth_to_space_params->block_size;
+ break;
+ case INPUT:
+ return DNN_ERROR;
+ }
+ }
+
+ // native mode does not support multiple outputs yet
+ if (nb_output > 1)
+ return DNN_ERROR;
+ outputs[0].data = network->layers[network->layers_num - 1].output;
+ outputs[0].height = cur_height;
+ outputs[0].width = cur_width;
+ outputs[0].channels = cur_channels;
+
+ return DNN_SUCCESS;
+}
+
+void ff_dnn_free_model_native(DNNModel **model)
+{
+ ConvolutionalNetwork *network;
+ ConvolutionalParams *conv_params;
+ int32_t layer;
+
+ if (*model)
+ {
+ network = (ConvolutionalNetwork *)(*model)->model;
+ for (layer = 0; layer < network->layers_num; ++layer){
+ av_freep(&network->layers[layer].output);
+ if (network->layers[layer].type == CONV){
+ conv_params = (ConvolutionalParams *)network->layers[layer].params;
+ av_freep(&conv_params->kernel);
+ av_freep(&conv_params->biases);
+ }
+ av_freep(&network->layers[layer].params);
+ }
+ av_freep(&network->layers);
+ av_freep(&network);
+ av_freep(model);
+ }
+}
diff --git a/libavfilter/dnn/dnn_backend_native.h b/libavfilter/dnn/dnn_backend_native.h
new file mode 100644
index 0000000..532103c
--- /dev/null
+++ b/libavfilter/dnn/dnn_backend_native.h
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2018 Sergey Lavrushkin
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file
+ * DNN inference functions interface for native backend.
+ */
+
+
+#ifndef AVFILTER_DNN_BACKEND_NATIVE_H
+#define AVFILTER_DNN_BACKEND_NATIVE_H
+
+#include "../dnn_interface.h"
+#include "libavformat/avio.h"
+
+typedef enum {INPUT, CONV, DEPTH_TO_SPACE} DNNLayerType;
+
+typedef enum {RELU, TANH, SIGMOID, NONE, LEAKY_RELU} DNNActivationFunc;
+
+typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} DNNConvPaddingParam;
+
+typedef struct Layer{
+ DNNLayerType type;
+ float *output;
+ void *params;
+} Layer;
+
+typedef struct ConvolutionalParams{
+ int32_t input_num, output_num, kernel_size;
+ DNNActivationFunc activation;
+ DNNConvPaddingParam padding_method;
+ int32_t dilation;
+ float *kernel;
+ float *biases;
+} ConvolutionalParams;
+
+typedef struct InputParams{
+ int height, width, channels;
+} InputParams;
+
+typedef struct DepthToSpaceParams{
+ int block_size;
+} DepthToSpaceParams;
+
+// Represents simple feed-forward convolutional network.
+typedef struct ConvolutionalNetwork{
+ Layer *layers;
+ int32_t layers_num;
+} ConvolutionalNetwork;
+
+DNNModel *ff_dnn_load_model_native(const char *model_filename);
+
+DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output);
+
+void ff_dnn_free_model_native(DNNModel **model);
+
+#endif
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
new file mode 100644
index 0000000..ba959ae
--- /dev/null
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -0,0 +1,603 @@
+/*
+ * Copyright (c) 2018 Sergey Lavrushkin
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file
+ * DNN tensorflow backend implementation.
+ */
+
+#include "dnn_backend_tf.h"
+#include "dnn_backend_native.h"
+#include "libavformat/avio.h"
+#include "libavutil/avassert.h"
+
+#include <tensorflow/c/c_api.h>
+
+typedef struct TFModel{
+ TF_Graph *graph;
+ TF_Session *session;
+ TF_Status *status;
+ TF_Output input;
+ TF_Tensor *input_tensor;
+ TF_Output *outputs;
+ TF_Tensor **output_tensors;
+ uint32_t nb_output;
+} TFModel;
+
+static void free_buffer(void *data, size_t length)
+{
+ av_freep(&data);
+}
+
+static TF_Buffer *read_graph(const char *model_filename)
+{
+ TF_Buffer *graph_buf;
+ unsigned char *graph_data = NULL;
+ AVIOContext *model_file_context;
+ long size, bytes_read;
+
+ if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
+ return NULL;
+ }
+
+ size = avio_size(model_file_context);
+
+ graph_data = av_malloc(size);
+ if (!graph_data){
+ avio_closep(&model_file_context);
+ return NULL;
+ }
+ bytes_read = avio_read(model_file_context, graph_data, size);
+ avio_closep(&model_file_context);
+ if (bytes_read != size){
+ av_freep(&graph_data);
+ return NULL;
+ }
+
+ graph_buf = TF_NewBuffer();
+ graph_buf->data = (void *)graph_data;
+ graph_buf->length = size;
+ graph_buf->data_deallocator = free_buffer;
+
+ return graph_buf;
+}
+
+static TF_Tensor *allocate_input_tensor(const DNNInputData *input)
+{
+ TF_DataType dt;
+ size_t size;
+ int64_t input_dims[] = {1, input->height, input->width, input->channels};
+ switch (input->dt) {
+ case DNN_FLOAT:
+ dt = TF_FLOAT;
+ size = sizeof(float);
+ break;
+ case DNN_UINT8:
+ dt = TF_UINT8;
+ size = sizeof(char);
+ break;
+ default:
+ av_assert0(!"should not reach here");
+ }
+
+ return TF_AllocateTensor(dt, input_dims, 4,
+ input_dims[1] * input_dims[2] * input_dims[3] * size);
+}
+
+static DNNReturnType set_input_output_tf(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output)
+{
+ TFModel *tf_model = (TFModel *)model;
+ TF_SessionOptions *sess_opts;
+ const TF_Operation *init_op = TF_GraphOperationByName(tf_model->graph, "init");
+
+ // Input operation
+ tf_model->input.oper = TF_GraphOperationByName(tf_model->graph, input_name);
+ if (!tf_model->input.oper){
+ return DNN_ERROR;
+ }
+ tf_model->input.index = 0;
+ if (tf_model->input_tensor){
+ TF_DeleteTensor(tf_model->input_tensor);
+ }
+ tf_model->input_tensor = allocate_input_tensor(input);
+ if (!tf_model->input_tensor){
+ return DNN_ERROR;
+ }
+ input->data = (float *)TF_TensorData(tf_model->input_tensor);
+
+ // Output operation
+ if (nb_output == 0)
+ return DNN_ERROR;
+
+ av_freep(&tf_model->outputs);
+ tf_model->outputs = av_malloc_array(nb_output, sizeof(*tf_model->outputs));
+ if (!tf_model->outputs)
+ return DNN_ERROR;
+ for (int i = 0; i < nb_output; ++i) {
+ tf_model->outputs[i].oper = TF_GraphOperationByName(tf_model->graph, output_names[i]);
+ if (!tf_model->outputs[i].oper){
+ av_freep(&tf_model->outputs);
+ return DNN_ERROR;
+ }
+ tf_model->outputs[i].index = 0;
+ }
+
+ if (tf_model->output_tensors) {
+ for (uint32_t i = 0; i < tf_model->nb_output; ++i) {
+ if (tf_model->output_tensors[i]) {
+ TF_DeleteTensor(tf_model->output_tensors[i]);
+ tf_model->output_tensors[i] = NULL;
+ }
+ }
+ }
+ av_freep(&tf_model->output_tensors);
+ tf_model->output_tensors = av_mallocz_array(nb_output, sizeof(*tf_model->output_tensors));
+ if (!tf_model->output_tensors) {
+ av_freep(&tf_model->outputs);
+ return DNN_ERROR;
+ }
+
+ tf_model->nb_output = nb_output;
+
+ if (tf_model->session){
+ TF_CloseSession(tf_model->session, tf_model->status);
+ TF_DeleteSession(tf_model->session, tf_model->status);
+ }
+
+ sess_opts = TF_NewSessionOptions();
+ tf_model->session = TF_NewSession(tf_model->graph, sess_opts, tf_model->status);
+ TF_DeleteSessionOptions(sess_opts);
+ if (TF_GetCode(tf_model->status) != TF_OK)
+ {
+ return DNN_ERROR;
+ }
+
+ // Run initialization operation with name "init" if it is present in graph
+ if (init_op){
+ TF_SessionRun(tf_model->session, NULL,
+ NULL, NULL, 0,
+ NULL, NULL, 0,
+ &init_op, 1, NULL, tf_model->status);
+ if (TF_GetCode(tf_model->status) != TF_OK)
+ {
+ return DNN_ERROR;
+ }
+ }
+
+ return DNN_SUCCESS;
+}
+
+static DNNReturnType load_tf_model(TFModel *tf_model, const char *model_filename)
+{
+ TF_Buffer *graph_def;
+ TF_ImportGraphDefOptions *graph_opts;
+
+ graph_def = read_graph(model_filename);
+ if (!graph_def){
+ return DNN_ERROR;
+ }
+ tf_model->graph = TF_NewGraph();
+ tf_model->status = TF_NewStatus();
+ graph_opts = TF_NewImportGraphDefOptions();
+ TF_GraphImportGraphDef(tf_model->graph, graph_def, graph_opts, tf_model->status);
+ TF_DeleteImportGraphDefOptions(graph_opts);
+ TF_DeleteBuffer(graph_def);
+ if (TF_GetCode(tf_model->status) != TF_OK){
+ TF_DeleteGraph(tf_model->graph);
+ TF_DeleteStatus(tf_model->status);
+ return DNN_ERROR;
+ }
+
+ return DNN_SUCCESS;
+}
+
+#define NAME_BUFFER_SIZE 256
+
+static DNNReturnType add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op,
+ ConvolutionalParams* params, const int layer)
+{
+ TF_Operation *op;
+ TF_OperationDescription *op_desc;
+ TF_Output input;
+ int64_t strides[] = {1, 1, 1, 1};
+ TF_Tensor *tensor;
+ int64_t dims[4];
+ int dims_len;
+ char name_buffer[NAME_BUFFER_SIZE];
+ int32_t size;
+
+ size = params->input_num * params->output_num * params->kernel_size * params->kernel_size;
+ input.index = 0;
+
+ snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_kernel%d", layer);
+ op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
+ TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
+ dims[0] = params->output_num;
+ dims[1] = params->kernel_size;
+ dims[2] = params->kernel_size;
+ dims[3] = params->input_num;
+ dims_len = 4;
+ tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, size * sizeof(float));
+ memcpy(TF_TensorData(tensor), params->kernel, size * sizeof(float));
+ TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
+ if (TF_GetCode(tf_model->status) != TF_OK){
+ return DNN_ERROR;
+ }
+ op = TF_FinishOperation(op_desc, tf_model->status);
+ if (TF_GetCode(tf_model->status) != TF_OK){
+ return DNN_ERROR;
+ }
+
+ snprintf(name_buffer, NAME_BUFFER_SIZE, "transpose%d", layer);
+ op_desc = TF_NewOperation(tf_model->graph, "Transpose", name_buffer);
+ input.oper = op;
+ TF_AddInput(op_desc, input);
+ input.oper = transpose_op;
+ TF_AddInput(op_desc, input);
+ TF_SetAttrType(op_desc, "T", TF_FLOAT);
+ TF_SetAttrType(op_desc, "Tperm", TF_INT32);
+ op = TF_FinishOperation(op_desc, tf_model->status);
+ if (TF_GetCode(tf_model->status) != TF_OK){
+ return DNN_ERROR;
+ }
+
+ snprintf(name_buffer, NAME_BUFFER_SIZE, "conv2d%d", layer);
+ op_desc = TF_NewOperation(tf_model->graph, "Conv2D", name_buffer);
+ input.oper = *cur_op;
+ TF_AddInput(op_desc, input);
+ input.oper = op;
+ TF_AddInput(op_desc, input);
+ TF_SetAttrType(op_desc, "T", TF_FLOAT);
+ TF_SetAttrIntList(op_desc, "strides", strides, 4);
+ TF_SetAttrString(op_desc, "padding", "VALID", 5);
+ *cur_op = TF_FinishOperation(op_desc, tf_model->status);
+ if (TF_GetCode(tf_model->status) != TF_OK){
+ return DNN_ERROR;
+ }
+
+ snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_biases%d", layer);
+ op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
+ TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
+ dims[0] = params->output_num;
+ dims_len = 1;
+ tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, params->output_num * sizeof(float));
+ memcpy(TF_TensorData(tensor), params->biases, params->output_num * sizeof(float));
+ TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
+ if (TF_GetCode(tf_model->status) != TF_OK){
+ return DNN_ERROR;
+ }
+ op = TF_FinishOperation(op_desc, tf_model->status);
+ if (TF_GetCode(tf_model->status) != TF_OK){
+ return DNN_ERROR;
+ }
+
+ snprintf(name_buffer, NAME_BUFFER_SIZE, "bias_add%d", layer);
+ op_desc = TF_NewOperation(tf_model->graph, "BiasAdd", name_buffer);
+ input.oper = *cur_op;
+ TF_AddInput(op_desc, input);
+ input.oper = op;
+ TF_AddInput(op_desc, input);
+ TF_SetAttrType(op_desc, "T", TF_FLOAT);
+ *cur_op = TF_FinishOperation(op_desc, tf_model->status);
+ if (TF_GetCode(tf_model->status) != TF_OK){
+ return DNN_ERROR;
+ }
+
+ snprintf(name_buffer, NAME_BUFFER_SIZE, "activation%d", layer);
+ switch (params->activation){
+ case RELU:
+ op_desc = TF_NewOperation(tf_model->graph, "Relu", name_buffer);
+ break;
+ case TANH:
+ op_desc = TF_NewOperation(tf_model->graph, "Tanh", name_buffer);
+ break;
+ case SIGMOID:
+ op_desc = TF_NewOperation(tf_model->graph, "Sigmoid", name_buffer);
+ break;
+ default:
+ return DNN_ERROR;
+ }
+ input.oper = *cur_op;
+ TF_AddInput(op_desc, input);
+ TF_SetAttrType(op_desc, "T", TF_FLOAT);
+ *cur_op = TF_FinishOperation(op_desc, tf_model->status);
+ if (TF_GetCode(tf_model->status) != TF_OK){
+ return DNN_ERROR;
+ }
+
+ return DNN_SUCCESS;
+}
+
+static DNNReturnType add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op,
+ DepthToSpaceParams *params, const int layer)
+{
+ TF_OperationDescription *op_desc;
+ TF_Output input;
+ char name_buffer[NAME_BUFFER_SIZE];
+
+ snprintf(name_buffer, NAME_BUFFER_SIZE, "depth_to_space%d", layer);
+ op_desc = TF_NewOperation(tf_model->graph, "DepthToSpace", name_buffer);
+ input.oper = *cur_op;
+ input.index = 0;
+ TF_AddInput(op_desc, input);
+ TF_SetAttrType(op_desc, "T", TF_FLOAT);
+ TF_SetAttrInt(op_desc, "block_size", params->block_size);
+ *cur_op = TF_FinishOperation(op_desc, tf_model->status);
+ if (TF_GetCode(tf_model->status) != TF_OK){
+ return DNN_ERROR;
+ }
+
+ return DNN_SUCCESS;
+}
+
+static int calculate_pad(const ConvolutionalNetwork *conv_network)
+{
+ ConvolutionalParams *params;
+ int32_t layer;
+ int pad = 0;
+
+ for (layer = 0; layer < conv_network->layers_num; ++layer){
+ if (conv_network->layers[layer].type == CONV){
+ params = (ConvolutionalParams *)conv_network->layers[layer].params;
+ pad += params->kernel_size >> 1;
+ }
+ }
+
+ return pad;
+}
+
+static DNNReturnType add_pad_op(TFModel *tf_model, TF_Operation **cur_op, const int32_t pad)
+{
+ TF_Operation *op;
+ TF_Tensor *tensor;
+ TF_OperationDescription *op_desc;
+ TF_Output input;
+ int32_t *pads;
+ int64_t pads_shape[] = {4, 2};
+
+ input.index = 0;
+
+ op_desc = TF_NewOperation(tf_model->graph, "Const", "pads");
+ TF_SetAttrType(op_desc, "dtype", TF_INT32);
+ tensor = TF_AllocateTensor(TF_INT32, pads_shape, 2, 4 * 2 * sizeof(int32_t));
+ pads = (int32_t *)TF_TensorData(tensor);
+ pads[0] = 0; pads[1] = 0;
+ pads[2] = pad; pads[3] = pad;
+ pads[4] = pad; pads[5] = pad;
+ pads[6] = 0; pads[7] = 0;
+ TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
+ if (TF_GetCode(tf_model->status) != TF_OK){
+ return DNN_ERROR;
+ }
+ op = TF_FinishOperation(op_desc, tf_model->status);
+ if (TF_GetCode(tf_model->status) != TF_OK){
+ return DNN_ERROR;
+ }
+
+ op_desc = TF_NewOperation(tf_model->graph, "MirrorPad", "mirror_pad");
+ input.oper = *cur_op;
+ TF_AddInput(op_desc, input);
+ input.oper = op;
+ TF_AddInput(op_desc, input);
+ TF_SetAttrType(op_desc, "T", TF_FLOAT);
+ TF_SetAttrType(op_desc, "Tpaddings", TF_INT32);
+ TF_SetAttrString(op_desc, "mode", "SYMMETRIC", 9);
+ *cur_op = TF_FinishOperation(op_desc, tf_model->status);
+ if (TF_GetCode(tf_model->status) != TF_OK){
+ return DNN_ERROR;
+ }
+
+ return DNN_SUCCESS;
+}
+
+static DNNReturnType load_native_model(TFModel *tf_model, const char *model_filename)
+{
+ int32_t layer;
+ TF_OperationDescription *op_desc;
+ TF_Operation *op;
+ TF_Operation *transpose_op;
+ TF_Tensor *tensor;
+ TF_Output input;
+ int32_t *transpose_perm;
+ int64_t transpose_perm_shape[] = {4};
+ int64_t input_shape[] = {1, -1, -1, -1};
+ int32_t pad;
+ DNNReturnType layer_add_res;
+ DNNModel *native_model = NULL;
+ ConvolutionalNetwork *conv_network;
+
+ native_model = ff_dnn_load_model_native(model_filename);
+ if (!native_model){
+ return DNN_ERROR;
+ }
+
+ conv_network = (ConvolutionalNetwork *)native_model->model;
+ pad = calculate_pad(conv_network);
+ tf_model->graph = TF_NewGraph();
+ tf_model->status = TF_NewStatus();
+
+#define CLEANUP_ON_ERROR(tf_model) \
+ { \
+ TF_DeleteGraph(tf_model->graph); \
+ TF_DeleteStatus(tf_model->status); \
+ return DNN_ERROR; \
+ }
+
+ op_desc = TF_NewOperation(tf_model->graph, "Placeholder", "x");
+ TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
+ TF_SetAttrShape(op_desc, "shape", input_shape, 4);
+ op = TF_FinishOperation(op_desc, tf_model->status);
+ if (TF_GetCode(tf_model->status) != TF_OK){
+ CLEANUP_ON_ERROR(tf_model);
+ }
+
+ if (add_pad_op(tf_model, &op, pad) != DNN_SUCCESS){
+ CLEANUP_ON_ERROR(tf_model);
+ }
+
+ op_desc = TF_NewOperation(tf_model->graph, "Const", "transpose_perm");
+ TF_SetAttrType(op_desc, "dtype", TF_INT32);
+ tensor = TF_AllocateTensor(TF_INT32, transpose_perm_shape, 1, 4 * sizeof(int32_t));
+ transpose_perm = (int32_t *)TF_TensorData(tensor);
+ transpose_perm[0] = 1;
+ transpose_perm[1] = 2;
+ transpose_perm[2] = 3;
+ transpose_perm[3] = 0;
+ TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
+ if (TF_GetCode(tf_model->status) != TF_OK){
+ CLEANUP_ON_ERROR(tf_model);
+ }
+ transpose_op = TF_FinishOperation(op_desc, tf_model->status);
+
+ for (layer = 0; layer < conv_network->layers_num; ++layer){
+ switch (conv_network->layers[layer].type){
+ case INPUT:
+ layer_add_res = DNN_SUCCESS;
+ break;
+ case CONV:
+ layer_add_res = add_conv_layer(tf_model, transpose_op, &op,
+ (ConvolutionalParams *)conv_network->layers[layer].params, layer);
+ break;
+ case DEPTH_TO_SPACE:
+ layer_add_res = add_depth_to_space_layer(tf_model, &op,
+ (DepthToSpaceParams *)conv_network->layers[layer].params, layer);
+ break;
+ default:
+ CLEANUP_ON_ERROR(tf_model);
+ }
+
+ if (layer_add_res != DNN_SUCCESS){
+ CLEANUP_ON_ERROR(tf_model);
+ }
+ }
+
+ op_desc = TF_NewOperation(tf_model->graph, "Identity", "y");
+ input.oper = op;
+ TF_AddInput(op_desc, input);
+ TF_FinishOperation(op_desc, tf_model->status);
+ if (TF_GetCode(tf_model->status) != TF_OK){
+ CLEANUP_ON_ERROR(tf_model);
+ }
+
+ ff_dnn_free_model_native(&native_model);
+
+ return DNN_SUCCESS;
+}
+
+DNNModel *ff_dnn_load_model_tf(const char *model_filename)
+{
+ DNNModel *model = NULL;
+ TFModel *tf_model = NULL;
+
+ model = av_malloc(sizeof(DNNModel));
+ if (!model){
+ return NULL;
+ }
+
+ tf_model = av_mallocz(sizeof(TFModel));
+ if (!tf_model){
+ av_freep(&model);
+ return NULL;
+ }
+
+ if (load_tf_model(tf_model, model_filename) != DNN_SUCCESS){
+ if (load_native_model(tf_model, model_filename) != DNN_SUCCESS){
+ av_freep(&tf_model);
+ av_freep(&model);
+
+ return NULL;
+ }
+ }
+
+ model->model = (void *)tf_model;
+ model->set_input_output = &set_input_output_tf;
+
+ return model;
+}
+
+
+
+DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, uint32_t nb_output)
+{
+ TFModel *tf_model = (TFModel *)model->model;
+ uint32_t nb = FFMIN(nb_output, tf_model->nb_output);
+ if (nb == 0)
+ return DNN_ERROR;
+
+ av_assert0(tf_model->output_tensors);
+ for (uint32_t i = 0; i < tf_model->nb_output; ++i) {
+ if (tf_model->output_tensors[i]) {
+ TF_DeleteTensor(tf_model->output_tensors[i]);
+ tf_model->output_tensors[i] = NULL;
+ }
+ }
+
+ TF_SessionRun(tf_model->session, NULL,
+ &tf_model->input, &tf_model->input_tensor, 1,
+ tf_model->outputs, tf_model->output_tensors, nb,
+ NULL, 0, NULL, tf_model->status);
+
+ if (TF_GetCode(tf_model->status) != TF_OK){
+ return DNN_ERROR;
+ }
+
+ for (uint32_t i = 0; i < nb; ++i) {
+ outputs[i].height = TF_Dim(tf_model->output_tensors[i], 1);
+ outputs[i].width = TF_Dim(tf_model->output_tensors[i], 2);
+ outputs[i].channels = TF_Dim(tf_model->output_tensors[i], 3);
+ outputs[i].data = TF_TensorData(tf_model->output_tensors[i]);
+ }
+
+ return DNN_SUCCESS;
+}
+
+void ff_dnn_free_model_tf(DNNModel **model)
+{
+ TFModel *tf_model;
+
+ if (*model){
+ tf_model = (TFModel *)(*model)->model;
+ if (tf_model->graph){
+ TF_DeleteGraph(tf_model->graph);
+ }
+ if (tf_model->session){
+ TF_CloseSession(tf_model->session, tf_model->status);
+ TF_DeleteSession(tf_model->session, tf_model->status);
+ }
+ if (tf_model->status){
+ TF_DeleteStatus(tf_model->status);
+ }
+ if (tf_model->input_tensor){
+ TF_DeleteTensor(tf_model->input_tensor);
+ }
+ if (tf_model->output_tensors) {
+ for (uint32_t i = 0; i < tf_model->nb_output; ++i) {
+ if (tf_model->output_tensors[i]) {
+ TF_DeleteTensor(tf_model->output_tensors[i]);
+ tf_model->output_tensors[i] = NULL;
+ }
+ }
+ }
+ av_freep(&tf_model->outputs);
+ av_freep(&tf_model->output_tensors);
+ av_freep(&tf_model);
+ av_freep(model);
+ }
+}
diff --git a/libavfilter/dnn/dnn_backend_tf.h b/libavfilter/dnn/dnn_backend_tf.h
new file mode 100644
index 0000000..bb1c85f
--- /dev/null
+++ b/libavfilter/dnn/dnn_backend_tf.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2018 Sergey Lavrushkin
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file
+ * DNN inference functions interface for TensorFlow backend.
+ */
+
+
+#ifndef AVFILTER_DNN_BACKEND_TF_H
+#define AVFILTER_DNN_BACKEND_TF_H
+
+#include "../dnn_interface.h"
+
+DNNModel *ff_dnn_load_model_tf(const char *model_filename);
+
+DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, uint32_t nb_output);
+
+void ff_dnn_free_model_tf(DNNModel **model);
+
+#endif
diff --git a/libavfilter/dnn/dnn_interface.c b/libavfilter/dnn/dnn_interface.c
new file mode 100644
index 0000000..62da55f
--- /dev/null
+++ b/libavfilter/dnn/dnn_interface.c
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2018 Sergey Lavrushkin
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file
+ * Implements DNN module initialization with specified backend.
+ */
+
+#include "../dnn_interface.h"
+#include "dnn_backend_native.h"
+#include "dnn_backend_tf.h"
+#include "libavutil/mem.h"
+
+DNNModule *ff_get_dnn_module(DNNBackendType backend_type)
+{
+ DNNModule *dnn_module;
+
+ dnn_module = av_malloc(sizeof(DNNModule));
+ if(!dnn_module){
+ return NULL;
+ }
+
+ switch(backend_type){
+ case DNN_NATIVE:
+ dnn_module->load_model = &ff_dnn_load_model_native;
+ dnn_module->execute_model = &ff_dnn_execute_model_native;
+ dnn_module->free_model = &ff_dnn_free_model_native;
+ break;
+ case DNN_TF:
+ #if (CONFIG_LIBTENSORFLOW == 1)
+ dnn_module->load_model = &ff_dnn_load_model_tf;
+ dnn_module->execute_model = &ff_dnn_execute_model_tf;
+ dnn_module->free_model = &ff_dnn_free_model_tf;
+ #else
+ av_freep(&dnn_module);
+ return NULL;
+ #endif
+ break;
+ default:
+ av_log(NULL, AV_LOG_ERROR, "Module backend_type is not native or tensorflow\n");
+ av_freep(&dnn_module);
+ return NULL;
+ }
+
+ return dnn_module;
+}
diff --git a/libavfilter/dnn_backend_native.c b/libavfilter/dnn_backend_native.c
deleted file mode 100644
index 82e900b..0000000
--- a/libavfilter/dnn_backend_native.c
+++ /dev/null
@@ -1,389 +0,0 @@
-/*
- * Copyright (c) 2018 Sergey Lavrushkin
- *
- * This file is part of FFmpeg.
- *
- * FFmpeg is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * FFmpeg is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with FFmpeg; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-/**
- * @file
- * DNN native backend implementation.
- */
-
-#include "dnn_backend_native.h"
-#include "libavutil/avassert.h"
-
-static DNNReturnType set_input_output_native(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output)
-{
- ConvolutionalNetwork *network = (ConvolutionalNetwork *)model;
- InputParams *input_params;
- ConvolutionalParams *conv_params;
- DepthToSpaceParams *depth_to_space_params;
- int cur_width, cur_height, cur_channels;
- int32_t layer;
-
- if (network->layers_num <= 0 || network->layers[0].type != INPUT){
- return DNN_ERROR;
- }
- else{
- input_params = (InputParams *)network->layers[0].params;
- input_params->width = cur_width = input->width;
- input_params->height = cur_height = input->height;
- input_params->channels = cur_channels = input->channels;
- if (input->data){
- av_freep(&input->data);
- }
- av_assert0(input->dt == DNN_FLOAT);
- network->layers[0].output = input->data = av_malloc(cur_height * cur_width * cur_channels * sizeof(float));
- if (!network->layers[0].output){
- return DNN_ERROR;
- }
- }
-
- for (layer = 1; layer < network->layers_num; ++layer){
- switch (network->layers[layer].type){
- case CONV:
- conv_params = (ConvolutionalParams *)network->layers[layer].params;
- if (conv_params->input_num != cur_channels){
- return DNN_ERROR;
- }
- cur_channels = conv_params->output_num;
-
- if (conv_params->padding_method == VALID) {
- int pad_size = (conv_params->kernel_size - 1) * conv_params->dilation;
- cur_height -= pad_size;
- cur_width -= pad_size;
- }
- break;
- case DEPTH_TO_SPACE:
- depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params;
- if (cur_channels % (depth_to_space_params->block_size * depth_to_space_params->block_size) != 0){
- return DNN_ERROR;
- }
- cur_channels = cur_channels / (depth_to_space_params->block_size * depth_to_space_params->block_size);
- cur_height *= depth_to_space_params->block_size;
- cur_width *= depth_to_space_params->block_size;
- break;
- default:
- return DNN_ERROR;
- }
- if (network->layers[layer].output){
- av_freep(&network->layers[layer].output);
- }
-
- if (cur_height <= 0 || cur_width <= 0)
- return DNN_ERROR;
-
- network->layers[layer].output = av_malloc(cur_height * cur_width * cur_channels * sizeof(float));
- if (!network->layers[layer].output){
- return DNN_ERROR;
- }
- }
-
- return DNN_SUCCESS;
-}
-
-// Loads model and its parameters that are stored in a binary file with following structure:
-// layers_num,layer_type,layer_parameterss,layer_type,layer_parameters...
-// For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases
-// For DEPTH_TO_SPACE layer: block_size
-DNNModel *ff_dnn_load_model_native(const char *model_filename)
-{
- DNNModel *model = NULL;
- ConvolutionalNetwork *network = NULL;
- AVIOContext *model_file_context;
- int file_size, dnn_size, kernel_size, i;
- int32_t layer;
- DNNLayerType layer_type;
- ConvolutionalParams *conv_params;
- DepthToSpaceParams *depth_to_space_params;
-
- model = av_malloc(sizeof(DNNModel));
- if (!model){
- return NULL;
- }
-
- if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
- av_freep(&model);
- return NULL;
- }
- file_size = avio_size(model_file_context);
-
- network = av_malloc(sizeof(ConvolutionalNetwork));
- if (!network){
- avio_closep(&model_file_context);
- av_freep(&model);
- return NULL;
- }
- model->model = (void *)network;
-
- network->layers_num = 1 + (int32_t)avio_rl32(model_file_context);
- dnn_size = 4;
-
- network->layers = av_malloc(network->layers_num * sizeof(Layer));
- if (!network->layers){
- av_freep(&network);
- avio_closep(&model_file_context);
- av_freep(&model);
- return NULL;
- }
-
- for (layer = 0; layer < network->layers_num; ++layer){
- network->layers[layer].output = NULL;
- network->layers[layer].params = NULL;
- }
- network->layers[0].type = INPUT;
- network->layers[0].params = av_malloc(sizeof(InputParams));
- if (!network->layers[0].params){
- avio_closep(&model_file_context);
- ff_dnn_free_model_native(&model);
- return NULL;
- }
-
- for (layer = 1; layer < network->layers_num; ++layer){
- layer_type = (int32_t)avio_rl32(model_file_context);
- dnn_size += 4;
- switch (layer_type){
- case CONV:
- conv_params = av_malloc(sizeof(ConvolutionalParams));
- if (!conv_params){
- avio_closep(&model_file_context);
- ff_dnn_free_model_native(&model);
- return NULL;
- }
- conv_params->dilation = (int32_t)avio_rl32(model_file_context);
- conv_params->padding_method = (int32_t)avio_rl32(model_file_context);
- conv_params->activation = (int32_t)avio_rl32(model_file_context);
- conv_params->input_num = (int32_t)avio_rl32(model_file_context);
- conv_params->output_num = (int32_t)avio_rl32(model_file_context);
- conv_params->kernel_size = (int32_t)avio_rl32(model_file_context);
- kernel_size = conv_params->input_num * conv_params->output_num *
- conv_params->kernel_size * conv_params->kernel_size;
- dnn_size += 24 + (kernel_size + conv_params->output_num << 2);
- if (dnn_size > file_size || conv_params->input_num <= 0 ||
- conv_params->output_num <= 0 || conv_params->kernel_size <= 0){
- avio_closep(&model_file_context);
- ff_dnn_free_model_native(&model);
- return NULL;
- }
- conv_params->kernel = av_malloc(kernel_size * sizeof(float));
- conv_params->biases = av_malloc(conv_params->output_num * sizeof(float));
- if (!conv_params->kernel || !conv_params->biases){
- avio_closep(&model_file_context);
- ff_dnn_free_model_native(&model);
- return NULL;
- }
- for (i = 0; i < kernel_size; ++i){
- conv_params->kernel[i] = av_int2float(avio_rl32(model_file_context));
- }
- for (i = 0; i < conv_params->output_num; ++i){
- conv_params->biases[i] = av_int2float(avio_rl32(model_file_context));
- }
- network->layers[layer].type = CONV;
- network->layers[layer].params = conv_params;
- break;
- case DEPTH_TO_SPACE:
- depth_to_space_params = av_malloc(sizeof(DepthToSpaceParams));
- if (!depth_to_space_params){
- avio_closep(&model_file_context);
- ff_dnn_free_model_native(&model);
- return NULL;
- }
- depth_to_space_params->block_size = (int32_t)avio_rl32(model_file_context);
- dnn_size += 4;
- network->layers[layer].type = DEPTH_TO_SPACE;
- network->layers[layer].params = depth_to_space_params;
- break;
- default:
- avio_closep(&model_file_context);
- ff_dnn_free_model_native(&model);
- return NULL;
- }
- }
-
- avio_closep(&model_file_context);
-
- if (dnn_size != file_size){
- ff_dnn_free_model_native(&model);
- return NULL;
- }
-
- model->set_input_output = &set_input_output_native;
-
- return model;
-}
-
-#define CLAMP_TO_EDGE(x, w) ((x) < 0 ? 0 : ((x) >= (w) ? (w - 1) : (x)))
-
-static void convolve(const float *input, float *output, const ConvolutionalParams *conv_params, int width, int height)
-{
- int radius = conv_params->kernel_size >> 1;
- int src_linesize = width * conv_params->input_num;
- int filter_linesize = conv_params->kernel_size * conv_params->input_num;
- int filter_size = conv_params->kernel_size * filter_linesize;
- int pad_size = (conv_params->padding_method == VALID) ? (conv_params->kernel_size - 1) / 2 * conv_params->dilation : 0;
-
- for (int y = pad_size; y < height - pad_size; ++y) {
- for (int x = pad_size; x < width - pad_size; ++x) {
- for (int n_filter = 0; n_filter < conv_params->output_num; ++n_filter) {
- output[n_filter] = conv_params->biases[n_filter];
-
- for (int ch = 0; ch < conv_params->input_num; ++ch) {
- for (int kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y) {
- for (int kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x) {
- float input_pel;
- if (conv_params->padding_method == SAME_CLAMP_TO_EDGE) {
- int y_pos = CLAMP_TO_EDGE(y + (kernel_y - radius) * conv_params->dilation, height);
- int x_pos = CLAMP_TO_EDGE(x + (kernel_x - radius) * conv_params->dilation, width);
- input_pel = input[y_pos * src_linesize + x_pos * conv_params->input_num + ch];
- } else {
- int y_pos = y + (kernel_y - radius) * conv_params->dilation;
- int x_pos = x + (kernel_x - radius) * conv_params->dilation;
- input_pel = (x_pos < 0 || x_pos >= width || y_pos < 0 || y_pos >= height) ? 0.0 :
- input[y_pos * src_linesize + x_pos * conv_params->input_num + ch];
- }
-
-
- output[n_filter] += input_pel * conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize +
- kernel_x * conv_params->input_num + ch];
- }
- }
- }
- switch (conv_params->activation){
- case RELU:
- output[n_filter] = FFMAX(output[n_filter], 0.0);
- break;
- case TANH:
- output[n_filter] = 2.0f / (1.0f + exp(-2.0f * output[n_filter])) - 1.0f;
- break;
- case SIGMOID:
- output[n_filter] = 1.0f / (1.0f + exp(-output[n_filter]));
- break;
- case NONE:
- break;
- case LEAKY_RELU:
- output[n_filter] = FFMAX(output[n_filter], 0.0) + 0.2 * FFMIN(output[n_filter], 0.0);
- }
- }
- output += conv_params->output_num;
- }
- }
-}
-
-static void depth_to_space(const float *input, float *output, int block_size, int width, int height, int channels)
-{
- int y, x, by, bx, ch;
- int new_channels = channels / (block_size * block_size);
- int output_linesize = width * channels;
- int by_linesize = output_linesize / block_size;
- int x_linesize = new_channels * block_size;
-
- for (y = 0; y < height; ++y){
- for (x = 0; x < width; ++x){
- for (by = 0; by < block_size; ++by){
- for (bx = 0; bx < block_size; ++bx){
- for (ch = 0; ch < new_channels; ++ch){
- output[by * by_linesize + x * x_linesize + bx * new_channels + ch] = input[ch];
- }
- input += new_channels;
- }
- }
- }
- output += output_linesize;
- }
-}
-
-DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output)
-{
- ConvolutionalNetwork *network = (ConvolutionalNetwork *)model->model;
- int cur_width, cur_height, cur_channels;
- int32_t layer;
- InputParams *input_params;
- ConvolutionalParams *conv_params;
- DepthToSpaceParams *depth_to_space_params;
-
- if (network->layers_num <= 0 || network->layers[0].type != INPUT || !network->layers[0].output){
- return DNN_ERROR;
- }
- else{
- input_params = (InputParams *)network->layers[0].params;
- cur_width = input_params->width;
- cur_height = input_params->height;
- cur_channels = input_params->channels;
- }
-
- for (layer = 1; layer < network->layers_num; ++layer){
- if (!network->layers[layer].output){
- return DNN_ERROR;
- }
- switch (network->layers[layer].type){
- case CONV:
- conv_params = (ConvolutionalParams *)network->layers[layer].params;
- convolve(network->layers[layer - 1].output, network->layers[layer].output, conv_params, cur_width, cur_height);
- cur_channels = conv_params->output_num;
- if (conv_params->padding_method == VALID) {
- int pad_size = (conv_params->kernel_size - 1) * conv_params->dilation;
- cur_height -= pad_size;
- cur_width -= pad_size;
- }
- break;
- case DEPTH_TO_SPACE:
- depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params;
- depth_to_space(network->layers[layer - 1].output, network->layers[layer].output,
- depth_to_space_params->block_size, cur_width, cur_height, cur_channels);
- cur_height *= depth_to_space_params->block_size;
- cur_width *= depth_to_space_params->block_size;
- cur_channels /= depth_to_space_params->block_size * depth_to_space_params->block_size;
- break;
- case INPUT:
- return DNN_ERROR;
- }
- }
-
- // native mode does not support multiple outputs yet
- if (nb_output > 1)
- return DNN_ERROR;
- outputs[0].data = network->layers[network->layers_num - 1].output;
- outputs[0].height = cur_height;
- outputs[0].width = cur_width;
- outputs[0].channels = cur_channels;
-
- return DNN_SUCCESS;
-}
-
-void ff_dnn_free_model_native(DNNModel **model)
-{
- ConvolutionalNetwork *network;
- ConvolutionalParams *conv_params;
- int32_t layer;
-
- if (*model)
- {
- network = (ConvolutionalNetwork *)(*model)->model;
- for (layer = 0; layer < network->layers_num; ++layer){
- av_freep(&network->layers[layer].output);
- if (network->layers[layer].type == CONV){
- conv_params = (ConvolutionalParams *)network->layers[layer].params;
- av_freep(&conv_params->kernel);
- av_freep(&conv_params->biases);
- }
- av_freep(&network->layers[layer].params);
- }
- av_freep(&network->layers);
- av_freep(&network);
- av_freep(model);
- }
-}
diff --git a/libavfilter/dnn_backend_native.h b/libavfilter/dnn_backend_native.h
deleted file mode 100644
index 5917955..0000000
--- a/libavfilter/dnn_backend_native.h
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Copyright (c) 2018 Sergey Lavrushkin
- *
- * This file is part of FFmpeg.
- *
- * FFmpeg is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * FFmpeg is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with FFmpeg; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-/**
- * @file
- * DNN inference functions interface for native backend.
- */
-
-
-#ifndef AVFILTER_DNN_BACKEND_NATIVE_H
-#define AVFILTER_DNN_BACKEND_NATIVE_H
-
-#include "dnn_interface.h"
-#include "libavformat/avio.h"
-
-typedef enum {INPUT, CONV, DEPTH_TO_SPACE} DNNLayerType;
-
-typedef enum {RELU, TANH, SIGMOID, NONE, LEAKY_RELU} DNNActivationFunc;
-
-typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} DNNConvPaddingParam;
-
-typedef struct Layer{
- DNNLayerType type;
- float *output;
- void *params;
-} Layer;
-
-typedef struct ConvolutionalParams{
- int32_t input_num, output_num, kernel_size;
- DNNActivationFunc activation;
- DNNConvPaddingParam padding_method;
- int32_t dilation;
- float *kernel;
- float *biases;
-} ConvolutionalParams;
-
-typedef struct InputParams{
- int height, width, channels;
-} InputParams;
-
-typedef struct DepthToSpaceParams{
- int block_size;
-} DepthToSpaceParams;
-
-// Represents simple feed-forward convolutional network.
-typedef struct ConvolutionalNetwork{
- Layer *layers;
- int32_t layers_num;
-} ConvolutionalNetwork;
-
-DNNModel *ff_dnn_load_model_native(const char *model_filename);
-
-DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output);
-
-void ff_dnn_free_model_native(DNNModel **model);
-
-#endif
diff --git a/libavfilter/dnn_backend_tf.c b/libavfilter/dnn_backend_tf.c
deleted file mode 100644
index ba959ae..0000000
--- a/libavfilter/dnn_backend_tf.c
+++ /dev/null
@@ -1,603 +0,0 @@
-/*
- * Copyright (c) 2018 Sergey Lavrushkin
- *
- * This file is part of FFmpeg.
- *
- * FFmpeg is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * FFmpeg is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with FFmpeg; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-/**
- * @file
- * DNN tensorflow backend implementation.
- */
-
-#include "dnn_backend_tf.h"
-#include "dnn_backend_native.h"
-#include "libavformat/avio.h"
-#include "libavutil/avassert.h"
-
-#include <tensorflow/c/c_api.h>
-
-typedef struct TFModel{
- TF_Graph *graph;
- TF_Session *session;
- TF_Status *status;
- TF_Output input;
- TF_Tensor *input_tensor;
- TF_Output *outputs;
- TF_Tensor **output_tensors;
- uint32_t nb_output;
-} TFModel;
-
-static void free_buffer(void *data, size_t length)
-{
- av_freep(&data);
-}
-
-static TF_Buffer *read_graph(const char *model_filename)
-{
- TF_Buffer *graph_buf;
- unsigned char *graph_data = NULL;
- AVIOContext *model_file_context;
- long size, bytes_read;
-
- if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
- return NULL;
- }
-
- size = avio_size(model_file_context);
-
- graph_data = av_malloc(size);
- if (!graph_data){
- avio_closep(&model_file_context);
- return NULL;
- }
- bytes_read = avio_read(model_file_context, graph_data, size);
- avio_closep(&model_file_context);
- if (bytes_read != size){
- av_freep(&graph_data);
- return NULL;
- }
-
- graph_buf = TF_NewBuffer();
- graph_buf->data = (void *)graph_data;
- graph_buf->length = size;
- graph_buf->data_deallocator = free_buffer;
-
- return graph_buf;
-}
-
-static TF_Tensor *allocate_input_tensor(const DNNInputData *input)
-{
- TF_DataType dt;
- size_t size;
- int64_t input_dims[] = {1, input->height, input->width, input->channels};
- switch (input->dt) {
- case DNN_FLOAT:
- dt = TF_FLOAT;
- size = sizeof(float);
- break;
- case DNN_UINT8:
- dt = TF_UINT8;
- size = sizeof(char);
- break;
- default:
- av_assert0(!"should not reach here");
- }
-
- return TF_AllocateTensor(dt, input_dims, 4,
- input_dims[1] * input_dims[2] * input_dims[3] * size);
-}
-
-static DNNReturnType set_input_output_tf(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output)
-{
- TFModel *tf_model = (TFModel *)model;
- TF_SessionOptions *sess_opts;
- const TF_Operation *init_op = TF_GraphOperationByName(tf_model->graph, "init");
-
- // Input operation
- tf_model->input.oper = TF_GraphOperationByName(tf_model->graph, input_name);
- if (!tf_model->input.oper){
- return DNN_ERROR;
- }
- tf_model->input.index = 0;
- if (tf_model->input_tensor){
- TF_DeleteTensor(tf_model->input_tensor);
- }
- tf_model->input_tensor = allocate_input_tensor(input);
- if (!tf_model->input_tensor){
- return DNN_ERROR;
- }
- input->data = (float *)TF_TensorData(tf_model->input_tensor);
-
- // Output operation
- if (nb_output == 0)
- return DNN_ERROR;
-
- av_freep(&tf_model->outputs);
- tf_model->outputs = av_malloc_array(nb_output, sizeof(*tf_model->outputs));
- if (!tf_model->outputs)
- return DNN_ERROR;
- for (int i = 0; i < nb_output; ++i) {
- tf_model->outputs[i].oper = TF_GraphOperationByName(tf_model->graph, output_names[i]);
- if (!tf_model->outputs[i].oper){
- av_freep(&tf_model->outputs);
- return DNN_ERROR;
- }
- tf_model->outputs[i].index = 0;
- }
-
- if (tf_model->output_tensors) {
- for (uint32_t i = 0; i < tf_model->nb_output; ++i) {
- if (tf_model->output_tensors[i]) {
- TF_DeleteTensor(tf_model->output_tensors[i]);
- tf_model->output_tensors[i] = NULL;
- }
- }
- }
- av_freep(&tf_model->output_tensors);
- tf_model->output_tensors = av_mallocz_array(nb_output, sizeof(*tf_model->output_tensors));
- if (!tf_model->output_tensors) {
- av_freep(&tf_model->outputs);
- return DNN_ERROR;
- }
-
- tf_model->nb_output = nb_output;
-
- if (tf_model->session){
- TF_CloseSession(tf_model->session, tf_model->status);
- TF_DeleteSession(tf_model->session, tf_model->status);
- }
-
- sess_opts = TF_NewSessionOptions();
- tf_model->session = TF_NewSession(tf_model->graph, sess_opts, tf_model->status);
- TF_DeleteSessionOptions(sess_opts);
- if (TF_GetCode(tf_model->status) != TF_OK)
- {
- return DNN_ERROR;
- }
-
- // Run initialization operation with name "init" if it is present in graph
- if (init_op){
- TF_SessionRun(tf_model->session, NULL,
- NULL, NULL, 0,
- NULL, NULL, 0,
- &init_op, 1, NULL, tf_model->status);
- if (TF_GetCode(tf_model->status) != TF_OK)
- {
- return DNN_ERROR;
- }
- }
-
- return DNN_SUCCESS;
-}
-
-static DNNReturnType load_tf_model(TFModel *tf_model, const char *model_filename)
-{
- TF_Buffer *graph_def;
- TF_ImportGraphDefOptions *graph_opts;
-
- graph_def = read_graph(model_filename);
- if (!graph_def){
- return DNN_ERROR;
- }
- tf_model->graph = TF_NewGraph();
- tf_model->status = TF_NewStatus();
- graph_opts = TF_NewImportGraphDefOptions();
- TF_GraphImportGraphDef(tf_model->graph, graph_def, graph_opts, tf_model->status);
- TF_DeleteImportGraphDefOptions(graph_opts);
- TF_DeleteBuffer(graph_def);
- if (TF_GetCode(tf_model->status) != TF_OK){
- TF_DeleteGraph(tf_model->graph);
- TF_DeleteStatus(tf_model->status);
- return DNN_ERROR;
- }
-
- return DNN_SUCCESS;
-}
-
-#define NAME_BUFFER_SIZE 256
-
-static DNNReturnType add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op,
- ConvolutionalParams* params, const int layer)
-{
- TF_Operation *op;
- TF_OperationDescription *op_desc;
- TF_Output input;
- int64_t strides[] = {1, 1, 1, 1};
- TF_Tensor *tensor;
- int64_t dims[4];
- int dims_len;
- char name_buffer[NAME_BUFFER_SIZE];
- int32_t size;
-
- size = params->input_num * params->output_num * params->kernel_size * params->kernel_size;
- input.index = 0;
-
- snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_kernel%d", layer);
- op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
- TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
- dims[0] = params->output_num;
- dims[1] = params->kernel_size;
- dims[2] = params->kernel_size;
- dims[3] = params->input_num;
- dims_len = 4;
- tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, size * sizeof(float));
- memcpy(TF_TensorData(tensor), params->kernel, size * sizeof(float));
- TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
- if (TF_GetCode(tf_model->status) != TF_OK){
- return DNN_ERROR;
- }
- op = TF_FinishOperation(op_desc, tf_model->status);
- if (TF_GetCode(tf_model->status) != TF_OK){
- return DNN_ERROR;
- }
-
- snprintf(name_buffer, NAME_BUFFER_SIZE, "transpose%d", layer);
- op_desc = TF_NewOperation(tf_model->graph, "Transpose", name_buffer);
- input.oper = op;
- TF_AddInput(op_desc, input);
- input.oper = transpose_op;
- TF_AddInput(op_desc, input);
- TF_SetAttrType(op_desc, "T", TF_FLOAT);
- TF_SetAttrType(op_desc, "Tperm", TF_INT32);
- op = TF_FinishOperation(op_desc, tf_model->status);
- if (TF_GetCode(tf_model->status) != TF_OK){
- return DNN_ERROR;
- }
-
- snprintf(name_buffer, NAME_BUFFER_SIZE, "conv2d%d", layer);
- op_desc = TF_NewOperation(tf_model->graph, "Conv2D", name_buffer);
- input.oper = *cur_op;
- TF_AddInput(op_desc, input);
- input.oper = op;
- TF_AddInput(op_desc, input);
- TF_SetAttrType(op_desc, "T", TF_FLOAT);
- TF_SetAttrIntList(op_desc, "strides", strides, 4);
- TF_SetAttrString(op_desc, "padding", "VALID", 5);
- *cur_op = TF_FinishOperation(op_desc, tf_model->status);
- if (TF_GetCode(tf_model->status) != TF_OK){
- return DNN_ERROR;
- }
-
- snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_biases%d", layer);
- op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
- TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
- dims[0] = params->output_num;
- dims_len = 1;
- tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, params->output_num * sizeof(float));
- memcpy(TF_TensorData(tensor), params->biases, params->output_num * sizeof(float));
- TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
- if (TF_GetCode(tf_model->status) != TF_OK){
- return DNN_ERROR;
- }
- op = TF_FinishOperation(op_desc, tf_model->status);
- if (TF_GetCode(tf_model->status) != TF_OK){
- return DNN_ERROR;
- }
-
- snprintf(name_buffer, NAME_BUFFER_SIZE, "bias_add%d", layer);
- op_desc = TF_NewOperation(tf_model->graph, "BiasAdd", name_buffer);
- input.oper = *cur_op;
- TF_AddInput(op_desc, input);
- input.oper = op;
- TF_AddInput(op_desc, input);
- TF_SetAttrType(op_desc, "T", TF_FLOAT);
- *cur_op = TF_FinishOperation(op_desc, tf_model->status);
- if (TF_GetCode(tf_model->status) != TF_OK){
- return DNN_ERROR;
- }
-
- snprintf(name_buffer, NAME_BUFFER_SIZE, "activation%d", layer);
- switch (params->activation){
- case RELU:
- op_desc = TF_NewOperation(tf_model->graph, "Relu", name_buffer);
- break;
- case TANH:
- op_desc = TF_NewOperation(tf_model->graph, "Tanh", name_buffer);
- break;
- case SIGMOID:
- op_desc = TF_NewOperation(tf_model->graph, "Sigmoid", name_buffer);
- break;
- default:
- return DNN_ERROR;
- }
- input.oper = *cur_op;
- TF_AddInput(op_desc, input);
- TF_SetAttrType(op_desc, "T", TF_FLOAT);
- *cur_op = TF_FinishOperation(op_desc, tf_model->status);
- if (TF_GetCode(tf_model->status) != TF_OK){
- return DNN_ERROR;
- }
-
- return DNN_SUCCESS;
-}
-
-static DNNReturnType add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op,
- DepthToSpaceParams *params, const int layer)
-{
- TF_OperationDescription *op_desc;
- TF_Output input;
- char name_buffer[NAME_BUFFER_SIZE];
-
- snprintf(name_buffer, NAME_BUFFER_SIZE, "depth_to_space%d", layer);
- op_desc = TF_NewOperation(tf_model->graph, "DepthToSpace", name_buffer);
- input.oper = *cur_op;
- input.index = 0;
- TF_AddInput(op_desc, input);
- TF_SetAttrType(op_desc, "T", TF_FLOAT);
- TF_SetAttrInt(op_desc, "block_size", params->block_size);
- *cur_op = TF_FinishOperation(op_desc, tf_model->status);
- if (TF_GetCode(tf_model->status) != TF_OK){
- return DNN_ERROR;
- }
-
- return DNN_SUCCESS;
-}
-
-static int calculate_pad(const ConvolutionalNetwork *conv_network)
-{
- ConvolutionalParams *params;
- int32_t layer;
- int pad = 0;
-
- for (layer = 0; layer < conv_network->layers_num; ++layer){
- if (conv_network->layers[layer].type == CONV){
- params = (ConvolutionalParams *)conv_network->layers[layer].params;
- pad += params->kernel_size >> 1;
- }
- }
-
- return pad;
-}
-
-static DNNReturnType add_pad_op(TFModel *tf_model, TF_Operation **cur_op, const int32_t pad)
-{
- TF_Operation *op;
- TF_Tensor *tensor;
- TF_OperationDescription *op_desc;
- TF_Output input;
- int32_t *pads;
- int64_t pads_shape[] = {4, 2};
-
- input.index = 0;
-
- op_desc = TF_NewOperation(tf_model->graph, "Const", "pads");
- TF_SetAttrType(op_desc, "dtype", TF_INT32);
- tensor = TF_AllocateTensor(TF_INT32, pads_shape, 2, 4 * 2 * sizeof(int32_t));
- pads = (int32_t *)TF_TensorData(tensor);
- pads[0] = 0; pads[1] = 0;
- pads[2] = pad; pads[3] = pad;
- pads[4] = pad; pads[5] = pad;
- pads[6] = 0; pads[7] = 0;
- TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
- if (TF_GetCode(tf_model->status) != TF_OK){
- return DNN_ERROR;
- }
- op = TF_FinishOperation(op_desc, tf_model->status);
- if (TF_GetCode(tf_model->status) != TF_OK){
- return DNN_ERROR;
- }
-
- op_desc = TF_NewOperation(tf_model->graph, "MirrorPad", "mirror_pad");
- input.oper = *cur_op;
- TF_AddInput(op_desc, input);
- input.oper = op;
- TF_AddInput(op_desc, input);
- TF_SetAttrType(op_desc, "T", TF_FLOAT);
- TF_SetAttrType(op_desc, "Tpaddings", TF_INT32);
- TF_SetAttrString(op_desc, "mode", "SYMMETRIC", 9);
- *cur_op = TF_FinishOperation(op_desc, tf_model->status);
- if (TF_GetCode(tf_model->status) != TF_OK){
- return DNN_ERROR;
- }
-
- return DNN_SUCCESS;
-}
-
-static DNNReturnType load_native_model(TFModel *tf_model, const char *model_filename)
-{
- int32_t layer;
- TF_OperationDescription *op_desc;
- TF_Operation *op;
- TF_Operation *transpose_op;
- TF_Tensor *tensor;
- TF_Output input;
- int32_t *transpose_perm;
- int64_t transpose_perm_shape[] = {4};
- int64_t input_shape[] = {1, -1, -1, -1};
- int32_t pad;
- DNNReturnType layer_add_res;
- DNNModel *native_model = NULL;
- ConvolutionalNetwork *conv_network;
-
- native_model = ff_dnn_load_model_native(model_filename);
- if (!native_model){
- return DNN_ERROR;
- }
-
- conv_network = (ConvolutionalNetwork *)native_model->model;
- pad = calculate_pad(conv_network);
- tf_model->graph = TF_NewGraph();
- tf_model->status = TF_NewStatus();
-
-#define CLEANUP_ON_ERROR(tf_model) \
- { \
- TF_DeleteGraph(tf_model->graph); \
- TF_DeleteStatus(tf_model->status); \
- return DNN_ERROR; \
- }
-
- op_desc = TF_NewOperation(tf_model->graph, "Placeholder", "x");
- TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
- TF_SetAttrShape(op_desc, "shape", input_shape, 4);
- op = TF_FinishOperation(op_desc, tf_model->status);
- if (TF_GetCode(tf_model->status) != TF_OK){
- CLEANUP_ON_ERROR(tf_model);
- }
-
- if (add_pad_op(tf_model, &op, pad) != DNN_SUCCESS){
- CLEANUP_ON_ERROR(tf_model);
- }
-
- op_desc = TF_NewOperation(tf_model->graph, "Const", "transpose_perm");
- TF_SetAttrType(op_desc, "dtype", TF_INT32);
- tensor = TF_AllocateTensor(TF_INT32, transpose_perm_shape, 1, 4 * sizeof(int32_t));
- transpose_perm = (int32_t *)TF_TensorData(tensor);
- transpose_perm[0] = 1;
- transpose_perm[1] = 2;
- transpose_perm[2] = 3;
- transpose_perm[3] = 0;
- TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
- if (TF_GetCode(tf_model->status) != TF_OK){
- CLEANUP_ON_ERROR(tf_model);
- }
- transpose_op = TF_FinishOperation(op_desc, tf_model->status);
-
- for (layer = 0; layer < conv_network->layers_num; ++layer){
- switch (conv_network->layers[layer].type){
- case INPUT:
- layer_add_res = DNN_SUCCESS;
- break;
- case CONV:
- layer_add_res = add_conv_layer(tf_model, transpose_op, &op,
- (ConvolutionalParams *)conv_network->layers[layer].params, layer);
- break;
- case DEPTH_TO_SPACE:
- layer_add_res = add_depth_to_space_layer(tf_model, &op,
- (DepthToSpaceParams *)conv_network->layers[layer].params, layer);
- break;
- default:
- CLEANUP_ON_ERROR(tf_model);
- }
-
- if (layer_add_res != DNN_SUCCESS){
- CLEANUP_ON_ERROR(tf_model);
- }
- }
-
- op_desc = TF_NewOperation(tf_model->graph, "Identity", "y");
- input.oper = op;
- TF_AddInput(op_desc, input);
- TF_FinishOperation(op_desc, tf_model->status);
- if (TF_GetCode(tf_model->status) != TF_OK){
- CLEANUP_ON_ERROR(tf_model);
- }
-
- ff_dnn_free_model_native(&native_model);
-
- return DNN_SUCCESS;
-}
-
-DNNModel *ff_dnn_load_model_tf(const char *model_filename)
-{
- DNNModel *model = NULL;
- TFModel *tf_model = NULL;
-
- model = av_malloc(sizeof(DNNModel));
- if (!model){
- return NULL;
- }
-
- tf_model = av_mallocz(sizeof(TFModel));
- if (!tf_model){
- av_freep(&model);
- return NULL;
- }
-
- if (load_tf_model(tf_model, model_filename) != DNN_SUCCESS){
- if (load_native_model(tf_model, model_filename) != DNN_SUCCESS){
- av_freep(&tf_model);
- av_freep(&model);
-
- return NULL;
- }
- }
-
- model->model = (void *)tf_model;
- model->set_input_output = &set_input_output_tf;
-
- return model;
-}
-
-
-
-DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, uint32_t nb_output)
-{
- TFModel *tf_model = (TFModel *)model->model;
- uint32_t nb = FFMIN(nb_output, tf_model->nb_output);
- if (nb == 0)
- return DNN_ERROR;
-
- av_assert0(tf_model->output_tensors);
- for (uint32_t i = 0; i < tf_model->nb_output; ++i) {
- if (tf_model->output_tensors[i]) {
- TF_DeleteTensor(tf_model->output_tensors[i]);
- tf_model->output_tensors[i] = NULL;
- }
- }
-
- TF_SessionRun(tf_model->session, NULL,
- &tf_model->input, &tf_model->input_tensor, 1,
- tf_model->outputs, tf_model->output_tensors, nb,
- NULL, 0, NULL, tf_model->status);
-
- if (TF_GetCode(tf_model->status) != TF_OK){
- return DNN_ERROR;
- }
-
- for (uint32_t i = 0; i < nb; ++i) {
- outputs[i].height = TF_Dim(tf_model->output_tensors[i], 1);
- outputs[i].width = TF_Dim(tf_model->output_tensors[i], 2);
- outputs[i].channels = TF_Dim(tf_model->output_tensors[i], 3);
- outputs[i].data = TF_TensorData(tf_model->output_tensors[i]);
- }
-
- return DNN_SUCCESS;
-}
-
-void ff_dnn_free_model_tf(DNNModel **model)
-{
- TFModel *tf_model;
-
- if (*model){
- tf_model = (TFModel *)(*model)->model;
- if (tf_model->graph){
- TF_DeleteGraph(tf_model->graph);
- }
- if (tf_model->session){
- TF_CloseSession(tf_model->session, tf_model->status);
- TF_DeleteSession(tf_model->session, tf_model->status);
- }
- if (tf_model->status){
- TF_DeleteStatus(tf_model->status);
- }
- if (tf_model->input_tensor){
- TF_DeleteTensor(tf_model->input_tensor);
- }
- if (tf_model->output_tensors) {
- for (uint32_t i = 0; i < tf_model->nb_output; ++i) {
- if (tf_model->output_tensors[i]) {
- TF_DeleteTensor(tf_model->output_tensors[i]);
- tf_model->output_tensors[i] = NULL;
- }
- }
- }
- av_freep(&tf_model->outputs);
- av_freep(&tf_model->output_tensors);
- av_freep(&tf_model);
- av_freep(model);
- }
-}
diff --git a/libavfilter/dnn_backend_tf.h b/libavfilter/dnn_backend_tf.h
deleted file mode 100644
index 07877b1..0000000
--- a/libavfilter/dnn_backend_tf.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright (c) 2018 Sergey Lavrushkin
- *
- * This file is part of FFmpeg.
- *
- * FFmpeg is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * FFmpeg is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with FFmpeg; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-/**
- * @file
- * DNN inference functions interface for TensorFlow backend.
- */
-
-
-#ifndef AVFILTER_DNN_BACKEND_TF_H
-#define AVFILTER_DNN_BACKEND_TF_H
-
-#include "dnn_interface.h"
-
-DNNModel *ff_dnn_load_model_tf(const char *model_filename);
-
-DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, uint32_t nb_output);
-
-void ff_dnn_free_model_tf(DNNModel **model);
-
-#endif
diff --git a/libavfilter/dnn_interface.c b/libavfilter/dnn_interface.c
deleted file mode 100644
index 86fc283..0000000
--- a/libavfilter/dnn_interface.c
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Copyright (c) 2018 Sergey Lavrushkin
- *
- * This file is part of FFmpeg.
- *
- * FFmpeg is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2.1 of the License, or (at your option) any later version.
- *
- * FFmpeg is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with FFmpeg; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- */
-
-/**
- * @file
- * Implements DNN module initialization with specified backend.
- */
-
-#include "dnn_interface.h"
-#include "dnn_backend_native.h"
-#include "dnn_backend_tf.h"
-#include "libavutil/mem.h"
-
-DNNModule *ff_get_dnn_module(DNNBackendType backend_type)
-{
- DNNModule *dnn_module;
-
- dnn_module = av_malloc(sizeof(DNNModule));
- if(!dnn_module){
- return NULL;
- }
-
- switch(backend_type){
- case DNN_NATIVE:
- dnn_module->load_model = &ff_dnn_load_model_native;
- dnn_module->execute_model = &ff_dnn_execute_model_native;
- dnn_module->free_model = &ff_dnn_free_model_native;
- break;
- case DNN_TF:
- #if (CONFIG_LIBTENSORFLOW == 1)
- dnn_module->load_model = &ff_dnn_load_model_tf;
- dnn_module->execute_model = &ff_dnn_execute_model_tf;
- dnn_module->free_model = &ff_dnn_free_model_tf;
- #else
- av_freep(&dnn_module);
- return NULL;
- #endif
- break;
- default:
- av_log(NULL, AV_LOG_ERROR, "Module backend_type is not native or tensorflow\n");
- av_freep(&dnn_module);
- return NULL;
- }
-
- return dnn_module;
-}
--
2.7.4
More information about the ffmpeg-devel
mailing list