[FFmpeg-devel] [PATCH 1/2] dnn_backend_native_layer_mathunary: add abs support
Ting Fu
ting.fu at intel.com
Mon May 25 17:46:26 EEST 2020
more math unary operations will be added here
It can be tested with the model file generated with below python scripy:
import tensorflow as tf
import numpy as np
import imageio
in_img = imageio.imread('input.jpeg')
in_img = in_img.astype(np.float32)/255.0
in_data = in_img[np.newaxis, :]
x = tf.placeholder(tf.float32, shape=[1, None, None, 3], name='dnn_in')
x1 = tf.subtract(x, 0.5)
x2 = tf.abs(x1)
y = tf.identity(x2, name='dnn_out')
sess=tf.Session()
sess.run(tf.global_variables_initializer())
graph_def = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, ['dnn_out'])
tf.train.write_graph(graph_def, '.', 'image_process.pb', as_text=False)
print("image_process.pb generated, please use \
path_to_ffmpeg/tools/python/convert.py to generate image_process.model\n")
output = sess.run(y, feed_dict={x: in_data})
imageio.imsave("out.jpg", np.squeeze(output))
Signed-off-by: Ting Fu <ting.fu at intel.com>
---
libavfilter/dnn/Makefile | 1 +
libavfilter/dnn/dnn_backend_native.h | 1 +
.../dnn/dnn_backend_native_layer_mathunary.c | 80 +++++++++++++++++++
.../dnn/dnn_backend_native_layer_mathunary.h | 45 +++++++++++
libavfilter/dnn/dnn_backend_native_layers.c | 2 +
tools/python/convert_from_tensorflow.py | 16 +++-
tools/python/convert_header.py | 2 +-
7 files changed, 145 insertions(+), 2 deletions(-)
create mode 100644 libavfilter/dnn/dnn_backend_native_layer_mathunary.c
create mode 100644 libavfilter/dnn/dnn_backend_native_layer_mathunary.h
diff --git a/libavfilter/dnn/Makefile b/libavfilter/dnn/Makefile
index ce529587e1..bb37298b58 100644
--- a/libavfilter/dnn/Makefile
+++ b/libavfilter/dnn/Makefile
@@ -6,6 +6,7 @@ OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native_layer_con
OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native_layer_depth2space.o
OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native_layer_maximum.o
OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native_layer_mathbinary.o
+OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native_layer_mathunary.o
DNN-OBJS-$(CONFIG_LIBTENSORFLOW) += dnn/dnn_backend_tf.o
diff --git a/libavfilter/dnn/dnn_backend_native.h b/libavfilter/dnn/dnn_backend_native.h
index 5d76d87915..61f0cb202f 100644
--- a/libavfilter/dnn/dnn_backend_native.h
+++ b/libavfilter/dnn/dnn_backend_native.h
@@ -42,6 +42,7 @@ typedef enum {
DLT_MIRROR_PAD = 3,
DLT_MAXIMUM = 4,
DLT_MATH_BINARY = 5,
+ DLT_MATH_UNARY = 6,
DLT_COUNT
} DNNLayerType;
diff --git a/libavfilter/dnn/dnn_backend_native_layer_mathunary.c b/libavfilter/dnn/dnn_backend_native_layer_mathunary.c
new file mode 100644
index 0000000000..d65af151cd
--- /dev/null
+++ b/libavfilter/dnn/dnn_backend_native_layer_mathunary.c
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2020
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file
+ * DNN native backend implementation.
+ */
+
+#include "dnn_backend_native.h"
+#include "libavutil/avassert.h"
+#include "dnn_backend_native_layer_mathunary.h"
+
+int dnn_load_layer_math_unary(Layer *layer, AVIOContext *model_file_context, int file_size)
+{
+ DnnLayerMathUnaryParams *params;
+ int dnn_size = 0;
+ params = av_malloc(sizeof(*params));
+ if(!params)
+ return 0;
+
+ params->un_op = (int32_t)avio_rl32(model_file_context);
+ dnn_size += 4;
+ layer->params = params;
+ layer->input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context);
+ layer->output_operand_index = (int32_t)avio_rl32(model_file_context);
+ dnn_size += 8;
+
+ return dnn_size;
+
+}
+
+int dnn_execute_layer_math_unary(DnnOperand *operands, const int32_t *input_operand_indexes,
+ int32_t output_operand_index, const void *parameters)
+{
+ const DnnOperand *input = &operands[input_operand_indexes[0]];
+ DnnOperand *output = &operands[output_operand_index];
+ const DnnLayerMathUnaryParams *params = (const DnnLayerMathUnaryParams *)parameters;
+ int dims_count;
+ const float *src;
+ float *dst;
+
+ for (int i = 0; i < 4; ++i)
+ output->dims[i] = input->dims[i];
+
+ output->data_type = input->data_type;
+ output->length = calculate_operand_data_length(output);
+ output->data = av_realloc(output->data, output->length);
+ if (!output->data)
+ return DNN_ERROR;
+
+ dims_count = calculate_operand_dims_count(output);
+ src = input->data;
+ dst = output->data;
+
+ switch (params->un_op) {
+ case DMUO_ABS:
+ for (int i = 0; i < dims_count; ++i)
+ dst[i] = FFABS(src[i]);
+ return 0;
+ default:
+ return -1;
+ }
+}
diff --git a/libavfilter/dnn/dnn_backend_native_layer_mathunary.h b/libavfilter/dnn/dnn_backend_native_layer_mathunary.h
new file mode 100644
index 0000000000..4e44003b66
--- /dev/null
+++ b/libavfilter/dnn/dnn_backend_native_layer_mathunary.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2020
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file
+ * DNN inference functions interface for native backend.
+ */
+
+#ifndef AVFILTER_DNN_DNN_BACKEND_NATIVE_LAYER_MATHUNARY_H
+#define AVFILTER_DNN_DNN_BACKEND_NATIVE_LAYER_MATHUNARY_H
+
+#include "libavformat/avio.h"
+#include "dnn_backend_native.h"
+
+typedef enum {
+ DMUO_ABS = 0,
+ DMUO_COUNT
+} DNNMathUnaryOperation;
+
+typedef struct DnnLayerMathUnaryParams{
+ DNNMathUnaryOperation un_op;
+} DnnLayerMathUnaryParams;
+
+int dnn_load_layer_math_unary(Layer *layer, AVIOContext *model_file_context, int file_size);
+int dnn_execute_layer_math_unary(DnnOperand *operands, const int32_t *input_operand_indexes,
+ int32_t output_operand_index, const void *parameters);
+
+#endif
diff --git a/libavfilter/dnn/dnn_backend_native_layers.c b/libavfilter/dnn/dnn_backend_native_layers.c
index af18552eb4..70f9a5f958 100644
--- a/libavfilter/dnn/dnn_backend_native_layers.c
+++ b/libavfilter/dnn/dnn_backend_native_layers.c
@@ -25,6 +25,7 @@
#include "dnn_backend_native_layer_depth2space.h"
#include "dnn_backend_native_layer_maximum.h"
#include "dnn_backend_native_layer_mathbinary.h"
+#include "dnn_backend_native_layer_mathunary.h"
LayerFunc layer_funcs[DLT_COUNT] = {
{NULL, NULL},
@@ -33,4 +34,5 @@ LayerFunc layer_funcs[DLT_COUNT] = {
{dnn_execute_layer_pad, dnn_load_layer_pad},
{dnn_execute_layer_maximum, dnn_load_layer_maximum},
{dnn_execute_layer_math_binary, dnn_load_layer_math_binary},
+ {dnn_execute_layer_math_unary, dnn_load_layer_math_unary},
};
diff --git a/tools/python/convert_from_tensorflow.py b/tools/python/convert_from_tensorflow.py
index 1c20891fcc..8c0a9be7be 100644
--- a/tools/python/convert_from_tensorflow.py
+++ b/tools/python/convert_from_tensorflow.py
@@ -70,8 +70,9 @@ class TFConverter:
self.converted_nodes = set()
self.conv2d_scope_names = set()
self.conv2d_scopename_inputname_dict = {}
- self.op2code = {'Conv2D':1, 'DepthToSpace':2, 'MirrorPad':3, 'Maximum':4, 'MathBinary':5}
+ self.op2code = {'Conv2D':1, 'DepthToSpace':2, 'MirrorPad':3, 'Maximum':4, 'MathBinary':5, 'MathUnary':6}
self.mathbin2code = {'Sub':0, 'Add':1, 'Mul':2, 'RealDiv':3, 'Minimum':4}
+ self.mathun2code = {'Abs':0}
self.mirrorpad_mode = {'CONSTANT':0, 'REFLECT':1, 'SYMMETRIC':2}
self.name_operand_dict = {}
@@ -286,6 +287,17 @@ class TFConverter:
np.array([output_operand_index], dtype=np.uint32).tofile(f)
+ def dump_mathunary_to_file(self, node, f):
+ self.layer_number = self.layer_number + 1
+ self.converted_nodes.add(node.name)
+ i0_node = self.name_node_dict[node.input[0]]
+ np.array([self.op2code['MathUnary'], self.mathun2code[node.op]], dtype=np.uint32).tofile(f)
+ input_operand_index = self.add_operand(i0_node.name, Operand.IOTYPE_INPUT)
+ np.array([input_operand_index], dtype=np.uint32).tofile(f)
+ output_operand_index = self.add_operand(node.name, Operand.IOTYPE_OUTPUT)
+ np.array([output_operand_index],dtype=np.uint32).tofile(f)
+
+
def dump_layers_to_file(self, f):
for node in self.nodes:
if node.name in self.converted_nodes:
@@ -307,6 +319,8 @@ class TFConverter:
self.dump_maximum_to_file(node, f)
elif node.op in self.mathbin2code:
self.dump_mathbinary_to_file(node, f)
+ elif node.op in self.mathun2code:
+ self.dump_mathunary_to_file(node, f)
def dump_operands_to_file(self, f):
diff --git a/tools/python/convert_header.py b/tools/python/convert_header.py
index e692a5e217..ad4491729a 100644
--- a/tools/python/convert_header.py
+++ b/tools/python/convert_header.py
@@ -23,4 +23,4 @@ str = 'FFMPEGDNNNATIVE'
major = 1
# increase minor when we don't have to re-convert the model file
-minor = 5
+minor = 6
--
2.17.1
More information about the ffmpeg-devel
mailing list