[FFmpeg-devel] [PATCH 1/2] dnn: add openvino as one of dnn backend
Guo, Yejun
yejun.guo at intel.com
Tue May 26 04:53:50 EEST 2020
OpenVINO is a Deep Learning Deployment Toolkit at
https://github.com/openvinotoolkit/openvino, it supports CPU, GPU
and heterogeneous plugins to accelerate deep learning inferencing.
Please refer to https://github.com/openvinotoolkit/openvino/blob/master/build-instruction.md
to build openvino (c library is built at the same time). Please add
option -DENABLE_MKL_DNN=ON for cmake to enable CPU path. The header
files and libraries are installed to /usr/local/deployment_tools/inference_engine/
with default options on my system.
To build FFmpeg with openvion, take my system as an example, run with:
$ ../ffmpeg/configure --enable-libopenvino --extra-cflags=-I/usr/local/deployment_tools/inference_engine/include/ --extra-ldflags=-L/usr/local/deployment_tools/inference_engine/lib/intel64
$ make
As dnn module maintainer, I do want to see it is utilized by customers,
so the dnn module can be improved on the right direction with developers/customers
collaboration, but I seldomly receive feedbacks.
On the other hand, I know that there are video analytics projects
accepted by customers based on FFmpeg + openvino, see more detail
at https://github.com/VCDP/FFmpeg-patch, but the code bypasses the
dnn interface layer and could not be upstreamed directly.
So, I introduce openvino as one of the dnn backend as a preparation
for later usage.
Signed-off-by: Guo, Yejun <yejun.guo at intel.com>
---
configure | 6 +-
libavfilter/dnn/Makefile | 1 +
libavfilter/dnn/dnn_backend_openvino.c | 261 +++++++++++++++++++++++++++++++++
libavfilter/dnn/dnn_backend_openvino.h | 38 +++++
libavfilter/dnn/dnn_interface.c | 11 ++
libavfilter/dnn_interface.h | 2 +-
6 files changed, 317 insertions(+), 2 deletions(-)
create mode 100644 libavfilter/dnn/dnn_backend_openvino.c
create mode 100644 libavfilter/dnn/dnn_backend_openvino.h
diff --git a/configure b/configure
index f97cad0..6a50351 100755
--- a/configure
+++ b/configure
@@ -253,6 +253,8 @@ External library support:
--enable-libopenh264 enable H.264 encoding via OpenH264 [no]
--enable-libopenjpeg enable JPEG 2000 de/encoding via OpenJPEG [no]
--enable-libopenmpt enable decoding tracked files via libopenmpt [no]
+ --enable-libopenvino enable OpenVINO as a DNN module backend
+ for DNN based filters like dnn_processing [no]
--enable-libopus enable Opus de/encoding via libopus [no]
--enable-libpulse enable Pulseaudio input via libpulse [no]
--enable-librabbitmq enable RabbitMQ library [no]
@@ -1790,6 +1792,7 @@ EXTERNAL_LIBRARY_LIST="
libopenh264
libopenjpeg
libopenmpt
+ libopenvino
libopus
libpulse
librabbitmq
@@ -2620,7 +2623,7 @@ cbs_mpeg2_select="cbs"
cbs_vp9_select="cbs"
dct_select="rdft"
dirac_parse_select="golomb"
-dnn_suggest="libtensorflow"
+dnn_suggest="libtensorflow libopenvino"
error_resilience_select="me_cmp"
faandct_deps="faan"
faandct_select="fdctdsp"
@@ -6346,6 +6349,7 @@ enabled libopenh264 && require_pkg_config libopenh264 openh264 wels/codec_
enabled libopenjpeg && { check_pkg_config libopenjpeg "libopenjp2 >= 2.1.0" openjpeg.h opj_version ||
{ require_pkg_config libopenjpeg "libopenjp2 >= 2.1.0" openjpeg.h opj_version -DOPJ_STATIC && add_cppflags -DOPJ_STATIC; } }
enabled libopenmpt && require_pkg_config libopenmpt "libopenmpt >= 0.2.6557" libopenmpt/libopenmpt.h openmpt_module_create -lstdc++ && append libopenmpt_extralibs "-lstdc++"
+enabled libopenvino && require libopenvino c_api/ie_c_api.h ie_c_api_version -linference_engine_c_api
enabled libopus && {
enabled libopus_decoder && {
require_pkg_config libopus opus opus_multistream.h opus_multistream_decoder_create
diff --git a/libavfilter/dnn/Makefile b/libavfilter/dnn/Makefile
index ce52958..a4900f6 100644
--- a/libavfilter/dnn/Makefile
+++ b/libavfilter/dnn/Makefile
@@ -8,5 +8,6 @@ OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native_layer_max
OBJS-$(CONFIG_DNN) += dnn/dnn_backend_native_layer_mathbinary.o
DNN-OBJS-$(CONFIG_LIBTENSORFLOW) += dnn/dnn_backend_tf.o
+DNN-OBJS-$(CONFIG_LIBOPENVINO) += dnn/dnn_backend_openvino.o
OBJS-$(CONFIG_DNN) += $(DNN-OBJS-yes)
diff --git a/libavfilter/dnn/dnn_backend_openvino.c b/libavfilter/dnn/dnn_backend_openvino.c
new file mode 100644
index 0000000..f048bc2
--- /dev/null
+++ b/libavfilter/dnn/dnn_backend_openvino.c
@@ -0,0 +1,261 @@
+/*
+ * Copyright (c) 2020
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file
+ * DNN OpenVINO backend implementation.
+ */
+
+#include "dnn_backend_openvino.h"
+#include "libavformat/avio.h"
+#include "libavutil/avassert.h"
+#include <c_api/ie_c_api.h>
+
+typedef struct OVModel{
+ ie_core_t *core;
+ ie_network_t *network;
+ ie_executable_network_t *exe_network;
+ ie_infer_request_t *infer_request;
+ ie_blob_t *input_blob;
+ ie_blob_t **output_blobs;
+ uint32_t nb_output;
+} OVModel;
+
+static DNNDataType precision_to_datatype(precision_e precision)
+{
+ switch (precision)
+ {
+ case FP32:
+ return DNN_FLOAT;
+ default:
+ av_assert0(!"not supported yet.");
+ return DNN_FLOAT;
+ }
+}
+
+static DNNReturnType get_input_ov(void *model, DNNData *input, const char *input_name)
+{
+ OVModel *ov_model = (OVModel *)model;
+ char *model_input_name = NULL;
+ IEStatusCode status;
+ size_t model_input_count = 0;
+ dimensions_t dims;
+ precision_e precision;
+
+ status = ie_network_get_inputs_number(ov_model->network, &model_input_count);
+ if (status != OK)
+ return DNN_ERROR;
+
+ for (size_t i = 0; i < model_input_count; i++) {
+ status = ie_network_get_input_name(ov_model->network, i, &model_input_name);
+ if (status != OK)
+ return DNN_ERROR;
+ if (strcmp(model_input_name, input_name) == 0) {
+ ie_network_name_free(&model_input_name);
+ status |= ie_network_get_input_dims(ov_model->network, input_name, &dims);
+ status |= ie_network_get_input_precision(ov_model->network, input_name, &precision);
+ if (status != OK)
+ return DNN_ERROR;
+
+ // The order of dims in the openvino is fixed and it is always NCHW for 4-D data.
+ // while we pass NHWC data from FFmpeg to openvino
+ status = ie_network_set_input_layout(ov_model->network, input_name, NHWC);
+ if (status != OK)
+ return DNN_ERROR;
+
+ input->channels = dims.dims[1];
+ input->height = dims.dims[2];
+ input->width = dims.dims[3];
+ input->dt = precision_to_datatype(precision);
+ return DNN_SUCCESS;
+ }
+
+ ie_network_name_free(&model_input_name);
+ }
+
+ return DNN_ERROR;
+}
+
+static DNNReturnType set_input_output_ov(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output)
+{
+ OVModel *ov_model = (OVModel *)model;
+ IEStatusCode status;
+ dimensions_t dims;
+ precision_e precision;
+ ie_blob_buffer_t blob_buffer;
+
+ status = ie_exec_network_create_infer_request(ov_model->exe_network, &ov_model->infer_request);
+ if (status != OK)
+ goto err;
+
+ status = ie_infer_request_get_blob(ov_model->infer_request, input_name, &ov_model->input_blob);
+ if (status != OK)
+ goto err;
+
+ status |= ie_blob_get_dims(ov_model->input_blob, &dims);
+ status |= ie_blob_get_precision(ov_model->input_blob, &precision);
+ if (status != OK)
+ goto err;
+
+ av_assert0(input->channels == dims.dims[1]);
+ av_assert0(input->height == dims.dims[2]);
+ av_assert0(input->width == dims.dims[3]);
+ av_assert0(input->dt == precision_to_datatype(precision));
+
+ status = ie_blob_get_buffer(ov_model->input_blob, &blob_buffer);
+ if (status != OK)
+ goto err;
+ input->data = blob_buffer.buffer;
+
+ // outputs
+ ov_model->nb_output = 0;
+ av_freep(&ov_model->output_blobs);
+ ov_model->output_blobs = av_mallocz_array(nb_output, sizeof(*ov_model->output_blobs));
+ if (!ov_model->output_blobs)
+ goto err;
+
+ for (int i = 0; i < nb_output; i++) {
+ const char *output_name = output_names[i];
+ status = ie_infer_request_get_blob(ov_model->infer_request, output_name, &(ov_model->output_blobs[i]));
+ if (status != OK)
+ goto err;
+ ov_model->nb_output++;
+ }
+
+ return DNN_SUCCESS;
+
+err:
+ if (ov_model->output_blobs) {
+ for (uint32_t i = 0; i < ov_model->nb_output; i++) {
+ ie_blob_free(&(ov_model->output_blobs[i]));
+ }
+ av_freep(&ov_model->output_blobs);
+ }
+ if (ov_model->input_blob)
+ ie_blob_free(&ov_model->input_blob);
+ if (ov_model->infer_request)
+ ie_infer_request_free(&ov_model->infer_request);
+ return DNN_ERROR;
+}
+
+DNNModel *ff_dnn_load_model_ov(const char *model_filename)
+{
+ DNNModel *model = NULL;
+ OVModel *ov_model = NULL;
+ IEStatusCode status;
+ ie_config_t config = {NULL, NULL, NULL};
+
+ model = av_malloc(sizeof(DNNModel));
+ if (!model){
+ return NULL;
+ }
+
+ ov_model = av_mallocz(sizeof(OVModel));
+ if (!ov_model)
+ goto err;
+
+ status = ie_core_create("", &ov_model->core);
+ if (status != OK)
+ goto err;
+
+ status = ie_core_read_network(ov_model->core, model_filename, NULL, &ov_model->network);
+ if (status != OK)
+ goto err;
+
+ status = ie_core_load_network(ov_model->core, ov_model->network, "CPU", &config, &ov_model->exe_network);
+ if (status != OK)
+ goto err;
+
+ model->model = (void *)ov_model;
+ model->set_input_output = &set_input_output_ov;
+ model->get_input = &get_input_ov;
+
+ return model;
+
+err:
+ if (model)
+ av_freep(&model);
+ if (ov_model) {
+ if (ov_model->exe_network)
+ ie_exec_network_free(&ov_model->exe_network);
+ if (ov_model->network)
+ ie_network_free(&ov_model->network);
+ if (ov_model->core)
+ ie_core_free(&ov_model->core);
+ av_freep(&ov_model);
+ }
+ return NULL;
+}
+
+DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, DNNData *outputs, uint32_t nb_output)
+{
+ dimensions_t dims;
+ precision_e precision;
+ ie_blob_buffer_t blob_buffer;
+ OVModel *ov_model = (OVModel *)model->model;
+ uint32_t nb = FFMIN(nb_output, ov_model->nb_output);
+ IEStatusCode status = ie_infer_request_infer(ov_model->infer_request);
+ if (status != OK)
+ return DNN_ERROR;
+
+ for (uint32_t i = 0; i < nb; ++i) {
+ status = ie_blob_get_buffer(ov_model->output_blobs[i], &blob_buffer);
+ if (status != OK)
+ return DNN_ERROR;
+
+ status |= ie_blob_get_dims(ov_model->output_blobs[i], &dims);
+ status |= ie_blob_get_precision(ov_model->output_blobs[i], &precision);
+ if (status != OK)
+ return DNN_ERROR;
+
+ outputs[i].channels = dims.dims[1];
+ outputs[i].height = dims.dims[2];
+ outputs[i].width = dims.dims[3];
+ outputs[i].dt = precision_to_datatype(precision);
+ outputs[i].data = blob_buffer.buffer;
+ }
+
+ return DNN_SUCCESS;
+}
+
+void ff_dnn_free_model_ov(DNNModel **model)
+{
+ if (*model){
+ OVModel *ov_model = (OVModel *)(*model)->model;
+ if (ov_model->output_blobs) {
+ for (uint32_t i = 0; i < ov_model->nb_output; i++) {
+ ie_blob_free(&(ov_model->output_blobs[i]));
+ }
+ av_freep(&ov_model->output_blobs);
+ }
+ if (ov_model->input_blob)
+ ie_blob_free(&ov_model->input_blob);
+ if (ov_model->infer_request)
+ ie_infer_request_free(&ov_model->infer_request);
+ if (ov_model->exe_network)
+ ie_exec_network_free(&ov_model->exe_network);
+ if (ov_model->network)
+ ie_network_free(&ov_model->network);
+ if (ov_model->core)
+ ie_core_free(&ov_model->core);
+ av_freep(&ov_model);
+ av_freep(model);
+ }
+}
diff --git a/libavfilter/dnn/dnn_backend_openvino.h b/libavfilter/dnn/dnn_backend_openvino.h
new file mode 100644
index 0000000..397847a
--- /dev/null
+++ b/libavfilter/dnn/dnn_backend_openvino.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2020
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+/**
+ * @file
+ * DNN inference functions interface for OpenVINO backend.
+ */
+
+
+#ifndef AVFILTER_DNN_DNN_BACKEND_OPENVINO_H
+#define AVFILTER_DNN_DNN_BACKEND_OPENVINO_H
+
+#include "../dnn_interface.h"
+
+DNNModel *ff_dnn_load_model_ov(const char *model_filename);
+
+DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, DNNData *outputs, uint32_t nb_output);
+
+void ff_dnn_free_model_ov(DNNModel **model);
+
+#endif
diff --git a/libavfilter/dnn/dnn_interface.c b/libavfilter/dnn/dnn_interface.c
index 62da55f..7973d3e 100644
--- a/libavfilter/dnn/dnn_interface.c
+++ b/libavfilter/dnn/dnn_interface.c
@@ -26,6 +26,7 @@
#include "../dnn_interface.h"
#include "dnn_backend_native.h"
#include "dnn_backend_tf.h"
+#include "dnn_backend_openvino.h"
#include "libavutil/mem.h"
DNNModule *ff_get_dnn_module(DNNBackendType backend_type)
@@ -53,6 +54,16 @@ DNNModule *ff_get_dnn_module(DNNBackendType backend_type)
return NULL;
#endif
break;
+ case DNN_OV:
+ #if (CONFIG_LIBOPENVINO == 1)
+ dnn_module->load_model = &ff_dnn_load_model_ov;
+ dnn_module->execute_model = &ff_dnn_execute_model_ov;
+ dnn_module->free_model = &ff_dnn_free_model_ov;
+ #else
+ av_freep(&dnn_module);
+ return NULL;
+ #endif
+ break;
default:
av_log(NULL, AV_LOG_ERROR, "Module backend_type is not native or tensorflow\n");
av_freep(&dnn_module);
diff --git a/libavfilter/dnn_interface.h b/libavfilter/dnn_interface.h
index b20e5c8..f914265 100644
--- a/libavfilter/dnn_interface.h
+++ b/libavfilter/dnn_interface.h
@@ -30,7 +30,7 @@
typedef enum {DNN_SUCCESS, DNN_ERROR} DNNReturnType;
-typedef enum {DNN_NATIVE, DNN_TF} DNNBackendType;
+typedef enum {DNN_NATIVE, DNN_TF, DNN_OV} DNNBackendType;
typedef enum {DNN_FLOAT = 1, DNN_UINT8 = 4} DNNDataType;
--
2.7.4
More information about the ffmpeg-devel
mailing list