[FFmpeg-cvslog] libavfiter/dnn_backend_openvino: Add multiple output support
Wenbin Chen
git at videolan.org
Sat Dec 16 16:31:06 EET 2023
ffmpeg | branch: master | Wenbin Chen <wenbin.chen at intel.com> | Tue Dec 12 10:33:31 2023 +0800| [22652b576c2a0670d341648c68ca469ebe08f1a1] | committer: Guo Yejun
libavfiter/dnn_backend_openvino: Add multiple output support
Add multiple output support to openvino backend. You can use '&' to
split different output when you set output name using command line.
Signed-off-by: Wenbin Chen <wenbin.chen at intel.com>
Reviewed-by: Guo Yejun <yejun.guo at intel.com>
> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=22652b576c2a0670d341648c68ca469ebe08f1a1
---
libavfilter/dnn/dnn_backend_common.c | 7 --
libavfilter/dnn/dnn_backend_openvino.c | 216 ++++++++++++++++++++++-----------
libavfilter/vf_dnn_detect.c | 11 +-
3 files changed, 150 insertions(+), 84 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_common.c b/libavfilter/dnn/dnn_backend_common.c
index 91a4a3c4bf..632832ec36 100644
--- a/libavfilter/dnn/dnn_backend_common.c
+++ b/libavfilter/dnn/dnn_backend_common.c
@@ -43,13 +43,6 @@ int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func
return AVERROR(EINVAL);
}
- if (exec_params->nb_output != 1 && backend != DNN_TF) {
- // currently, the filter does not need multiple outputs,
- // so we just pending the support until we really need it.
- avpriv_report_missing_feature(ctx, "multiple outputs");
- return AVERROR(ENOSYS);
- }
-
return 0;
}
diff --git a/libavfilter/dnn/dnn_backend_openvino.c b/libavfilter/dnn/dnn_backend_openvino.c
index 6fe8b9c243..089e028818 100644
--- a/libavfilter/dnn/dnn_backend_openvino.c
+++ b/libavfilter/dnn/dnn_backend_openvino.c
@@ -64,7 +64,7 @@ typedef struct OVModel{
ov_compiled_model_t *compiled_model;
ov_output_const_port_t* input_port;
ov_preprocess_input_info_t* input_info;
- ov_output_const_port_t* output_port;
+ ov_output_const_port_t** output_ports;
ov_preprocess_output_info_t* output_info;
ov_preprocess_prepostprocessor_t* preprocess;
#else
@@ -77,6 +77,7 @@ typedef struct OVModel{
SafeQueue *request_queue; // holds OVRequestItem
Queue *task_queue; // holds TaskItem
Queue *lltask_queue; // holds LastLevelTaskItem
+ int nb_outputs;
} OVModel;
// one request for one call to openvino
@@ -349,7 +350,7 @@ static void infer_completion_callback(void *args)
TaskItem *task = lltask->task;
OVModel *ov_model = task->model;
SafeQueue *requestq = ov_model->request_queue;
- DNNData output;
+ DNNData *outputs;
OVContext *ctx = &ov_model->ctx;
#if HAVE_OPENVINO2
size_t* dims;
@@ -358,45 +359,61 @@ static void infer_completion_callback(void *args)
ov_shape_t output_shape = {0};
ov_element_type_e precision;
- memset(&output, 0, sizeof(output));
- status = ov_infer_request_get_output_tensor_by_index(request->infer_request, 0, &output_tensor);
- if (status != OK) {
- av_log(ctx, AV_LOG_ERROR,
- "Failed to get output tensor.");
+ outputs = av_calloc(ov_model->nb_outputs, sizeof(*outputs));
+ if (!outputs) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to alloc outputs.");
return;
}
- status = ov_tensor_data(output_tensor, &output.data);
- if (status != OK) {
- av_log(ctx, AV_LOG_ERROR,
- "Failed to get output data.");
- return;
- }
+ for (int i = 0; i < ov_model->nb_outputs; i++) {
+ status = ov_infer_request_get_tensor_by_const_port(request->infer_request,
+ ov_model->output_ports[i],
+ &output_tensor);
+ if (status != OK) {
+ av_log(ctx, AV_LOG_ERROR,
+ "Failed to get output tensor.");
+ goto end;
+ }
- status = ov_tensor_get_shape(output_tensor, &output_shape);
- if (status != OK) {
- av_log(ctx, AV_LOG_ERROR, "Failed to get output port shape.\n");
- return;
- }
- dims = output_shape.dims;
+ status = ov_tensor_data(output_tensor, &outputs[i].data);
+ if (status != OK) {
+ av_log(ctx, AV_LOG_ERROR,
+ "Failed to get output data.");
+ goto end;
+ }
- status = ov_port_get_element_type(ov_model->output_port, &precision);
- if (status != OK) {
- av_log(ctx, AV_LOG_ERROR, "Failed to get output port data type.\n");
+ status = ov_tensor_get_shape(output_tensor, &output_shape);
+ if (status != OK) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to get output port shape.\n");
+ goto end;
+ }
+ dims = output_shape.dims;
+
+ status = ov_port_get_element_type(ov_model->output_ports[i], &precision);
+ if (status != OK) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to get output port data type.\n");
+ goto end;
+ }
+ outputs[i].dt = precision_to_datatype(precision);
+
+ outputs[i].channels = output_shape.rank > 2 ? dims[output_shape.rank - 3] : 1;
+ outputs[i].height = output_shape.rank > 1 ? dims[output_shape.rank - 2] : 1;
+ outputs[i].width = output_shape.rank > 0 ? dims[output_shape.rank - 1] : 1;
+ av_assert0(request->lltask_count <= dims[0]);
+ outputs[i].layout = ctx->options.layout;
+ outputs[i].scale = ctx->options.scale;
+ outputs[i].mean = ctx->options.mean;
ov_shape_free(&output_shape);
- return;
+ ov_tensor_free(output_tensor);
+ output_tensor = NULL;
}
- output.channels = output_shape.rank > 2 ? dims[output_shape.rank - 3] : 1;
- output.height = output_shape.rank > 1 ? dims[output_shape.rank - 2] : 1;
- output.width = output_shape.rank > 0 ? dims[output_shape.rank - 1] : 1;
- av_assert0(request->lltask_count <= dims[0]);
- ov_shape_free(&output_shape);
#else
IEStatusCode status;
dimensions_t dims;
ie_blob_t *output_blob = NULL;
ie_blob_buffer_t blob_buffer;
precision_e precision;
+ DNNData output;
status = ie_infer_request_get_blob(request->infer_request, task->output_names[0], &output_blob);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR,
@@ -424,11 +441,12 @@ static void infer_completion_callback(void *args)
output.height = dims.dims[2];
output.width = dims.dims[3];
av_assert0(request->lltask_count <= dims.dims[0]);
-#endif
output.dt = precision_to_datatype(precision);
output.layout = ctx->options.layout;
output.scale = ctx->options.scale;
output.mean = ctx->options.mean;
+ outputs = &output;
+#endif
av_assert0(request->lltask_count >= 1);
for (int i = 0; i < request->lltask_count; ++i) {
@@ -438,28 +456,33 @@ static void infer_completion_callback(void *args)
case DFT_PROCESS_FRAME:
if (task->do_ioproc) {
if (ov_model->model->frame_post_proc != NULL) {
- ov_model->model->frame_post_proc(task->out_frame, &output, ov_model->model->filter_ctx);
+ ov_model->model->frame_post_proc(task->out_frame, outputs, ov_model->model->filter_ctx);
} else {
- ff_proc_from_dnn_to_frame(task->out_frame, &output, ctx);
+ ff_proc_from_dnn_to_frame(task->out_frame, outputs, ctx);
}
} else {
- task->out_frame->width = output.width;
- task->out_frame->height = output.height;
+ task->out_frame->width = outputs[0].width;
+ task->out_frame->height = outputs[0].height;
}
break;
case DFT_ANALYTICS_DETECT:
if (!ov_model->model->detect_post_proc) {
av_log(ctx, AV_LOG_ERROR, "detect filter needs to provide post proc\n");
- return;
+ goto end;
}
- ov_model->model->detect_post_proc(task->in_frame, &output, 1, ov_model->model->filter_ctx);
+ ov_model->model->detect_post_proc(task->in_frame, outputs,
+ ov_model->nb_outputs,
+ ov_model->model->filter_ctx);
break;
case DFT_ANALYTICS_CLASSIFY:
if (!ov_model->model->classify_post_proc) {
av_log(ctx, AV_LOG_ERROR, "classify filter needs to provide post proc\n");
- return;
+ goto end;
}
- ov_model->model->classify_post_proc(task->in_frame, &output, request->lltasks[i]->bbox_index, ov_model->model->filter_ctx);
+ for (int output_i = 0; output_i < ov_model->nb_outputs; output_i++)
+ ov_model->model->classify_post_proc(task->in_frame, outputs,
+ request->lltasks[i]->bbox_index,
+ ov_model->model->filter_ctx);
break;
default:
av_assert0(!"should not reach here");
@@ -468,10 +491,17 @@ static void infer_completion_callback(void *args)
task->inference_done++;
av_freep(&request->lltasks[i]);
- output.data = (uint8_t *)output.data
- + output.width * output.height * output.channels * get_datatype_size(output.dt);
+ for (int i = 0; i < ov_model->nb_outputs; i++)
+ outputs[i].data = (uint8_t *)outputs[i].data +
+ outputs[i].width * outputs[i].height * outputs[i].channels * get_datatype_size(outputs[i].dt);
}
-#if !HAVE_OPENVINO2
+end:
+#if HAVE_OPENVINO2
+ av_freep(&outputs);
+ ov_shape_free(&output_shape);
+ if (output_tensor)
+ ov_tensor_free(output_tensor);
+#else
ie_blob_free(&output_blob);
#endif
request->lltask_count = 0;
@@ -525,8 +555,10 @@ static void dnn_free_model_ov(DNNModel **model)
#if HAVE_OPENVINO2
if (ov_model->input_port)
ov_output_const_port_free(ov_model->input_port);
- if (ov_model->output_port)
- ov_output_const_port_free(ov_model->output_port);
+ for (int i = 0; i < ov_model->nb_outputs; i++)
+ if (ov_model->output_ports[i])
+ ov_output_const_port_free(ov_model->output_ports[i]);
+ av_freep(&ov_model->output_ports);
if (ov_model->preprocess)
ov_preprocess_prepostprocessor_free(ov_model->preprocess);
if (ov_model->compiled_model)
@@ -551,7 +583,7 @@ static void dnn_free_model_ov(DNNModel **model)
}
-static int init_model_ov(OVModel *ov_model, const char *input_name, const char *output_name)
+static int init_model_ov(OVModel *ov_model, const char *input_name, const char **output_names, int nb_outputs)
{
int ret = 0;
OVContext *ctx = &ov_model->ctx;
@@ -594,17 +626,15 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char *
}
status = ov_preprocess_prepostprocessor_get_input_info_by_name(ov_model->preprocess, input_name, &ov_model->input_info);
- status |= ov_preprocess_prepostprocessor_get_output_info_by_name(ov_model->preprocess, output_name, &ov_model->output_info);
if (status != OK) {
- av_log(ctx, AV_LOG_ERROR, "Failed to get input/output info from preprocess.\n");
+ av_log(ctx, AV_LOG_ERROR, "Failed to get input info from preprocess.\n");
ret = ov2_map_error(status, NULL);
goto err;
}
status = ov_preprocess_input_info_get_tensor_info(ov_model->input_info, &input_tensor_info);
- status |= ov_preprocess_output_info_get_tensor_info(ov_model->output_info, &output_tensor_info);
if (status != OK) {
- av_log(ctx, AV_LOG_ERROR, "Failed to get tensor info from input/output.\n");
+ av_log(ctx, AV_LOG_ERROR, "Failed to get tensor info from input.\n");
ret = ov2_map_error(status, NULL);
goto err;
}
@@ -642,17 +672,43 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char *
}
status = ov_preprocess_input_tensor_info_set_element_type(input_tensor_info, U8);
- if (ov_model->model->func_type != DFT_PROCESS_FRAME)
- status |= ov_preprocess_output_set_element_type(output_tensor_info, F32);
- else if (fabsf(ctx->options.scale - 1) > 1e-6f || fabsf(ctx->options.mean) > 1e-6f)
- status |= ov_preprocess_output_set_element_type(output_tensor_info, F32);
- else
- status |= ov_preprocess_output_set_element_type(output_tensor_info, U8);
if (status != OK) {
- av_log(ctx, AV_LOG_ERROR, "Failed to set input/output element type\n");
+ av_log(ctx, AV_LOG_ERROR, "Failed to set input element type\n");
ret = ov2_map_error(status, NULL);
goto err;
}
+
+ ov_model->nb_outputs = nb_outputs;
+ for (int i = 0; i < nb_outputs; i++) {
+ status = ov_preprocess_prepostprocessor_get_output_info_by_name(
+ ov_model->preprocess, output_names[i], &ov_model->output_info);
+ if (status != OK) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to get output info from preprocess.\n");
+ ret = ov2_map_error(status, NULL);
+ goto err;
+ }
+ status |= ov_preprocess_output_info_get_tensor_info(ov_model->output_info, &output_tensor_info);
+ if (status != OK) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to get tensor info from input/output.\n");
+ ret = ov2_map_error(status, NULL);
+ goto err;
+ }
+ if (ov_model->model->func_type != DFT_PROCESS_FRAME)
+ status |= ov_preprocess_output_set_element_type(output_tensor_info, F32);
+ else if (fabsf(ctx->options.scale - 1) > 1e-6f || fabsf(ctx->options.mean) > 1e-6f)
+ status |= ov_preprocess_output_set_element_type(output_tensor_info, F32);
+ else
+ status |= ov_preprocess_output_set_element_type(output_tensor_info, U8);
+ if (status != OK) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to set output element type\n");
+ ret = ov2_map_error(status, NULL);
+ goto err;
+ }
+ ov_preprocess_output_tensor_info_free(output_tensor_info);
+ output_tensor_info = NULL;
+ ov_preprocess_output_info_free(ov_model->output_info);
+ ov_model->output_info = NULL;
+ }
// set preprocess steps.
if (fabsf(ctx->options.scale - 1) > 1e-6f || fabsf(ctx->options.mean) > 1e-6f) {
ov_preprocess_preprocess_steps_t* input_process_steps = NULL;
@@ -667,11 +723,18 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char *
status |= ov_preprocess_preprocess_steps_scale(input_process_steps, ctx->options.scale);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR, "Failed to set preprocess steps\n");
+ ov_preprocess_preprocess_steps_free(input_process_steps);
+ input_process_steps = NULL;
ret = ov2_map_error(status, NULL);
goto err;
}
ov_preprocess_preprocess_steps_free(input_process_steps);
+ input_process_steps = NULL;
}
+ ov_preprocess_input_tensor_info_free(input_tensor_info);
+ input_tensor_info = NULL;
+ ov_preprocess_input_info_free(ov_model->input_info);
+ ov_model->input_info = NULL;
//update model
if(ov_model->ov_model)
@@ -679,20 +742,33 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char *
status = ov_preprocess_prepostprocessor_build(ov_model->preprocess, &ov_model->ov_model);
if (status != OK) {
av_log(ctx, AV_LOG_ERROR, "Failed to update OV model\n");
+ ov_model_free(tmp_ov_model);
+ tmp_ov_model = NULL;
ret = ov2_map_error(status, NULL);
goto err;
}
ov_model_free(tmp_ov_model);
//update output_port
- if (ov_model->output_port) {
- ov_output_const_port_free(ov_model->output_port);
- ov_model->output_port = NULL;
- }
- status = ov_model_const_output_by_name(ov_model->ov_model, output_name, &ov_model->output_port);
- if (status != OK) {
- av_log(ctx, AV_LOG_ERROR, "Failed to get output port.\n");
- goto err;
+ if (!ov_model->output_ports) {
+ ov_model->output_ports = av_calloc(nb_outputs, sizeof(*ov_model->output_ports));
+ if (!ov_model->output_ports) {
+ ret = AVERROR(ENOMEM);
+ goto err;
+ }
+ } else
+ for (int i = 0; i < nb_outputs; i++) {
+ ov_output_const_port_free(ov_model->output_ports[i]);
+ ov_model->output_ports[i] = NULL;
+ }
+
+ for (int i = 0; i < nb_outputs; i++) {
+ status = ov_model_const_output_by_name(ov_model->ov_model, output_names[i],
+ &ov_model->output_ports[i]);
+ if (status != OK) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to get output port %s.\n", output_names[i]);
+ goto err;
+ }
}
//compile network
status = ov_core_compile_model(ov_model->core, ov_model->ov_model, device, 0, &ov_model->compiled_model);
@@ -701,6 +777,7 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char *
goto err;
}
ov_preprocess_input_model_info_free(input_model_info);
+ input_model_info = NULL;
ov_layout_free(NCHW_layout);
ov_layout_free(NHWC_layout);
#else
@@ -745,6 +822,7 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char *
ret = DNN_GENERIC_ERROR;
goto err;
}
+ ov_model->nb_outputs = 1;
// all models in openvino open model zoo use BGR with range [0.0f, 255.0f] as input,
// we don't have a AVPixelFormat to describe it, so we'll use AV_PIX_FMT_BGR24 and
@@ -848,6 +926,10 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char *
err:
#if HAVE_OPENVINO2
+ if (output_tensor_info)
+ ov_preprocess_output_tensor_info_free(output_tensor_info);
+ if (ov_model->output_info)
+ ov_preprocess_output_info_free(ov_model->output_info);
if (NCHW_layout)
ov_layout_free(NCHW_layout);
if (NHWC_layout)
@@ -1204,11 +1286,6 @@ static int get_output_ov(void *model, const char *input_name, int input_width, i
}
}
- status = ov_model_const_output_by_name(ov_model->ov_model, output_name, &ov_model->output_port);
- if (status != OK) {
- av_log(ctx, AV_LOG_ERROR, "Failed to get output port.\n");
- return ov2_map_error(status, NULL);
- }
if (!ov_model->compiled_model) {
#else
if (ctx->options.input_resizable) {
@@ -1224,7 +1301,7 @@ static int get_output_ov(void *model, const char *input_name, int input_width, i
}
if (!ov_model->exe_network) {
#endif
- ret = init_model_ov(ov_model, input_name, output_name);
+ ret = init_model_ov(ov_model, input_name, &output_name, 1);
if (ret != 0) {
av_log(ctx, AV_LOG_ERROR, "Failed init OpenVINO exectuable network or inference request\n");
return ret;
@@ -1397,7 +1474,8 @@ static int dnn_execute_model_ov(const DNNModel *model, DNNExecBaseParams *exec_p
#else
if (!ov_model->exe_network) {
#endif
- ret = init_model_ov(ov_model, exec_params->input_name, exec_params->output_names[0]);
+ ret = init_model_ov(ov_model, exec_params->input_name,
+ exec_params->output_names, exec_params->nb_output);
if (ret != 0) {
av_log(ctx, AV_LOG_ERROR, "Failed init OpenVINO exectuable network or inference request\n");
return ret;
diff --git a/libavfilter/vf_dnn_detect.c b/libavfilter/vf_dnn_detect.c
index 3464af86c8..5862100b86 100644
--- a/libavfilter/vf_dnn_detect.c
+++ b/libavfilter/vf_dnn_detect.c
@@ -360,11 +360,11 @@ static int dnn_detect_post_proc_ssd(AVFrame *frame, DNNData *output, AVFilterCon
break;
}
}
-
return 0;
}
-static int dnn_detect_post_proc_ov(AVFrame *frame, DNNData *output, AVFilterContext *filter_ctx)
+static int dnn_detect_post_proc_ov(AVFrame *frame, DNNData *output, int nb_outputs,
+ AVFilterContext *filter_ctx)
{
AVFrameSideData *sd;
DnnDetectContext *ctx = filter_ctx->priv;
@@ -472,7 +472,7 @@ static int dnn_detect_post_proc(AVFrame *frame, DNNData *output, uint32_t nb, AV
DnnContext *dnn_ctx = &ctx->dnnctx;
switch (dnn_ctx->backend_type) {
case DNN_OV:
- return dnn_detect_post_proc_ov(frame, output, filter_ctx);
+ return dnn_detect_post_proc_ov(frame, output, nb, filter_ctx);
case DNN_TF:
return dnn_detect_post_proc_tf(frame, output, filter_ctx);
default:
@@ -559,11 +559,6 @@ static int check_output_nb(DnnDetectContext *ctx, DNNBackendType backend_type, i
}
return 0;
case DNN_OV:
- if (output_nb != 1) {
- av_log(ctx, AV_LOG_ERROR, "Dnn detect filter with openvino backend needs 1 output only, \
- but get %d instead\n", output_nb);
- return AVERROR(EINVAL);
- }
return 0;
default:
avpriv_report_missing_feature(ctx, "Dnn detect filter does not support current backend\n");
More information about the ffmpeg-cvslog
mailing list