[FFmpeg-devel] [PATCH v4 7/7] libavfilter: Send only input frame for DNN Detect and Classify
Shubhanshu Saxena
shubhanshu.e01 at gmail.com
Wed Aug 25 13:39:40 EEST 2021
This commit updates the following two filters to send only the
input frame and send NULL as output frame instead of input frame
to the DNN backends.
1. vf_dnn_detect
2. vf_dnn_classify
Signed-off-by: Shubhanshu Saxena <shubhanshu.e01 at gmail.com>
---
libavfilter/dnn/dnn_backend_common.c | 2 +-
libavfilter/dnn/dnn_backend_openvino.c | 5 +++--
libavfilter/dnn/dnn_backend_tf.c | 2 +-
libavfilter/vf_dnn_classify.c | 14 ++++++--------
libavfilter/vf_dnn_detect.c | 14 ++++++--------
5 files changed, 17 insertions(+), 20 deletions(-)
diff --git a/libavfilter/dnn/dnn_backend_common.c b/libavfilter/dnn/dnn_backend_common.c
index d2bc016fef..6a9c4cc87f 100644
--- a/libavfilter/dnn/dnn_backend_common.c
+++ b/libavfilter/dnn/dnn_backend_common.c
@@ -38,7 +38,7 @@ int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func
return AVERROR(EINVAL);
}
- if (!exec_params->out_frame) {
+ if (!exec_params->out_frame && func_type == DFT_PROCESS_FRAME) {
av_log(ctx, AV_LOG_ERROR, "out frame is NULL when execute model.\n");
return AVERROR(EINVAL);
}
diff --git a/libavfilter/dnn/dnn_backend_openvino.c b/libavfilter/dnn/dnn_backend_openvino.c
index 76dc06c6d7..f5b1454d21 100644
--- a/libavfilter/dnn/dnn_backend_openvino.c
+++ b/libavfilter/dnn/dnn_backend_openvino.c
@@ -272,14 +272,14 @@ static void infer_completion_callback(void *args)
av_log(ctx, AV_LOG_ERROR, "detect filter needs to provide post proc\n");
return;
}
- ov_model->model->detect_post_proc(task->out_frame, &output, 1, ov_model->model->filter_ctx);
+ ov_model->model->detect_post_proc(task->in_frame, &output, 1, ov_model->model->filter_ctx);
break;
case DFT_ANALYTICS_CLASSIFY:
if (!ov_model->model->classify_post_proc) {
av_log(ctx, AV_LOG_ERROR, "classify filter needs to provide post proc\n");
return;
}
- ov_model->model->classify_post_proc(task->out_frame, &output, request->lltasks[i]->bbox_index, ov_model->model->filter_ctx);
+ ov_model->model->classify_post_proc(task->in_frame, &output, request->lltasks[i]->bbox_index, ov_model->model->filter_ctx);
break;
default:
av_assert0(!"should not reach here");
@@ -819,6 +819,7 @@ DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, DNNExecBaseParams *
if (model->func_type == DFT_ANALYTICS_CLASSIFY) {
// Classification filter has not been completely
// tested with the sync mode. So, do not support now.
+ avpriv_report_missing_feature(ctx, "classify for sync execution");
return DNN_ERROR;
}
diff --git a/libavfilter/dnn/dnn_backend_tf.c b/libavfilter/dnn/dnn_backend_tf.c
index dfac58b357..c95cad7944 100644
--- a/libavfilter/dnn/dnn_backend_tf.c
+++ b/libavfilter/dnn/dnn_backend_tf.c
@@ -1069,7 +1069,7 @@ static void infer_completion_callback(void *args) {
av_log(ctx, AV_LOG_ERROR, "Detect filter needs provide post proc\n");
return;
}
- tf_model->model->detect_post_proc(task->out_frame, outputs, task->nb_output, tf_model->model->filter_ctx);
+ tf_model->model->detect_post_proc(task->in_frame, outputs, task->nb_output, tf_model->model->filter_ctx);
break;
default:
av_log(ctx, AV_LOG_ERROR, "Tensorflow backend does not support this kind of dnn filter now\n");
diff --git a/libavfilter/vf_dnn_classify.c b/libavfilter/vf_dnn_classify.c
index d5ee65d403..d1ba8dffbc 100644
--- a/libavfilter/vf_dnn_classify.c
+++ b/libavfilter/vf_dnn_classify.c
@@ -225,13 +225,12 @@ static int dnn_classify_flush_frame(AVFilterLink *outlink, int64_t pts, int64_t
AVFrame *in_frame = NULL;
AVFrame *out_frame = NULL;
async_state = ff_dnn_get_result(&ctx->dnnctx, &in_frame, &out_frame);
- if (out_frame) {
- av_assert0(in_frame == out_frame);
- ret = ff_filter_frame(outlink, out_frame);
+ if (async_state == DAST_SUCCESS) {
+ ret = ff_filter_frame(outlink, in_frame);
if (ret < 0)
return ret;
if (out_pts)
- *out_pts = out_frame->pts + pts;
+ *out_pts = in_frame->pts + pts;
}
av_usleep(5000);
} while (async_state >= DAST_NOT_READY);
@@ -258,7 +257,7 @@ static int dnn_classify_activate(AVFilterContext *filter_ctx)
if (ret < 0)
return ret;
if (ret > 0) {
- if (ff_dnn_execute_model_classification(&ctx->dnnctx, in, in, ctx->target) != DNN_SUCCESS) {
+ if (ff_dnn_execute_model_classification(&ctx->dnnctx, in, NULL, ctx->target) != DNN_SUCCESS) {
return AVERROR(EIO);
}
}
@@ -269,9 +268,8 @@ static int dnn_classify_activate(AVFilterContext *filter_ctx)
AVFrame *in_frame = NULL;
AVFrame *out_frame = NULL;
async_state = ff_dnn_get_result(&ctx->dnnctx, &in_frame, &out_frame);
- if (out_frame) {
- av_assert0(in_frame == out_frame);
- ret = ff_filter_frame(outlink, out_frame);
+ if (async_state == DAST_SUCCESS) {
+ ret = ff_filter_frame(outlink, in_frame);
if (ret < 0)
return ret;
got_frame = 1;
diff --git a/libavfilter/vf_dnn_detect.c b/libavfilter/vf_dnn_detect.c
index 809d70b930..637874b2a1 100644
--- a/libavfilter/vf_dnn_detect.c
+++ b/libavfilter/vf_dnn_detect.c
@@ -368,13 +368,12 @@ static int dnn_detect_flush_frame(AVFilterLink *outlink, int64_t pts, int64_t *o
AVFrame *in_frame = NULL;
AVFrame *out_frame = NULL;
async_state = ff_dnn_get_result(&ctx->dnnctx, &in_frame, &out_frame);
- if (out_frame) {
- av_assert0(in_frame == out_frame);
- ret = ff_filter_frame(outlink, out_frame);
+ if (async_state == DAST_SUCCESS) {
+ ret = ff_filter_frame(outlink, in_frame);
if (ret < 0)
return ret;
if (out_pts)
- *out_pts = out_frame->pts + pts;
+ *out_pts = in_frame->pts + pts;
}
av_usleep(5000);
} while (async_state >= DAST_NOT_READY);
@@ -401,7 +400,7 @@ static int dnn_detect_activate(AVFilterContext *filter_ctx)
if (ret < 0)
return ret;
if (ret > 0) {
- if (ff_dnn_execute_model(&ctx->dnnctx, in, in) != DNN_SUCCESS) {
+ if (ff_dnn_execute_model(&ctx->dnnctx, in, NULL) != DNN_SUCCESS) {
return AVERROR(EIO);
}
}
@@ -412,9 +411,8 @@ static int dnn_detect_activate(AVFilterContext *filter_ctx)
AVFrame *in_frame = NULL;
AVFrame *out_frame = NULL;
async_state = ff_dnn_get_result(&ctx->dnnctx, &in_frame, &out_frame);
- if (out_frame) {
- av_assert0(in_frame == out_frame);
- ret = ff_filter_frame(outlink, out_frame);
+ if (async_state == DAST_SUCCESS) {
+ ret = ff_filter_frame(outlink, in_frame);
if (ret < 0)
return ret;
got_frame = 1;
--
2.25.1
More information about the ffmpeg-devel
mailing list