Message ID | 20201222074741.6511-1-yejun.guo@intel.com |
---|---|
State | Accepted |
Commit | c720286ee3bf59420e6f6ddd1f999f2f327c8967 |
Headers | show |
Series | [FFmpeg-devel,1/8] vf_dnn_processing.c: replace filter_frame with activate func | expand |
Context | Check | Description |
---|---|---|
andriy/x86_make | success | Make finished |
andriy/x86_make_fate | success | Make fate finished |
andriy/PPC64_make | success | Make finished |
andriy/PPC64_make_fate | success | Make fate finished |
> -----Original Message----- > From: Guo, Yejun <yejun.guo@intel.com> > Sent: 2020年12月22日 15:48 > To: ffmpeg-devel@ffmpeg.org > Cc: Guo, Yejun <yejun.guo@intel.com> > Subject: [PATCH 8/8] vf_dnn_processing.c: add async support > > Signed-off-by: Xie, Lin <lin.xie@intel.com> > Signed-off-by: Wu Zhiwen <zhiwen.wu@intel.com> > Signed-off-by: Guo, Yejun <yejun.guo@intel.com> > --- > doc/filters.texi | 4 ++ > libavfilter/vf_dnn_processing.c | 78 > ++++++++++++++++++++++++++++++++- > 2 files changed, 81 insertions(+), 1 deletion(-) > > diff --git a/doc/filters.texi b/doc/filters.texi index d6a53624ee..3f10a29739 > 100644 > --- a/doc/filters.texi > +++ b/doc/filters.texi > @@ -9959,6 +9959,10 @@ Set the input name of the dnn network. > @item output > Set the output name of the dnn network. > > +@item async > +use DNN async execution if set (default: set), roll back to sync > +execution if the backend does not support async. > + > @end table > > @subsection Examples > diff --git a/libavfilter/vf_dnn_processing.c b/libavfilter/vf_dnn_processing.c > index 342b06e6ae..da4508b50e 100644 > --- a/libavfilter/vf_dnn_processing.c > +++ b/libavfilter/vf_dnn_processing.c > @@ -42,6 +42,7 @@ typedef struct DnnProcessingContext { > char *model_inputname; > char *model_outputname; > char *backend_options; > + int async; > > DNNModule *dnn_module; > DNNModel *model; > @@ -65,6 +66,7 @@ static const AVOption dnn_processing_options[] = { > { "input", "input name of the model", > OFFSET(model_inputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, > FLAGS }, > { "output", "output name of the model", > OFFSET(model_outputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, > FLAGS }, > { "options", "backend options", > OFFSET(backend_options), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, > FLAGS }, > + { "async", "use DNN async inference", OFFSET(async), > AV_OPT_TYPE_BOOL, { .i64 = 1}, 0, 1, FLAGS}, > { NULL } > }; > > @@ -103,6 +105,11 @@ static av_cold int init(AVFilterContext *context) > return AVERROR(EINVAL); > } > > + if (!ctx->dnn_module->execute_model_async && ctx->async) { > + ctx->async = 0; > + av_log(ctx, AV_LOG_WARNING, "this backend does not support > async execution, roll back to sync.\n"); > + } > + > return 0; > } > > @@ -355,9 +362,78 @@ static int activate_sync(AVFilterContext *filter_ctx) > return FFERROR_NOT_READY; > } > > +static int activate_async(AVFilterContext *filter_ctx) { > + AVFilterLink *inlink = filter_ctx->inputs[0]; > + AVFilterLink *outlink = filter_ctx->outputs[0]; > + DnnProcessingContext *ctx = (DnnProcessingContext *)filter_ctx->priv; > + AVFrame *in = NULL, *out = NULL; > + int64_t pts; > + int ret, status; > + int got_frame = 0; > + int async_state; > + > + FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink); > + > + do { > + // drain all input frames > + ret = ff_inlink_consume_frame(inlink, &in); > + if (ret < 0) > + return ret; > + if (ret > 0) { > + out = ff_get_video_buffer(outlink, outlink->w, outlink->h); > + if (!out) { > + av_frame_free(&in); > + return AVERROR(ENOMEM); > + } > + av_frame_copy_props(out, in); > + if ((ctx->dnn_module->execute_model_async)(ctx->model, > ctx->model_inputname, in, > + (const char > **)&ctx->model_outputname, 1, out) != DNN_SUCCESS) { > + return FFERROR_NOT_READY; > + } > + } > + } while (ret > 0); > + > + // drain all processed frames > + do { > + AVFrame *in_frame = NULL; > + AVFrame *out_frame = NULL; > + async_state = (ctx->dnn_module->get_async_result)(ctx->model, > &in_frame, &out_frame); > + if (out_frame) { > + if (isPlanarYUV(in_frame->format)) > + copy_uv_planes(ctx, out_frame, in_frame); > + av_frame_free(&in_frame); > + ret = ff_filter_frame(outlink, out_frame); > + if (ret < 0) > + return ret; > + got_frame = 1; > + } > + } while (async_state == DAST_SUCCESS); > + > + // if frame got, schedule to next filter > + if (got_frame) > + return 0; > + > + if (ff_inlink_acknowledge_status(inlink, &status, &pts)) { > + if (status == AVERROR_EOF) { > + ff_outlink_set_status(outlink, status, pts); > + return ret; > + } > + } > + > + FF_FILTER_FORWARD_WANTED(outlink, inlink); > + > + return FFERROR_NOT_READY; > +} > + > static int activate(AVFilterContext *filter_ctx) { > - return activate_sync(filter_ctx); > + DnnProcessingContext *ctx = filter_ctx->priv; > + > + if (ctx->async) > + return activate_async(filter_ctx); > + else > + return activate_sync(filter_ctx); > } > > static av_cold void uninit(AVFilterContext *ctx) > -- > 2.17.1 any comment? plan to push tomorrow if no other comment, thanks.
diff --git a/doc/filters.texi b/doc/filters.texi index d6a53624ee..3f10a29739 100644 --- a/doc/filters.texi +++ b/doc/filters.texi @@ -9959,6 +9959,10 @@ Set the input name of the dnn network. @item output Set the output name of the dnn network. +@item async +use DNN async execution if set (default: set), +roll back to sync execution if the backend does not support async. + @end table @subsection Examples diff --git a/libavfilter/vf_dnn_processing.c b/libavfilter/vf_dnn_processing.c index 342b06e6ae..da4508b50e 100644 --- a/libavfilter/vf_dnn_processing.c +++ b/libavfilter/vf_dnn_processing.c @@ -42,6 +42,7 @@ typedef struct DnnProcessingContext { char *model_inputname; char *model_outputname; char *backend_options; + int async; DNNModule *dnn_module; DNNModel *model; @@ -65,6 +66,7 @@ static const AVOption dnn_processing_options[] = { { "input", "input name of the model", OFFSET(model_inputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, { "output", "output name of the model", OFFSET(model_outputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, { "options", "backend options", OFFSET(backend_options), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS }, + { "async", "use DNN async inference", OFFSET(async), AV_OPT_TYPE_BOOL, { .i64 = 1}, 0, 1, FLAGS}, { NULL } }; @@ -103,6 +105,11 @@ static av_cold int init(AVFilterContext *context) return AVERROR(EINVAL); } + if (!ctx->dnn_module->execute_model_async && ctx->async) { + ctx->async = 0; + av_log(ctx, AV_LOG_WARNING, "this backend does not support async execution, roll back to sync.\n"); + } + return 0; } @@ -355,9 +362,78 @@ static int activate_sync(AVFilterContext *filter_ctx) return FFERROR_NOT_READY; } +static int activate_async(AVFilterContext *filter_ctx) +{ + AVFilterLink *inlink = filter_ctx->inputs[0]; + AVFilterLink *outlink = filter_ctx->outputs[0]; + DnnProcessingContext *ctx = (DnnProcessingContext *)filter_ctx->priv; + AVFrame *in = NULL, *out = NULL; + int64_t pts; + int ret, status; + int got_frame = 0; + int async_state; + + FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink); + + do { + // drain all input frames + ret = ff_inlink_consume_frame(inlink, &in); + if (ret < 0) + return ret; + if (ret > 0) { + out = ff_get_video_buffer(outlink, outlink->w, outlink->h); + if (!out) { + av_frame_free(&in); + return AVERROR(ENOMEM); + } + av_frame_copy_props(out, in); + if ((ctx->dnn_module->execute_model_async)(ctx->model, ctx->model_inputname, in, + (const char **)&ctx->model_outputname, 1, out) != DNN_SUCCESS) { + return FFERROR_NOT_READY; + } + } + } while (ret > 0); + + // drain all processed frames + do { + AVFrame *in_frame = NULL; + AVFrame *out_frame = NULL; + async_state = (ctx->dnn_module->get_async_result)(ctx->model, &in_frame, &out_frame); + if (out_frame) { + if (isPlanarYUV(in_frame->format)) + copy_uv_planes(ctx, out_frame, in_frame); + av_frame_free(&in_frame); + ret = ff_filter_frame(outlink, out_frame); + if (ret < 0) + return ret; + got_frame = 1; + } + } while (async_state == DAST_SUCCESS); + + // if frame got, schedule to next filter + if (got_frame) + return 0; + + if (ff_inlink_acknowledge_status(inlink, &status, &pts)) { + if (status == AVERROR_EOF) { + ff_outlink_set_status(outlink, status, pts); + return ret; + } + } + + FF_FILTER_FORWARD_WANTED(outlink, inlink); + + return FFERROR_NOT_READY; +} + static int activate(AVFilterContext *filter_ctx) { - return activate_sync(filter_ctx); + DnnProcessingContext *ctx = filter_ctx->priv; + + if (ctx->async) + return activate_async(filter_ctx); + else + return activate_sync(filter_ctx); } static av_cold void uninit(AVFilterContext *ctx)