diff mbox series

[FFmpeg-devel,8/8] vf_dnn_processing.c: add async support

Message ID 20201222074741.6511-1-yejun.guo@intel.com
State Accepted
Commit c720286ee3bf59420e6f6ddd1f999f2f327c8967
Headers show
Series [FFmpeg-devel,1/8] vf_dnn_processing.c: replace filter_frame with activate func
Related show

Checks

Context Check Description
andriy/x86_make success Make finished
andriy/x86_make_fate success Make fate finished
andriy/PPC64_make success Make finished
andriy/PPC64_make_fate success Make fate finished

Commit Message

Guo, Yejun Dec. 22, 2020, 7:47 a.m. UTC
Signed-off-by: Xie, Lin <lin.xie@intel.com>
Signed-off-by: Wu Zhiwen <zhiwen.wu@intel.com>
Signed-off-by: Guo, Yejun <yejun.guo@intel.com>
---
 doc/filters.texi                |  4 ++
 libavfilter/vf_dnn_processing.c | 78 ++++++++++++++++++++++++++++++++-
 2 files changed, 81 insertions(+), 1 deletion(-)

Comments

Guo, Yejun Dec. 28, 2020, 12:51 a.m. UTC | #1
> -----Original Message-----
> From: Guo, Yejun <yejun.guo@intel.com>
> Sent: 2020年12月22日 15:48
> To: ffmpeg-devel@ffmpeg.org
> Cc: Guo, Yejun <yejun.guo@intel.com>
> Subject: [PATCH 8/8] vf_dnn_processing.c: add async support
> 
> Signed-off-by: Xie, Lin <lin.xie@intel.com>
> Signed-off-by: Wu Zhiwen <zhiwen.wu@intel.com>
> Signed-off-by: Guo, Yejun <yejun.guo@intel.com>
> ---
>  doc/filters.texi                |  4 ++
>  libavfilter/vf_dnn_processing.c | 78
> ++++++++++++++++++++++++++++++++-
>  2 files changed, 81 insertions(+), 1 deletion(-)
> 
> diff --git a/doc/filters.texi b/doc/filters.texi index d6a53624ee..3f10a29739
> 100644
> --- a/doc/filters.texi
> +++ b/doc/filters.texi
> @@ -9959,6 +9959,10 @@ Set the input name of the dnn network.
>  @item output
>  Set the output name of the dnn network.
> 
> +@item async
> +use DNN async execution if set (default: set), roll back to sync
> +execution if the backend does not support async.
> +
>  @end table
> 
>  @subsection Examples
> diff --git a/libavfilter/vf_dnn_processing.c b/libavfilter/vf_dnn_processing.c
> index 342b06e6ae..da4508b50e 100644
> --- a/libavfilter/vf_dnn_processing.c
> +++ b/libavfilter/vf_dnn_processing.c
> @@ -42,6 +42,7 @@ typedef struct DnnProcessingContext {
>      char *model_inputname;
>      char *model_outputname;
>      char *backend_options;
> +    int async;
> 
>      DNNModule *dnn_module;
>      DNNModel *model;
> @@ -65,6 +66,7 @@ static const AVOption dnn_processing_options[] = {
>      { "input",       "input name of the model",
> OFFSET(model_inputname),  AV_OPT_TYPE_STRING,    { .str = NULL }, 0, 0,
> FLAGS },
>      { "output",      "output name of the model",
> OFFSET(model_outputname), AV_OPT_TYPE_STRING,    { .str = NULL }, 0, 0,
> FLAGS },
>      { "options",     "backend options",
> OFFSET(backend_options),  AV_OPT_TYPE_STRING,    { .str = NULL }, 0, 0,
> FLAGS },
> +    { "async",       "use DNN async inference",    OFFSET(async),
> AV_OPT_TYPE_BOOL,      { .i64 = 1},     0, 1, FLAGS},
>      { NULL }
>  };
> 
> @@ -103,6 +105,11 @@ static av_cold int init(AVFilterContext *context)
>          return AVERROR(EINVAL);
>      }
> 
> +    if (!ctx->dnn_module->execute_model_async && ctx->async) {
> +        ctx->async = 0;
> +        av_log(ctx, AV_LOG_WARNING, "this backend does not support
> async execution, roll back to sync.\n");
> +    }
> +
>      return 0;
>  }
> 
> @@ -355,9 +362,78 @@ static int activate_sync(AVFilterContext *filter_ctx)
>      return FFERROR_NOT_READY;
>  }
> 
> +static int activate_async(AVFilterContext *filter_ctx) {
> +    AVFilterLink *inlink = filter_ctx->inputs[0];
> +    AVFilterLink *outlink = filter_ctx->outputs[0];
> +    DnnProcessingContext *ctx = (DnnProcessingContext *)filter_ctx->priv;
> +    AVFrame *in = NULL, *out = NULL;
> +    int64_t pts;
> +    int ret, status;
> +    int got_frame = 0;
> +    int async_state;
> +
> +    FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink);
> +
> +    do {
> +        // drain all input frames
> +        ret = ff_inlink_consume_frame(inlink, &in);
> +        if (ret < 0)
> +            return ret;
> +        if (ret > 0) {
> +            out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
> +            if (!out) {
> +                av_frame_free(&in);
> +                return AVERROR(ENOMEM);
> +            }
> +            av_frame_copy_props(out, in);
> +            if ((ctx->dnn_module->execute_model_async)(ctx->model,
> ctx->model_inputname, in,
> +                                                       (const char
> **)&ctx->model_outputname, 1, out) != DNN_SUCCESS) {
> +                return FFERROR_NOT_READY;
> +            }
> +        }
> +    } while (ret > 0);
> +
> +    // drain all processed frames
> +    do {
> +        AVFrame *in_frame = NULL;
> +        AVFrame *out_frame = NULL;
> +        async_state = (ctx->dnn_module->get_async_result)(ctx->model,
> &in_frame, &out_frame);
> +        if (out_frame) {
> +            if (isPlanarYUV(in_frame->format))
> +                copy_uv_planes(ctx, out_frame, in_frame);
> +            av_frame_free(&in_frame);
> +            ret = ff_filter_frame(outlink, out_frame);
> +            if (ret < 0)
> +                return ret;
> +            got_frame = 1;
> +        }
> +    } while (async_state == DAST_SUCCESS);
> +
> +    // if frame got, schedule to next filter
> +    if (got_frame)
> +        return 0;
> +
> +    if (ff_inlink_acknowledge_status(inlink, &status, &pts)) {
> +        if (status == AVERROR_EOF) {
> +            ff_outlink_set_status(outlink, status, pts);
> +            return ret;
> +        }
> +    }
> +
> +    FF_FILTER_FORWARD_WANTED(outlink, inlink);
> +
> +    return FFERROR_NOT_READY;
> +}
> +
>  static int activate(AVFilterContext *filter_ctx)  {
> -    return activate_sync(filter_ctx);
> +    DnnProcessingContext *ctx = filter_ctx->priv;
> +
> +    if (ctx->async)
> +        return activate_async(filter_ctx);
> +    else
> +        return activate_sync(filter_ctx);
>  }
> 
>  static av_cold void uninit(AVFilterContext *ctx)
> --
> 2.17.1
any comment? plan to push tomorrow if no other comment, thanks.
diff mbox series

Patch

diff --git a/doc/filters.texi b/doc/filters.texi
index d6a53624ee..3f10a29739 100644
--- a/doc/filters.texi
+++ b/doc/filters.texi
@@ -9959,6 +9959,10 @@  Set the input name of the dnn network.
 @item output
 Set the output name of the dnn network.
 
+@item async
+use DNN async execution if set (default: set),
+roll back to sync execution if the backend does not support async.
+
 @end table
 
 @subsection Examples
diff --git a/libavfilter/vf_dnn_processing.c b/libavfilter/vf_dnn_processing.c
index 342b06e6ae..da4508b50e 100644
--- a/libavfilter/vf_dnn_processing.c
+++ b/libavfilter/vf_dnn_processing.c
@@ -42,6 +42,7 @@  typedef struct DnnProcessingContext {
     char *model_inputname;
     char *model_outputname;
     char *backend_options;
+    int async;
 
     DNNModule *dnn_module;
     DNNModel *model;
@@ -65,6 +66,7 @@  static const AVOption dnn_processing_options[] = {
     { "input",       "input name of the model",    OFFSET(model_inputname),  AV_OPT_TYPE_STRING,    { .str = NULL }, 0, 0, FLAGS },
     { "output",      "output name of the model",   OFFSET(model_outputname), AV_OPT_TYPE_STRING,    { .str = NULL }, 0, 0, FLAGS },
     { "options",     "backend options",            OFFSET(backend_options),  AV_OPT_TYPE_STRING,    { .str = NULL }, 0, 0, FLAGS },
+    { "async",       "use DNN async inference",    OFFSET(async),            AV_OPT_TYPE_BOOL,      { .i64 = 1},     0, 1, FLAGS},
     { NULL }
 };
 
@@ -103,6 +105,11 @@  static av_cold int init(AVFilterContext *context)
         return AVERROR(EINVAL);
     }
 
+    if (!ctx->dnn_module->execute_model_async && ctx->async) {
+        ctx->async = 0;
+        av_log(ctx, AV_LOG_WARNING, "this backend does not support async execution, roll back to sync.\n");
+    }
+
     return 0;
 }
 
@@ -355,9 +362,78 @@  static int activate_sync(AVFilterContext *filter_ctx)
     return FFERROR_NOT_READY;
 }
 
+static int activate_async(AVFilterContext *filter_ctx)
+{
+    AVFilterLink *inlink = filter_ctx->inputs[0];
+    AVFilterLink *outlink = filter_ctx->outputs[0];
+    DnnProcessingContext *ctx = (DnnProcessingContext *)filter_ctx->priv;
+    AVFrame *in = NULL, *out = NULL;
+    int64_t pts;
+    int ret, status;
+    int got_frame = 0;
+    int async_state;
+
+    FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink);
+
+    do {
+        // drain all input frames
+        ret = ff_inlink_consume_frame(inlink, &in);
+        if (ret < 0)
+            return ret;
+        if (ret > 0) {
+            out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
+            if (!out) {
+                av_frame_free(&in);
+                return AVERROR(ENOMEM);
+            }
+            av_frame_copy_props(out, in);
+            if ((ctx->dnn_module->execute_model_async)(ctx->model, ctx->model_inputname, in,
+                                                       (const char **)&ctx->model_outputname, 1, out) != DNN_SUCCESS) {
+                return FFERROR_NOT_READY;
+            }
+        }
+    } while (ret > 0);
+
+    // drain all processed frames
+    do {
+        AVFrame *in_frame = NULL;
+        AVFrame *out_frame = NULL;
+        async_state = (ctx->dnn_module->get_async_result)(ctx->model, &in_frame, &out_frame);
+        if (out_frame) {
+            if (isPlanarYUV(in_frame->format))
+                copy_uv_planes(ctx, out_frame, in_frame);
+            av_frame_free(&in_frame);
+            ret = ff_filter_frame(outlink, out_frame);
+            if (ret < 0)
+                return ret;
+            got_frame = 1;
+        }
+    } while (async_state == DAST_SUCCESS);
+
+    // if frame got, schedule to next filter
+    if (got_frame)
+        return 0;
+
+    if (ff_inlink_acknowledge_status(inlink, &status, &pts)) {
+        if (status == AVERROR_EOF) {
+            ff_outlink_set_status(outlink, status, pts);
+            return ret;
+        }
+    }
+
+    FF_FILTER_FORWARD_WANTED(outlink, inlink);
+
+    return FFERROR_NOT_READY;
+}
+
 static int activate(AVFilterContext *filter_ctx)
 {
-    return activate_sync(filter_ctx);
+    DnnProcessingContext *ctx = filter_ctx->priv;
+
+    if (ctx->async)
+        return activate_async(filter_ctx);
+    else
+        return activate_sync(filter_ctx);
 }
 
 static av_cold void uninit(AVFilterContext *ctx)