diff mbox

[FFmpeg-devel,GSOC] TensorFlow backend introduction for DNN module

Message ID CAAeE=qomw4B5nJ5VB69DQyxr2mj4wR+m3_2-kAUDRGZ3bpZeyg@mail.gmail.com
State Superseded
Headers show

Commit Message

Sergey Lavrushkin June 5, 2018, 11:23 p.m. UTC 
2018-06-05 17:20 GMT+03:00 James Almer <jamrial@gmail.com>:

> On 6/3/2018 3:02 PM, Sergey Lavrushkin wrote:
> > diff --git a/libavfilter/vf_srcnn.c b/libavfilter/vf_srcnn.c
> > index d6efe9b478..5c5e26b33a 100644
> > --- a/libavfilter/vf_srcnn.c
> > +++ b/libavfilter/vf_srcnn.c
> > @@ -41,7 +41,6 @@ typedef struct SRCNNContext {
> >      DNNData input_output;
> >  } SRCNNContext;
> >
> > -
> >  #define OFFSET(x) offsetof(SRCNNContext, x)
> >  #define FLAGS AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM
> >  static const AVOption srcnn_options[] = {
> > @@ -55,10 +54,19 @@ static av_cold int init(AVFilterContext* context)
> >  {
> >      SRCNNContext* srcnn_context = context->priv;
> >
> > -    srcnn_context->dnn_module = ff_get_dnn_module(DNN_NATIVE);
> > +    srcnn_context->dnn_module = ff_get_dnn_module(DNN_TF);
>
> This should be a filter AVOption, not hardcoded to one or another. What
> if i, for whatever reason, want to use the native backend when i have
> libtensorflow enabled?
>
> >      if (!srcnn_context->dnn_module){
> > -        av_log(context, AV_LOG_ERROR, "could not create dnn module\n");
> > -        return AVERROR(ENOMEM);
> > +        srcnn_context->dnn_module = ff_get_dnn_module(DNN_NATIVE);
> > +        if (!srcnn_context->dnn_module){
> > +            av_log(context, AV_LOG_ERROR, "could not create dnn
> module\n");
> > +            return AVERROR(ENOMEM);
> > +        }
> > +        else{
> > +            av_log(context, AV_LOG_INFO, "using native backend for DNN
> inference\n");
>
> VERBOSE, not INFO
>
> > +        }
> > +    }
> > +    else{
> > +        av_log(context, AV_LOG_INFO, "using tensorflow backend for DNN
> inference\n");
>
> Ditto.
>
> >      }
> >      if (!srcnn_context->model_filename){
> >          av_log(context, AV_LOG_INFO, "model file for network was not
> specified, using default network for x2 upsampling\n");


Here is the patch, that fixes described issues.

Comments

Pedro Arthur June 6, 2018, 2:22 p.m. UTC | #1 
Hi,

2018-06-05 20:23 GMT-03:00 Sergey Lavrushkin <dualfal@gmail.com>:
> Here is the patch, that fixes described issues.
When I try to run (video input), when tf is not enabled in configure it crashes.


$ffmpeg -i in.mp4 -vf srcnn=dnn_backend=tensorflow out.mp4

ffmpeg version N-91232-g256386fd3e Copyright (c) 2000-2018 the FFmpeg developers
  built with gcc 7 (Ubuntu 7.3.0-16ubuntu3)
  configuration:
  libavutil      56. 18.102 / 56. 18.102
  libavcodec     58. 19.105 / 58. 19.105
  libavformat    58. 17.100 / 58. 17.100
  libavdevice    58.  4.100 / 58.  4.100
  libavfilter     7. 25.100 /  7. 25.100
  libswscale      5.  2.100 /  5.  2.100
  libswresample   3.  2.100 /  3.  2.100
Input #0, mov,mp4,m4a,3gp,3g2,mj2, from 'in.mp4':
  Metadata:
    major_brand     : isom
    minor_version   : 512
    compatible_brands: isomiso2mp41
    encoder         : Lavf58.17.100
  Duration: 00:06:13.70, start: 0.000000, bitrate: 5912 kb/s
    Stream #0:0(und): Video: mpeg4 (Simple Profile) (mp4v /
0x7634706D), yuv420p, 1280x720 [SAR 1:1 DAR 16:9], 5777 kb/s, 29.97
fps, 29.97 tbr, 30k tbn, 30k tbc (default)
    Metadata:
      handler_name    : VideoHandler
    Stream #0:1(und): Audio: aac (LC) (mp4a / 0x6134706D), 44100 Hz,
stereo, fltp, 128 kb/s (default)
    Metadata:
      handler_name    : SoundHandler
Stream mapping:
  Stream #0:0 -> #0:0 (mpeg4 (native) -> mpeg4 (native))
  Stream #0:1 -> #0:1 (aac (native) -> aac (native))
Press [q] to stop, [?] for help
free(): invalid pointer
Aborted (core dumped)



When the output is an image, t does not crashes but neither fallback to native


$ffmpeg -i in.jpg -vf srcnn=dnn_backend=tensorflow out.png

ffmpeg version N-91232-g256386fd3e Copyright (c) 2000-2018 the FFmpeg developers
  built with gcc 7 (Ubuntu 7.3.0-16ubuntu3)
  configuration:
  libavutil      56. 18.102 / 56. 18.102
  libavcodec     58. 19.105 / 58. 19.105
  libavformat    58. 17.100 / 58. 17.100
  libavdevice    58.  4.100 / 58.  4.100
  libavfilter     7. 25.100 /  7. 25.100
  libswscale      5.  2.100 /  5.  2.100
  libswresample   3.  2.100 /  3.  2.100
Input #0, image2, from 'in.jpg':
  Duration: 00:00:00.04, start: 0.000000, bitrate: 43469 kb/s
    Stream #0:0: Video: mjpeg, yuvj444p(pc, bt470bg/unknown/unknown),
1192x670 [SAR 1:1 DAR 596:335], 25 tbr, 25 tbn, 25 tbc
Stream mapping:
  Stream #0:0 -> #0:0 (mjpeg (native) -> png (native))
Press [q] to stop, [?] for help
[Parsed_srcnn_0 @ 0x557d3ea55980] could not create DNN module for
requested backend
[AVFilterGraph @ 0x557d3ea102c0] Error initializing filter 'srcnn'
with args 'dnn_backend=tensorflow'
Error reinitializing filters!
Failed to inject frame into filter network: Cannot allocate memory
Error while processing the decoded data for stream #0:0
Conversion failed!


I think you could disable the tensorflow option if it is not enable in
configure or fallback to native, either solution is ok for me.
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> http://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
diff mbox

Patch

From 971e15b4b1e3f2747aa07d0221f99226cba622ac Mon Sep 17 00:00:00 2001
From: Sergey Lavrushkin <dualfal@gmail.com>
Date: Wed, 6 Jun 2018 01:44:40 +0300
Subject: [PATCH] libavfilter/vf_srcnn.c: adds DNN module backend AVOption,
 changes AV_LOG_INFO message to AV_LOG_VERBOSE.

---
 libavfilter/vf_srcnn.c | 23 +++++++++--------------
 1 file changed, 9 insertions(+), 14 deletions(-)

diff --git a/libavfilter/vf_srcnn.c b/libavfilter/vf_srcnn.c
index 5c5e26b33a..17e380503e 100644
--- a/libavfilter/vf_srcnn.c
+++ b/libavfilter/vf_srcnn.c
@@ -36,6 +36,7 @@  typedef struct SRCNNContext {
 
     char* model_filename;
     float* input_output_buf;
+    DNNBackendType backend_type;
     DNNModule* dnn_module;
     DNNModel* model;
     DNNData input_output;
@@ -44,6 +45,9 @@  typedef struct SRCNNContext {
 #define OFFSET(x) offsetof(SRCNNContext, x)
 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM
 static const AVOption srcnn_options[] = {
+    { "dnn_backend", "DNN backend used for model execution", OFFSET(backend_type), AV_OPT_TYPE_FLAGS, { .i64 = 0 }, 0, 1, FLAGS, "backend" },
+    { "native", "native backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 0 }, 0, 0, FLAGS, "backend" },
+    { "tensorflow", "tensorflow backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, 0, 0, FLAGS, "backend" },
     { "model_filename", "path to model file specifying network architecture and its parameters", OFFSET(model_filename), AV_OPT_TYPE_STRING, {.str=NULL}, 0, 0, FLAGS },
     { NULL }
 };
@@ -54,29 +58,20 @@  static av_cold int init(AVFilterContext* context)
 {
     SRCNNContext* srcnn_context = context->priv;
 
-    srcnn_context->dnn_module = ff_get_dnn_module(DNN_TF);
+    srcnn_context->dnn_module = ff_get_dnn_module(srcnn_context->backend_type);
     if (!srcnn_context->dnn_module){
-        srcnn_context->dnn_module = ff_get_dnn_module(DNN_NATIVE);
-        if (!srcnn_context->dnn_module){
-            av_log(context, AV_LOG_ERROR, "could not create dnn module\n");
-            return AVERROR(ENOMEM);
-        }
-        else{
-            av_log(context, AV_LOG_INFO, "using native backend for DNN inference\n");
-        }
-    }
-    else{
-        av_log(context, AV_LOG_INFO, "using tensorflow backend for DNN inference\n");
+        av_log(context, AV_LOG_ERROR, "could not create DNN module for requested backend\n");
+        return AVERROR(ENOMEM);
     }
     if (!srcnn_context->model_filename){
-        av_log(context, AV_LOG_INFO, "model file for network was not specified, using default network for x2 upsampling\n");
+        av_log(context, AV_LOG_VERBOSE, "model file for network was not specified, using default network for x2 upsampling\n");
         srcnn_context->model = (srcnn_context->dnn_module->load_default_model)(DNN_SRCNN);
     }
     else{
         srcnn_context->model = (srcnn_context->dnn_module->load_model)(srcnn_context->model_filename);
     }
     if (!srcnn_context->model){
-        av_log(context, AV_LOG_ERROR, "could not load dnn model\n");
+        av_log(context, AV_LOG_ERROR, "could not load DNN model\n");
         return AVERROR(EIO);
     }
 
-- 
2.14.1