[FFmpeg-devel,V2,7/7] libavfilter/dnn: add more data type support for dnn model input

Submitted by Guo, Yejun on April 25, 2019, 2:14 a.m.

Details

Message ID 1556158482-15306-1-git-send-email-yejun.guo@intel.com
State New
Headers show

Commit Message

Guo, Yejun April 25, 2019, 2:14 a.m.
currently, only float is supported as model input, actually, there
are other data types, this patch adds uint8.

Signed-off-by: Guo, Yejun <yejun.guo@intel.com>
---
 libavfilter/dnn_backend_native.c |  4 +++-
 libavfilter/dnn_backend_tf.c     | 28 ++++++++++++++++++++++++----
 libavfilter/dnn_interface.h      | 10 +++++++++-
 libavfilter/vf_sr.c              |  4 +++-
 4 files changed, 39 insertions(+), 7 deletions(-)

Comments

Pedro Arthur April 29, 2019, 5:47 p.m.
Em qua, 24 de abr de 2019 às 23:15, Guo, Yejun <yejun.guo@intel.com> escreveu:
>
> currently, only float is supported as model input, actually, there
> are other data types, this patch adds uint8.
>
> Signed-off-by: Guo, Yejun <yejun.guo@intel.com>
> ---
>  libavfilter/dnn_backend_native.c |  4 +++-
>  libavfilter/dnn_backend_tf.c     | 28 ++++++++++++++++++++++++----
>  libavfilter/dnn_interface.h      | 10 +++++++++-
>  libavfilter/vf_sr.c              |  4 +++-
>  4 files changed, 39 insertions(+), 7 deletions(-)
>
> diff --git a/libavfilter/dnn_backend_native.c b/libavfilter/dnn_backend_native.c
> index 8a83c63..06fbdf3 100644
> --- a/libavfilter/dnn_backend_native.c
> +++ b/libavfilter/dnn_backend_native.c
> @@ -24,8 +24,9 @@
>   */
>
>  #include "dnn_backend_native.h"
> +#include "libavutil/avassert.h"
>
> -static DNNReturnType set_input_output_native(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output)
> +static DNNReturnType set_input_output_native(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output)
>  {
>      ConvolutionalNetwork *network = (ConvolutionalNetwork *)model;
>      InputParams *input_params;
> @@ -45,6 +46,7 @@ static DNNReturnType set_input_output_native(void *model, DNNData *input, const
>          if (input->data){
>              av_freep(&input->data);
>          }
> +        av_assert0(input->dt == DNN_FLOAT);
>          network->layers[0].output = input->data = av_malloc(cur_height * cur_width * cur_channels * sizeof(float));
>          if (!network->layers[0].output){
>              return DNN_ERROR;
> diff --git a/libavfilter/dnn_backend_tf.c b/libavfilter/dnn_backend_tf.c
> index ca6472d..ba959ae 100644
> --- a/libavfilter/dnn_backend_tf.c
> +++ b/libavfilter/dnn_backend_tf.c
> @@ -79,10 +79,31 @@ static TF_Buffer *read_graph(const char *model_filename)
>      return graph_buf;
>  }
>
> -static DNNReturnType set_input_output_tf(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output)
> +static TF_Tensor *allocate_input_tensor(const DNNInputData *input)
>  {
> -    TFModel *tf_model = (TFModel *)model;
> +    TF_DataType dt;
> +    size_t size;
>      int64_t input_dims[] = {1, input->height, input->width, input->channels};
> +    switch (input->dt) {
> +    case DNN_FLOAT:
> +        dt = TF_FLOAT;
> +        size = sizeof(float);
> +        break;
> +    case DNN_UINT8:
> +        dt = TF_UINT8;
> +        size = sizeof(char);
> +        break;
> +    default:
> +        av_assert0(!"should not reach here");
> +    }
> +
> +    return TF_AllocateTensor(dt, input_dims, 4,
> +                             input_dims[1] * input_dims[2] * input_dims[3] * size);
> +}
> +
> +static DNNReturnType set_input_output_tf(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output)
> +{
> +    TFModel *tf_model = (TFModel *)model;
>      TF_SessionOptions *sess_opts;
>      const TF_Operation *init_op = TF_GraphOperationByName(tf_model->graph, "init");
>
> @@ -95,8 +116,7 @@ static DNNReturnType set_input_output_tf(void *model, DNNData *input, const char
>      if (tf_model->input_tensor){
>          TF_DeleteTensor(tf_model->input_tensor);
>      }
> -    tf_model->input_tensor = TF_AllocateTensor(TF_FLOAT, input_dims, 4,
> -                                               input_dims[1] * input_dims[2] * input_dims[3] * sizeof(float));
> +    tf_model->input_tensor = allocate_input_tensor(input);
>      if (!tf_model->input_tensor){
>          return DNN_ERROR;
>      }
> diff --git a/libavfilter/dnn_interface.h b/libavfilter/dnn_interface.h
> index 73d226e..c24df0e 100644
> --- a/libavfilter/dnn_interface.h
> +++ b/libavfilter/dnn_interface.h
> @@ -32,6 +32,14 @@ typedef enum {DNN_SUCCESS, DNN_ERROR} DNNReturnType;
>
>  typedef enum {DNN_NATIVE, DNN_TF} DNNBackendType;
>
> +typedef enum {DNN_FLOAT, DNN_UINT8} DNNDataType;
> +
> +typedef struct DNNInputData{
> +    void *data;
> +    DNNDataType dt;
> +    int width, height, channels;
> +} DNNInputData;
> +
>  typedef struct DNNData{
>      float *data;
>      int width, height, channels;
> @@ -42,7 +50,7 @@ typedef struct DNNModel{
>      void *model;
>      // Sets model input and output.
>      // Should be called at least once before model execution.
> -    DNNReturnType (*set_input_output)(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output);
> +    DNNReturnType (*set_input_output)(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output);
>  } DNNModel;
>
>  // Stores pointers to functions for loading, executing, freeing DNN models for one of the backends.
> diff --git a/libavfilter/vf_sr.c b/libavfilter/vf_sr.c
> index b4d4165..c0d7126 100644
> --- a/libavfilter/vf_sr.c
> +++ b/libavfilter/vf_sr.c
> @@ -40,7 +40,8 @@ typedef struct SRContext {
>      DNNBackendType backend_type;
>      DNNModule *dnn_module;
>      DNNModel *model;
> -    DNNData input, output;
> +    DNNInputData input;
> +    DNNData output;
>      int scale_factor;
>      struct SwsContext *sws_contexts[3];
>      int sws_slice_h, sws_input_linesize, sws_output_linesize;
> @@ -87,6 +88,7 @@ static av_cold int init(AVFilterContext *context)
>          return AVERROR(EIO);
>      }
>
> +    sr_context->input.dt = DNN_FLOAT;
>      sr_context->sws_contexts[0] = NULL;
>      sr_context->sws_contexts[1] = NULL;
>      sr_context->sws_contexts[2] = NULL;
> --
> 2.7.4
>

LGTM.

I think it would be valuable to add a few tests covering the features
added by this patch series.

> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
Guo, Yejun April 30, 2019, 1:21 a.m.
> -----Original Message-----

> From: Pedro Arthur [mailto:bygrandao@gmail.com]

> Sent: Tuesday, April 30, 2019 1:47 AM

> To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org>

> Cc: Guo, Yejun <yejun.guo@intel.com>

> Subject: Re: [FFmpeg-devel] [PATCH V2 7/7] libavfilter/dnn: add more data type

> support for dnn model input

> 

> Em qua, 24 de abr de 2019 às 23:15, Guo, Yejun <yejun.guo@intel.com>

> escreveu:

> >

> > currently, only float is supported as model input, actually, there

> > are other data types, this patch adds uint8.

> >

> > Signed-off-by: Guo, Yejun <yejun.guo@intel.com>

>

> 

> LGTM.

> 

> I think it would be valuable to add a few tests covering the features

> added by this patch series.


thanks, good point. Do you mean FATE? Is there any previous test for DNN module that I can refer to? thanks. I'll investigate it after my holiday, I'm starting vacation today.


> 

> > _______________________________________________

> > ffmpeg-devel mailing list

> > ffmpeg-devel@ffmpeg.org

> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

> >

> > To unsubscribe, visit link above, or email

> > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
Pedro Arthur April 30, 2019, 1:33 a.m.
Em seg, 29 de abr de 2019 às 22:21, Guo, Yejun <yejun.guo@intel.com> escreveu:
>
>
>
> > -----Original Message-----
> > From: Pedro Arthur [mailto:bygrandao@gmail.com]
> > Sent: Tuesday, April 30, 2019 1:47 AM
> > To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org>
> > Cc: Guo, Yejun <yejun.guo@intel.com>
> > Subject: Re: [FFmpeg-devel] [PATCH V2 7/7] libavfilter/dnn: add more data type
> > support for dnn model input
> >
> > Em qua, 24 de abr de 2019 às 23:15, Guo, Yejun <yejun.guo@intel.com>
> > escreveu:
> > >
> > > currently, only float is supported as model input, actually, there
> > > are other data types, this patch adds uint8.
> > >
> > > Signed-off-by: Guo, Yejun <yejun.guo@intel.com>
> >
> >
> > LGTM.
> >
> > I think it would be valuable to add a few tests covering the features
> > added by this patch series.
>
> thanks, good point. Do you mean FATE? Is there any previous test for DNN module that I can refer to? thanks. I'll investigate it after my holiday, I'm starting vacation today.

Yes, I mean FATE, unfortunately there isn't any tests for DNN atm.

Have a nice vacation!
Guo, Yejun May 5, 2019, 8:45 a.m.
> -----Original Message-----

> From: Pedro Arthur [mailto:bygrandao@gmail.com]

> Sent: Tuesday, April 30, 2019 9:33 AM

> To: Guo, Yejun <yejun.guo@intel.com>

> Cc: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org>

> Subject: Re: [FFmpeg-devel] [PATCH V2 7/7] libavfilter/dnn: add more data type

> support for dnn model input

> 

> Em seg, 29 de abr de 2019 às 22:21, Guo, Yejun <yejun.guo@intel.com>

> escreveu:

> >

> >

> >

> > > -----Original Message-----

> > > From: Pedro Arthur [mailto:bygrandao@gmail.com]

> > > Sent: Tuesday, April 30, 2019 1:47 AM

> > > To: FFmpeg development discussions and patches

> <ffmpeg-devel@ffmpeg.org>

> > > Cc: Guo, Yejun <yejun.guo@intel.com>

> > > Subject: Re: [FFmpeg-devel] [PATCH V2 7/7] libavfilter/dnn: add more data

> type

> > > support for dnn model input

> > >

> > > Em qua, 24 de abr de 2019 às 23:15, Guo, Yejun <yejun.guo@intel.com>

> > > escreveu:

> > > >

> > > > currently, only float is supported as model input, actually, there

> > > > are other data types, this patch adds uint8.

> > > >

> > > > Signed-off-by: Guo, Yejun <yejun.guo@intel.com>

> > >

> > >

> > > LGTM.

> > >

> > > I think it would be valuable to add a few tests covering the features

> > > added by this patch series.

> >

> > thanks, good point. Do you mean FATE? Is there any previous test for DNN

> module that I can refer to? thanks. I'll investigate it after my holiday, I'm

> starting vacation today.

> 

> Yes, I mean FATE, unfortunately there isn't any tests for DNN atm.


I plan to first add unit tests in FATE for DNN module, since the DNN interfaces is not exported as ffmpeg APIs.
I'll add tests/fate/dnn.mak, and also a new folder at tests/dnn. There will be .c files under tests/dnn, and
each .c file contains main() function to verify one feature of dnn module. It will be a self-contained test, no
source samples nor reference data is required.

In the future, we can add tests/fate/dnn-filters.mak to verify deep learning based filters with source samples and reference data.

> 

> Have a nice vacation!
Carl Eugen Hoyos May 5, 2019, 10:51 a.m.
Am So., 5. Mai 2019 um 10:45 Uhr schrieb Guo, Yejun <yejun.guo@intel.com>:

> I plan to first add unit tests in FATE for DNN module

Does this module work without an external dependency?

If not, you cannot add a fate test.

Carl Eugen
Guo, Yejun May 5, 2019, 12:10 p.m.
> -----Original Message-----

> From: ffmpeg-devel [mailto:ffmpeg-devel-bounces@ffmpeg.org] On Behalf Of

> Carl Eugen Hoyos

> Sent: Sunday, May 05, 2019 6:51 PM

> To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org>

> Subject: Re: [FFmpeg-devel] [PATCH V2 7/7] libavfilter/dnn: add more data type

> support for dnn model input

> 

> Am So., 5. Mai 2019 um 10:45 Uhr schrieb Guo, Yejun <yejun.guo@intel.com>:

> 

> > I plan to first add unit tests in FATE for DNN module

> 

> Does this module work without an external dependency?

> 

> If not, you cannot add a fate test.


The DNN module contains two parts, the native part with no external dependency,
and the other part depending on tensorflow c lib.

Thanks for your info, how about to limit the fate test only on the native part of DNN module? thanks.

> 

> Carl Eugen

> _______________________________________________

> ffmpeg-devel mailing list

> ffmpeg-devel@ffmpeg.org

> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

> 

> To unsubscribe, visit link above, or email

> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
Carl Eugen Hoyos May 5, 2019, 12:46 p.m.
Am So., 5. Mai 2019 um 14:10 Uhr schrieb Guo, Yejun <yejun.guo@intel.com>:

> > -----Original Message-----
> > From: ffmpeg-devel [mailto:ffmpeg-devel-bounces@ffmpeg.org] On Behalf Of
> > Carl Eugen Hoyos
> > Sent: Sunday, May 05, 2019 6:51 PM
> > To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org>
> > Subject: Re: [FFmpeg-devel] [PATCH V2 7/7] libavfilter/dnn: add more data type
> > support for dnn model input

Please consider telling your mail client that this is more information
than necessary.

> > Am So., 5. Mai 2019 um 10:45 Uhr schrieb Guo, Yejun <yejun.guo@intel.com>:
> >
> > > I plan to first add unit tests in FATE for DNN module
> >
> > Does this module work without an external dependency?
> >
> > If not, you cannot add a fate test.
>
> The DNN module contains two parts, the native part with no external dependency,
> and the other part depending on tensorflow c lib.
>
> Thanks for your info, how about to limit the fate test only on the native part of
> DNN module?

If it is possible to test DNN without tensorflow, there is no problem
(I wasn't sure).

Carl Eugen
Guo, Yejun May 8, 2019, 8:28 a.m.
> -----Original Message-----

> From: Pedro Arthur [mailto:bygrandao@gmail.com]

> Sent: Tuesday, April 30, 2019 1:47 AM

> To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org>

> Cc: Guo, Yejun <yejun.guo@intel.com>

> Subject: Re: [FFmpeg-devel] [PATCH V2 7/7] libavfilter/dnn: add more data type

> support for dnn model input

> > +    sr_context->input.dt = DNN_FLOAT;

> >      sr_context->sws_contexts[0] = NULL;

> >      sr_context->sws_contexts[1] = NULL;

> >      sr_context->sws_contexts[2] = NULL;

> > --

> > 2.7.4

> >

> 

> LGTM.

> 

> I think it would be valuable to add a few tests covering the features

> added by this patch series.


I tried a bit to add FATE for dnn module, see basic code in attached file.
We can only test native mode because FATE does not allow external dependency.

The native mode is still in early stage, and I plan to add support to
import TF model as native model with ffmpeg c code (as discussed in another thread).
That's might be a better time to add FATE after the import is finished. 
We can add unit tests for all the native ops at that time.

As for this patch series, it mainly focus on the TF mode, it might not be
suitable to add FATE for it.

So, how about to push this patch set, and add FATE when the native mode is a little more mature? thanks.


> 

> > _______________________________________________

> > ffmpeg-devel mailing list

> > ffmpeg-devel@ffmpeg.org

> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

> >

> > To unsubscribe, visit link above, or email

> > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
Pedro Arthur May 8, 2019, 3:40 p.m.
Em qua, 8 de mai de 2019 às 05:28, Guo, Yejun <yejun.guo@intel.com> escreveu:
>
>
>
> > -----Original Message-----
> > From: Pedro Arthur [mailto:bygrandao@gmail.com]
> > Sent: Tuesday, April 30, 2019 1:47 AM
> > To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org>
> > Cc: Guo, Yejun <yejun.guo@intel.com>
> > Subject: Re: [FFmpeg-devel] [PATCH V2 7/7] libavfilter/dnn: add more data type
> > support for dnn model input
> > > +    sr_context->input.dt = DNN_FLOAT;
> > >      sr_context->sws_contexts[0] = NULL;
> > >      sr_context->sws_contexts[1] = NULL;
> > >      sr_context->sws_contexts[2] = NULL;
> > > --
> > > 2.7.4
> > >
> >
> > LGTM.
> >
> > I think it would be valuable to add a few tests covering the features
> > added by this patch series.
>
> I tried a bit to add FATE for dnn module, see basic code in attached file.
> We can only test native mode because FATE does not allow external dependency.
>
> The native mode is still in early stage, and I plan to add support to
> import TF model as native model with ffmpeg c code (as discussed in another thread).
> That's might be a better time to add FATE after the import is finished.
> We can add unit tests for all the native ops at that time.
>
> As for this patch series, it mainly focus on the TF mode, it might not be
> suitable to add FATE for it.
>
> So, how about to push this patch set, and add FATE when the native mode is a little more mature? thanks.
Patch set pushed, sorry for the delay.

Later I'll properly review the unit test patch.

Patch hide | download patch | download mbox

diff --git a/libavfilter/dnn_backend_native.c b/libavfilter/dnn_backend_native.c
index 8a83c63..06fbdf3 100644
--- a/libavfilter/dnn_backend_native.c
+++ b/libavfilter/dnn_backend_native.c
@@ -24,8 +24,9 @@ 
  */
 
 #include "dnn_backend_native.h"
+#include "libavutil/avassert.h"
 
-static DNNReturnType set_input_output_native(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output)
+static DNNReturnType set_input_output_native(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output)
 {
     ConvolutionalNetwork *network = (ConvolutionalNetwork *)model;
     InputParams *input_params;
@@ -45,6 +46,7 @@  static DNNReturnType set_input_output_native(void *model, DNNData *input, const
         if (input->data){
             av_freep(&input->data);
         }
+        av_assert0(input->dt == DNN_FLOAT);
         network->layers[0].output = input->data = av_malloc(cur_height * cur_width * cur_channels * sizeof(float));
         if (!network->layers[0].output){
             return DNN_ERROR;
diff --git a/libavfilter/dnn_backend_tf.c b/libavfilter/dnn_backend_tf.c
index ca6472d..ba959ae 100644
--- a/libavfilter/dnn_backend_tf.c
+++ b/libavfilter/dnn_backend_tf.c
@@ -79,10 +79,31 @@  static TF_Buffer *read_graph(const char *model_filename)
     return graph_buf;
 }
 
-static DNNReturnType set_input_output_tf(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output)
+static TF_Tensor *allocate_input_tensor(const DNNInputData *input)
 {
-    TFModel *tf_model = (TFModel *)model;
+    TF_DataType dt;
+    size_t size;
     int64_t input_dims[] = {1, input->height, input->width, input->channels};
+    switch (input->dt) {
+    case DNN_FLOAT:
+        dt = TF_FLOAT;
+        size = sizeof(float);
+        break;
+    case DNN_UINT8:
+        dt = TF_UINT8;
+        size = sizeof(char);
+        break;
+    default:
+        av_assert0(!"should not reach here");
+    }
+
+    return TF_AllocateTensor(dt, input_dims, 4,
+                             input_dims[1] * input_dims[2] * input_dims[3] * size);
+}
+
+static DNNReturnType set_input_output_tf(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output)
+{
+    TFModel *tf_model = (TFModel *)model;
     TF_SessionOptions *sess_opts;
     const TF_Operation *init_op = TF_GraphOperationByName(tf_model->graph, "init");
 
@@ -95,8 +116,7 @@  static DNNReturnType set_input_output_tf(void *model, DNNData *input, const char
     if (tf_model->input_tensor){
         TF_DeleteTensor(tf_model->input_tensor);
     }
-    tf_model->input_tensor = TF_AllocateTensor(TF_FLOAT, input_dims, 4,
-                                               input_dims[1] * input_dims[2] * input_dims[3] * sizeof(float));
+    tf_model->input_tensor = allocate_input_tensor(input);
     if (!tf_model->input_tensor){
         return DNN_ERROR;
     }
diff --git a/libavfilter/dnn_interface.h b/libavfilter/dnn_interface.h
index 73d226e..c24df0e 100644
--- a/libavfilter/dnn_interface.h
+++ b/libavfilter/dnn_interface.h
@@ -32,6 +32,14 @@  typedef enum {DNN_SUCCESS, DNN_ERROR} DNNReturnType;
 
 typedef enum {DNN_NATIVE, DNN_TF} DNNBackendType;
 
+typedef enum {DNN_FLOAT, DNN_UINT8} DNNDataType;
+
+typedef struct DNNInputData{
+    void *data;
+    DNNDataType dt;
+    int width, height, channels;
+} DNNInputData;
+
 typedef struct DNNData{
     float *data;
     int width, height, channels;
@@ -42,7 +50,7 @@  typedef struct DNNModel{
     void *model;
     // Sets model input and output.
     // Should be called at least once before model execution.
-    DNNReturnType (*set_input_output)(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output);
+    DNNReturnType (*set_input_output)(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output);
 } DNNModel;
 
 // Stores pointers to functions for loading, executing, freeing DNN models for one of the backends.
diff --git a/libavfilter/vf_sr.c b/libavfilter/vf_sr.c
index b4d4165..c0d7126 100644
--- a/libavfilter/vf_sr.c
+++ b/libavfilter/vf_sr.c
@@ -40,7 +40,8 @@  typedef struct SRContext {
     DNNBackendType backend_type;
     DNNModule *dnn_module;
     DNNModel *model;
-    DNNData input, output;
+    DNNInputData input;
+    DNNData output;
     int scale_factor;
     struct SwsContext *sws_contexts[3];
     int sws_slice_h, sws_input_linesize, sws_output_linesize;
@@ -87,6 +88,7 @@  static av_cold int init(AVFilterContext *context)
         return AVERROR(EIO);
     }
 
+    sr_context->input.dt = DNN_FLOAT;
     sr_context->sws_contexts[0] = NULL;
     sr_context->sws_contexts[1] = NULL;
     sr_context->sws_contexts[2] = NULL;