| Message ID | 1556158482-15306-1-git-send-email-yejun.guo@intel.com |
|---|---|
| State | New |
| Headers | show |
Em qua, 24 de abr de 2019 às 23:15, Guo, Yejun <yejun.guo@intel.com> escreveu: > > currently, only float is supported as model input, actually, there > are other data types, this patch adds uint8. > > Signed-off-by: Guo, Yejun <yejun.guo@intel.com> > --- > libavfilter/dnn_backend_native.c | 4 +++- > libavfilter/dnn_backend_tf.c | 28 ++++++++++++++++++++++++---- > libavfilter/dnn_interface.h | 10 +++++++++- > libavfilter/vf_sr.c | 4 +++- > 4 files changed, 39 insertions(+), 7 deletions(-) > > diff --git a/libavfilter/dnn_backend_native.c b/libavfilter/dnn_backend_native.c > index 8a83c63..06fbdf3 100644 > --- a/libavfilter/dnn_backend_native.c > +++ b/libavfilter/dnn_backend_native.c > @@ -24,8 +24,9 @@ > */ > > #include "dnn_backend_native.h" > +#include "libavutil/avassert.h" > > -static DNNReturnType set_input_output_native(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output) > +static DNNReturnType set_input_output_native(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output) > { > ConvolutionalNetwork *network = (ConvolutionalNetwork *)model; > InputParams *input_params; > @@ -45,6 +46,7 @@ static DNNReturnType set_input_output_native(void *model, DNNData *input, const > if (input->data){ > av_freep(&input->data); > } > + av_assert0(input->dt == DNN_FLOAT); > network->layers[0].output = input->data = av_malloc(cur_height * cur_width * cur_channels * sizeof(float)); > if (!network->layers[0].output){ > return DNN_ERROR; > diff --git a/libavfilter/dnn_backend_tf.c b/libavfilter/dnn_backend_tf.c > index ca6472d..ba959ae 100644 > --- a/libavfilter/dnn_backend_tf.c > +++ b/libavfilter/dnn_backend_tf.c > @@ -79,10 +79,31 @@ static TF_Buffer *read_graph(const char *model_filename) > return graph_buf; > } > > -static DNNReturnType set_input_output_tf(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output) > +static TF_Tensor *allocate_input_tensor(const DNNInputData *input) > { > - TFModel *tf_model = (TFModel *)model; > + TF_DataType dt; > + size_t size; > int64_t input_dims[] = {1, input->height, input->width, input->channels}; > + switch (input->dt) { > + case DNN_FLOAT: > + dt = TF_FLOAT; > + size = sizeof(float); > + break; > + case DNN_UINT8: > + dt = TF_UINT8; > + size = sizeof(char); > + break; > + default: > + av_assert0(!"should not reach here"); > + } > + > + return TF_AllocateTensor(dt, input_dims, 4, > + input_dims[1] * input_dims[2] * input_dims[3] * size); > +} > + > +static DNNReturnType set_input_output_tf(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output) > +{ > + TFModel *tf_model = (TFModel *)model; > TF_SessionOptions *sess_opts; > const TF_Operation *init_op = TF_GraphOperationByName(tf_model->graph, "init"); > > @@ -95,8 +116,7 @@ static DNNReturnType set_input_output_tf(void *model, DNNData *input, const char > if (tf_model->input_tensor){ > TF_DeleteTensor(tf_model->input_tensor); > } > - tf_model->input_tensor = TF_AllocateTensor(TF_FLOAT, input_dims, 4, > - input_dims[1] * input_dims[2] * input_dims[3] * sizeof(float)); > + tf_model->input_tensor = allocate_input_tensor(input); > if (!tf_model->input_tensor){ > return DNN_ERROR; > } > diff --git a/libavfilter/dnn_interface.h b/libavfilter/dnn_interface.h > index 73d226e..c24df0e 100644 > --- a/libavfilter/dnn_interface.h > +++ b/libavfilter/dnn_interface.h > @@ -32,6 +32,14 @@ typedef enum {DNN_SUCCESS, DNN_ERROR} DNNReturnType; > > typedef enum {DNN_NATIVE, DNN_TF} DNNBackendType; > > +typedef enum {DNN_FLOAT, DNN_UINT8} DNNDataType; > + > +typedef struct DNNInputData{ > + void *data; > + DNNDataType dt; > + int width, height, channels; > +} DNNInputData; > + > typedef struct DNNData{ > float *data; > int width, height, channels; > @@ -42,7 +50,7 @@ typedef struct DNNModel{ > void *model; > // Sets model input and output. > // Should be called at least once before model execution. > - DNNReturnType (*set_input_output)(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output); > + DNNReturnType (*set_input_output)(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output); > } DNNModel; > > // Stores pointers to functions for loading, executing, freeing DNN models for one of the backends. > diff --git a/libavfilter/vf_sr.c b/libavfilter/vf_sr.c > index b4d4165..c0d7126 100644 > --- a/libavfilter/vf_sr.c > +++ b/libavfilter/vf_sr.c > @@ -40,7 +40,8 @@ typedef struct SRContext { > DNNBackendType backend_type; > DNNModule *dnn_module; > DNNModel *model; > - DNNData input, output; > + DNNInputData input; > + DNNData output; > int scale_factor; > struct SwsContext *sws_contexts[3]; > int sws_slice_h, sws_input_linesize, sws_output_linesize; > @@ -87,6 +88,7 @@ static av_cold int init(AVFilterContext *context) > return AVERROR(EIO); > } > > + sr_context->input.dt = DNN_FLOAT; > sr_context->sws_contexts[0] = NULL; > sr_context->sws_contexts[1] = NULL; > sr_context->sws_contexts[2] = NULL; > -- > 2.7.4 > LGTM. I think it would be valuable to add a few tests covering the features added by this patch series. > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
> -----Original Message----- > From: Pedro Arthur [mailto:bygrandao@gmail.com] > Sent: Tuesday, April 30, 2019 1:47 AM > To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org> > Cc: Guo, Yejun <yejun.guo@intel.com> > Subject: Re: [FFmpeg-devel] [PATCH V2 7/7] libavfilter/dnn: add more data type > support for dnn model input > > Em qua, 24 de abr de 2019 às 23:15, Guo, Yejun <yejun.guo@intel.com> > escreveu: > > > > currently, only float is supported as model input, actually, there > > are other data types, this patch adds uint8. > > > > Signed-off-by: Guo, Yejun <yejun.guo@intel.com> > > > LGTM. > > I think it would be valuable to add a few tests covering the features > added by this patch series. thanks, good point. Do you mean FATE? Is there any previous test for DNN module that I can refer to? thanks. I'll investigate it after my holiday, I'm starting vacation today. > > > _______________________________________________ > > ffmpeg-devel mailing list > > ffmpeg-devel@ffmpeg.org > > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > > > To unsubscribe, visit link above, or email > > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
Em seg, 29 de abr de 2019 às 22:21, Guo, Yejun <yejun.guo@intel.com> escreveu: > > > > > -----Original Message----- > > From: Pedro Arthur [mailto:bygrandao@gmail.com] > > Sent: Tuesday, April 30, 2019 1:47 AM > > To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org> > > Cc: Guo, Yejun <yejun.guo@intel.com> > > Subject: Re: [FFmpeg-devel] [PATCH V2 7/7] libavfilter/dnn: add more data type > > support for dnn model input > > > > Em qua, 24 de abr de 2019 às 23:15, Guo, Yejun <yejun.guo@intel.com> > > escreveu: > > > > > > currently, only float is supported as model input, actually, there > > > are other data types, this patch adds uint8. > > > > > > Signed-off-by: Guo, Yejun <yejun.guo@intel.com> > > > > > > LGTM. > > > > I think it would be valuable to add a few tests covering the features > > added by this patch series. > > thanks, good point. Do you mean FATE? Is there any previous test for DNN module that I can refer to? thanks. I'll investigate it after my holiday, I'm starting vacation today. Yes, I mean FATE, unfortunately there isn't any tests for DNN atm. Have a nice vacation!
> -----Original Message----- > From: Pedro Arthur [mailto:bygrandao@gmail.com] > Sent: Tuesday, April 30, 2019 9:33 AM > To: Guo, Yejun <yejun.guo@intel.com> > Cc: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org> > Subject: Re: [FFmpeg-devel] [PATCH V2 7/7] libavfilter/dnn: add more data type > support for dnn model input > > Em seg, 29 de abr de 2019 às 22:21, Guo, Yejun <yejun.guo@intel.com> > escreveu: > > > > > > > > > -----Original Message----- > > > From: Pedro Arthur [mailto:bygrandao@gmail.com] > > > Sent: Tuesday, April 30, 2019 1:47 AM > > > To: FFmpeg development discussions and patches > <ffmpeg-devel@ffmpeg.org> > > > Cc: Guo, Yejun <yejun.guo@intel.com> > > > Subject: Re: [FFmpeg-devel] [PATCH V2 7/7] libavfilter/dnn: add more data > type > > > support for dnn model input > > > > > > Em qua, 24 de abr de 2019 às 23:15, Guo, Yejun <yejun.guo@intel.com> > > > escreveu: > > > > > > > > currently, only float is supported as model input, actually, there > > > > are other data types, this patch adds uint8. > > > > > > > > Signed-off-by: Guo, Yejun <yejun.guo@intel.com> > > > > > > > > > LGTM. > > > > > > I think it would be valuable to add a few tests covering the features > > > added by this patch series. > > > > thanks, good point. Do you mean FATE? Is there any previous test for DNN > module that I can refer to? thanks. I'll investigate it after my holiday, I'm > starting vacation today. > > Yes, I mean FATE, unfortunately there isn't any tests for DNN atm. I plan to first add unit tests in FATE for DNN module, since the DNN interfaces is not exported as ffmpeg APIs. I'll add tests/fate/dnn.mak, and also a new folder at tests/dnn. There will be .c files under tests/dnn, and each .c file contains main() function to verify one feature of dnn module. It will be a self-contained test, no source samples nor reference data is required. In the future, we can add tests/fate/dnn-filters.mak to verify deep learning based filters with source samples and reference data. > > Have a nice vacation!
Am So., 5. Mai 2019 um 10:45 Uhr schrieb Guo, Yejun <yejun.guo@intel.com>:
> I plan to first add unit tests in FATE for DNN module
Does this module work without an external dependency?
If not, you cannot add a fate test.
Carl Eugen
> -----Original Message----- > From: ffmpeg-devel [mailto:ffmpeg-devel-bounces@ffmpeg.org] On Behalf Of > Carl Eugen Hoyos > Sent: Sunday, May 05, 2019 6:51 PM > To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org> > Subject: Re: [FFmpeg-devel] [PATCH V2 7/7] libavfilter/dnn: add more data type > support for dnn model input > > Am So., 5. Mai 2019 um 10:45 Uhr schrieb Guo, Yejun <yejun.guo@intel.com>: > > > I plan to first add unit tests in FATE for DNN module > > Does this module work without an external dependency? > > If not, you cannot add a fate test. The DNN module contains two parts, the native part with no external dependency, and the other part depending on tensorflow c lib. Thanks for your info, how about to limit the fate test only on the native part of DNN module? thanks. > > Carl Eugen > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
Am So., 5. Mai 2019 um 14:10 Uhr schrieb Guo, Yejun <yejun.guo@intel.com>: > > -----Original Message----- > > From: ffmpeg-devel [mailto:ffmpeg-devel-bounces@ffmpeg.org] On Behalf Of > > Carl Eugen Hoyos > > Sent: Sunday, May 05, 2019 6:51 PM > > To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org> > > Subject: Re: [FFmpeg-devel] [PATCH V2 7/7] libavfilter/dnn: add more data type > > support for dnn model input Please consider telling your mail client that this is more information than necessary. > > Am So., 5. Mai 2019 um 10:45 Uhr schrieb Guo, Yejun <yejun.guo@intel.com>: > > > > > I plan to first add unit tests in FATE for DNN module > > > > Does this module work without an external dependency? > > > > If not, you cannot add a fate test. > > The DNN module contains two parts, the native part with no external dependency, > and the other part depending on tensorflow c lib. > > Thanks for your info, how about to limit the fate test only on the native part of > DNN module? If it is possible to test DNN without tensorflow, there is no problem (I wasn't sure). Carl Eugen
> -----Original Message----- > From: Pedro Arthur [mailto:bygrandao@gmail.com] > Sent: Tuesday, April 30, 2019 1:47 AM > To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org> > Cc: Guo, Yejun <yejun.guo@intel.com> > Subject: Re: [FFmpeg-devel] [PATCH V2 7/7] libavfilter/dnn: add more data type > support for dnn model input > > + sr_context->input.dt = DNN_FLOAT; > > sr_context->sws_contexts[0] = NULL; > > sr_context->sws_contexts[1] = NULL; > > sr_context->sws_contexts[2] = NULL; > > -- > > 2.7.4 > > > > LGTM. > > I think it would be valuable to add a few tests covering the features > added by this patch series. I tried a bit to add FATE for dnn module, see basic code in attached file. We can only test native mode because FATE does not allow external dependency. The native mode is still in early stage, and I plan to add support to import TF model as native model with ffmpeg c code (as discussed in another thread). That's might be a better time to add FATE after the import is finished. We can add unit tests for all the native ops at that time. As for this patch series, it mainly focus on the TF mode, it might not be suitable to add FATE for it. So, how about to push this patch set, and add FATE when the native mode is a little more mature? thanks. > > > _______________________________________________ > > ffmpeg-devel mailing list > > ffmpeg-devel@ffmpeg.org > > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > > > To unsubscribe, visit link above, or email > > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
Em qua, 8 de mai de 2019 às 05:28, Guo, Yejun <yejun.guo@intel.com> escreveu: > > > > > -----Original Message----- > > From: Pedro Arthur [mailto:bygrandao@gmail.com] > > Sent: Tuesday, April 30, 2019 1:47 AM > > To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org> > > Cc: Guo, Yejun <yejun.guo@intel.com> > > Subject: Re: [FFmpeg-devel] [PATCH V2 7/7] libavfilter/dnn: add more data type > > support for dnn model input > > > + sr_context->input.dt = DNN_FLOAT; > > > sr_context->sws_contexts[0] = NULL; > > > sr_context->sws_contexts[1] = NULL; > > > sr_context->sws_contexts[2] = NULL; > > > -- > > > 2.7.4 > > > > > > > LGTM. > > > > I think it would be valuable to add a few tests covering the features > > added by this patch series. > > I tried a bit to add FATE for dnn module, see basic code in attached file. > We can only test native mode because FATE does not allow external dependency. > > The native mode is still in early stage, and I plan to add support to > import TF model as native model with ffmpeg c code (as discussed in another thread). > That's might be a better time to add FATE after the import is finished. > We can add unit tests for all the native ops at that time. > > As for this patch series, it mainly focus on the TF mode, it might not be > suitable to add FATE for it. > > So, how about to push this patch set, and add FATE when the native mode is a little more mature? thanks. Patch set pushed, sorry for the delay. Later I'll properly review the unit test patch.
diff --git a/libavfilter/dnn_backend_native.c b/libavfilter/dnn_backend_native.c index 8a83c63..06fbdf3 100644 --- a/libavfilter/dnn_backend_native.c +++ b/libavfilter/dnn_backend_native.c @@ -24,8 +24,9 @@ */ #include "dnn_backend_native.h" +#include "libavutil/avassert.h" -static DNNReturnType set_input_output_native(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output) +static DNNReturnType set_input_output_native(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output) { ConvolutionalNetwork *network = (ConvolutionalNetwork *)model; InputParams *input_params; @@ -45,6 +46,7 @@ static DNNReturnType set_input_output_native(void *model, DNNData *input, const if (input->data){ av_freep(&input->data); } + av_assert0(input->dt == DNN_FLOAT); network->layers[0].output = input->data = av_malloc(cur_height * cur_width * cur_channels * sizeof(float)); if (!network->layers[0].output){ return DNN_ERROR; diff --git a/libavfilter/dnn_backend_tf.c b/libavfilter/dnn_backend_tf.c index ca6472d..ba959ae 100644 --- a/libavfilter/dnn_backend_tf.c +++ b/libavfilter/dnn_backend_tf.c @@ -79,10 +79,31 @@ static TF_Buffer *read_graph(const char *model_filename) return graph_buf; } -static DNNReturnType set_input_output_tf(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output) +static TF_Tensor *allocate_input_tensor(const DNNInputData *input) { - TFModel *tf_model = (TFModel *)model; + TF_DataType dt; + size_t size; int64_t input_dims[] = {1, input->height, input->width, input->channels}; + switch (input->dt) { + case DNN_FLOAT: + dt = TF_FLOAT; + size = sizeof(float); + break; + case DNN_UINT8: + dt = TF_UINT8; + size = sizeof(char); + break; + default: + av_assert0(!"should not reach here"); + } + + return TF_AllocateTensor(dt, input_dims, 4, + input_dims[1] * input_dims[2] * input_dims[3] * size); +} + +static DNNReturnType set_input_output_tf(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output) +{ + TFModel *tf_model = (TFModel *)model; TF_SessionOptions *sess_opts; const TF_Operation *init_op = TF_GraphOperationByName(tf_model->graph, "init"); @@ -95,8 +116,7 @@ static DNNReturnType set_input_output_tf(void *model, DNNData *input, const char if (tf_model->input_tensor){ TF_DeleteTensor(tf_model->input_tensor); } - tf_model->input_tensor = TF_AllocateTensor(TF_FLOAT, input_dims, 4, - input_dims[1] * input_dims[2] * input_dims[3] * sizeof(float)); + tf_model->input_tensor = allocate_input_tensor(input); if (!tf_model->input_tensor){ return DNN_ERROR; } diff --git a/libavfilter/dnn_interface.h b/libavfilter/dnn_interface.h index 73d226e..c24df0e 100644 --- a/libavfilter/dnn_interface.h +++ b/libavfilter/dnn_interface.h @@ -32,6 +32,14 @@ typedef enum {DNN_SUCCESS, DNN_ERROR} DNNReturnType; typedef enum {DNN_NATIVE, DNN_TF} DNNBackendType; +typedef enum {DNN_FLOAT, DNN_UINT8} DNNDataType; + +typedef struct DNNInputData{ + void *data; + DNNDataType dt; + int width, height, channels; +} DNNInputData; + typedef struct DNNData{ float *data; int width, height, channels; @@ -42,7 +50,7 @@ typedef struct DNNModel{ void *model; // Sets model input and output. // Should be called at least once before model execution. - DNNReturnType (*set_input_output)(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output); + DNNReturnType (*set_input_output)(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output); } DNNModel; // Stores pointers to functions for loading, executing, freeing DNN models for one of the backends. diff --git a/libavfilter/vf_sr.c b/libavfilter/vf_sr.c index b4d4165..c0d7126 100644 --- a/libavfilter/vf_sr.c +++ b/libavfilter/vf_sr.c @@ -40,7 +40,8 @@ typedef struct SRContext { DNNBackendType backend_type; DNNModule *dnn_module; DNNModel *model; - DNNData input, output; + DNNInputData input; + DNNData output; int scale_factor; struct SwsContext *sws_contexts[3]; int sws_slice_h, sws_input_linesize, sws_output_linesize; @@ -87,6 +88,7 @@ static av_cold int init(AVFilterContext *context) return AVERROR(EIO); } + sr_context->input.dt = DNN_FLOAT; sr_context->sws_contexts[0] = NULL; sr_context->sws_contexts[1] = NULL; sr_context->sws_contexts[2] = NULL;
currently, only float is supported as model input, actually, there are other data types, this patch adds uint8. Signed-off-by: Guo, Yejun <yejun.guo@intel.com> --- libavfilter/dnn_backend_native.c | 4 +++- libavfilter/dnn_backend_tf.c | 28 ++++++++++++++++++++++++---- libavfilter/dnn_interface.h | 10 +++++++++- libavfilter/vf_sr.c | 4 +++- 4 files changed, 39 insertions(+), 7 deletions(-)