[FFmpeg-devel,v2] Add multiple padding method in dnn native

Submitted by Xuewei Meng on May 11, 2019, 3:11 a.m.

Details

Message ID 20190511031118.584-1-xwmeng96@gmail.com
State New
Headers show

Commit Message

Xuewei Meng May 11, 2019, 3:11 a.m.
---
 libavfilter/dnn_backend_native.c | 52 ++++++++++++++++++++++++--------
 libavfilter/dnn_backend_native.h |  3 ++
 2 files changed, 43 insertions(+), 12 deletions(-)

Comments

Steven Liu May 15, 2019, 2:37 a.m.
Xuewei Meng <xwmeng96@gmail.com> 于2019年5月11日周六 上午11:11写道:
>
> ---
>  libavfilter/dnn_backend_native.c | 52 ++++++++++++++++++++++++--------
>  libavfilter/dnn_backend_native.h |  3 ++
>  2 files changed, 43 insertions(+), 12 deletions(-)
>
> diff --git a/libavfilter/dnn_backend_native.c b/libavfilter/dnn_backend_native.c
> index 06fbdf368b..171a756385 100644
> --- a/libavfilter/dnn_backend_native.c
> +++ b/libavfilter/dnn_backend_native.c
> @@ -61,6 +61,12 @@ static DNNReturnType set_input_output_native(void *model, DNNInputData *input, c
>                  return DNN_ERROR;
>              }
>              cur_channels = conv_params->output_num;
> +
> +            if(conv_params->padding_method == VALID){
> +                int pad_size = conv_params->kernel_size - 1;
> +                cur_height -= pad_size;
> +                cur_width -= pad_size;
> +            }
>              break;
>          case DEPTH_TO_SPACE:
>              depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params;
> @@ -77,6 +83,10 @@ static DNNReturnType set_input_output_native(void *model, DNNInputData *input, c
>          if (network->layers[layer].output){
>              av_freep(&network->layers[layer].output);
>          }
> +
> +        if(cur_height <= 0 || cur_width <= 0)
> +            return DNN_ERROR;
> +
>          network->layers[layer].output = av_malloc(cur_height * cur_width * cur_channels * sizeof(float));
>          if (!network->layers[layer].output){
>              return DNN_ERROR;
> @@ -154,13 +164,14 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
>                  ff_dnn_free_model_native(&model);
>                  return NULL;
>              }
> +            conv_params->padding_method = (int32_t)avio_rl32(model_file_context);
>              conv_params->activation = (int32_t)avio_rl32(model_file_context);
>              conv_params->input_num = (int32_t)avio_rl32(model_file_context);
>              conv_params->output_num = (int32_t)avio_rl32(model_file_context);
>              conv_params->kernel_size = (int32_t)avio_rl32(model_file_context);
>              kernel_size = conv_params->input_num * conv_params->output_num *
>                            conv_params->kernel_size * conv_params->kernel_size;
> -            dnn_size += 16 + (kernel_size + conv_params->output_num << 2);
> +            dnn_size += 20 + (kernel_size + conv_params->output_num << 2);
>              if (dnn_size > file_size || conv_params->input_num <= 0 ||
>                  conv_params->output_num <= 0 || conv_params->kernel_size <= 0){
>                  avio_closep(&model_file_context);
> @@ -218,23 +229,35 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
>
>  static void convolve(const float *input, float *output, const ConvolutionalParams *conv_params, int width, int height)
>  {
> -    int y, x, n_filter, ch, kernel_y, kernel_x;
>      int radius = conv_params->kernel_size >> 1;
>      int src_linesize = width * conv_params->input_num;
>      int filter_linesize = conv_params->kernel_size * conv_params->input_num;
>      int filter_size = conv_params->kernel_size * filter_linesize;
> +    int pad_size = (conv_params->padding_method == VALID) ? (conv_params->kernel_size - 1) / 2 : 0;
>
> -    for (y = 0; y < height; ++y){
> -        for (x = 0; x < width; ++x){
> -            for (n_filter = 0; n_filter < conv_params->output_num; ++n_filter){
> +    for (int y = pad_size; y < height - pad_size; ++y){
> +        for (int x = pad_size; x < width - pad_size; ++x){
> +            for (int n_filter = 0; n_filter < conv_params->output_num; ++n_filter){
>                  output[n_filter] = conv_params->biases[n_filter];
> -                for (ch = 0; ch < conv_params->input_num; ++ch){
> -                    for (kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y){
> -                        for (kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x){
> -                            output[n_filter] += input[CLAMP_TO_EDGE(y + kernel_y - radius, height) * src_linesize +
> -                                                      CLAMP_TO_EDGE(x + kernel_x - radius, width) * conv_params->input_num + ch] *
> -                                                conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize +
> -                                                                    kernel_x * conv_params->input_num + ch];
> +
> +                for (int ch = 0; ch < conv_params->input_num; ++ch){
> +                    for (int kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y){
> +                        for (int kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x){
> +                            float input_pel;
> +                            if(conv_params->padding_method == SAME_CLAMP_TO_EDGE){
> +                                int y_pos = CLAMP_TO_EDGE(y + kernel_y - radius, height);
> +                                int x_pos = CLAMP_TO_EDGE(x + kernel_x - radius, width);
> +                                input_pel = input[y_pos * src_linesize + x_pos * conv_params->input_num + ch];
> +                            }else{
> +                                int y_pos = y + kernel_y - radius;
> +                                int x_pos = x + kernel_x - radius;
> +                                input_pel = (x_pos < 0 || x_pos >= width || y_pos < 0 || y_pos >= height) ? 0.0 :
> +                                                   input[y_pos * src_linesize + x_pos * conv_params->input_num + ch];
> +                            }
> +
> +
> +                            output[n_filter] += input_pel * conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize +
> +                                                                                kernel_x * conv_params->input_num + ch];
>                          }
>                      }
>                  }
> @@ -305,6 +328,11 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *output
>              conv_params = (ConvolutionalParams *)network->layers[layer].params;
>              convolve(network->layers[layer - 1].output, network->layers[layer].output, conv_params, cur_width, cur_height);
>              cur_channels = conv_params->output_num;
> +            if(conv_params->padding_method == VALID){
> +                int pad_size = conv_params->kernel_size - 1;
> +                cur_height -= pad_size;
> +                cur_width -= pad_size;
> +            }
>              break;
>          case DEPTH_TO_SPACE:
>              depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params;
> diff --git a/libavfilter/dnn_backend_native.h b/libavfilter/dnn_backend_native.h
> index e13a68a168..d70cd16387 100644
> --- a/libavfilter/dnn_backend_native.h
> +++ b/libavfilter/dnn_backend_native.h
> @@ -34,6 +34,8 @@ typedef enum {INPUT, CONV, DEPTH_TO_SPACE} DNNLayerType;
>
>  typedef enum {RELU, TANH, SIGMOID} DNNActivationFunc;
>
> +typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} DNNConvPaddingParam;
> +
>  typedef struct Layer{
>      DNNLayerType type;
>      float *output;
> @@ -43,6 +45,7 @@ typedef struct Layer{
>  typedef struct ConvolutionalParams{
>      int32_t input_num, output_num, kernel_size;
>      DNNActivationFunc activation;
> +    DNNConvPaddingParam padding_method;
>      float *kernel;
>      float *biases;
>  } ConvolutionalParams;
> --
> 2.17.1
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".


The https://github.com/HighVoltageRocknRoll/sr has  loss of
communication,and the project
https://github.com/HighVoltageRocknRoll/sr has no maintainer now, so i
think the pull request cannot be merge.
1. So i recommend Xuewei fork the project to his github, and merge the
pr to his fork project, and modify the sr document of
libavfilter/vf_sr.c. makes GSoC derain mentor project continue.

2. If 1st way cannot be acceptable, Xuewei should duplicate DNN code
for the derain.

Comments welcome.

Thanks

Steven
Guo, Yejun May 15, 2019, 6:21 a.m.
> -----Original Message-----

> From: ffmpeg-devel [mailto:ffmpeg-devel-bounces@ffmpeg.org] On Behalf Of

> Steven Liu

> Sent: Wednesday, May 15, 2019 10:38 AM

> To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org>

> Cc: Xuewei Meng <xwmeng96@gmail.com>

> Subject: Re: [FFmpeg-devel] [PATCH v2] Add multiple padding method in dnn

> native

> 

> Xuewei Meng <xwmeng96@gmail.com> 于2019年5月11日周六 上午11:11

> 写道:

> >

> > ---

> >  libavfilter/dnn_backend_native.c | 52 ++++++++++++++++++++++++--------

> >  libavfilter/dnn_backend_native.h |  3 ++

> >  2 files changed, 43 insertions(+), 12 deletions(-)


@xuewei, we still need to mention the impact of sr filter, and explain why same_clamp_to_edge is needed.

> >

> > diff --git a/libavfilter/dnn_backend_native.c

> b/libavfilter/dnn_backend_native.c

> > index 06fbdf368b..171a756385 100644

> > --- a/libavfilter/dnn_backend_native.c

> > +++ b/libavfilter/dnn_backend_native.c

> > @@ -61,6 +61,12 @@ static DNNReturnType set_input_output_native(void

> *model, DNNInputData *input, c

> >                  return DNN_ERROR;

> >              }

> >              cur_channels = conv_params->output_num;

> > +

> > +            if(conv_params->padding_method == VALID){

> > +                int pad_size = conv_params->kernel_size - 1;

> > +                cur_height -= pad_size;

> > +                cur_width -= pad_size;

> > +            }

> >              break;

> >          case DEPTH_TO_SPACE:

> >              depth_to_space_params = (DepthToSpaceParams

> *)network->layers[layer].params;

> > @@ -77,6 +83,10 @@ static DNNReturnType set_input_output_native(void

> *model, DNNInputData *input, c

> >          if (network->layers[layer].output){

> >              av_freep(&network->layers[layer].output);

> >          }

> > +

> > +        if(cur_height <= 0 || cur_width <= 0)

> > +            return DNN_ERROR;

> > +

> >          network->layers[layer].output = av_malloc(cur_height *

> cur_width * cur_channels * sizeof(float));

> >          if (!network->layers[layer].output){

> >              return DNN_ERROR;

> > @@ -154,13 +164,14 @@ DNNModel *ff_dnn_load_model_native(const

> char *model_filename)

> >                  ff_dnn_free_model_native(&model);

> >                  return NULL;

> >              }

> > +            conv_params->padding_method =

> (int32_t)avio_rl32(model_file_context);

> >              conv_params->activation =

> (int32_t)avio_rl32(model_file_context);

> >              conv_params->input_num =

> (int32_t)avio_rl32(model_file_context);

> >              conv_params->output_num =

> (int32_t)avio_rl32(model_file_context);

> >              conv_params->kernel_size =

> (int32_t)avio_rl32(model_file_context);

> >              kernel_size = conv_params->input_num *

> conv_params->output_num *

> >                            conv_params->kernel_size *

> conv_params->kernel_size;

> > -            dnn_size += 16 + (kernel_size + conv_params->output_num

> << 2);

> > +            dnn_size += 20 + (kernel_size + conv_params->output_num

> << 2);

> >              if (dnn_size > file_size || conv_params->input_num <= 0 ||

> >                  conv_params->output_num <= 0 ||

> conv_params->kernel_size <= 0){

> >                  avio_closep(&model_file_context);

> > @@ -218,23 +229,35 @@ DNNModel *ff_dnn_load_model_native(const

> char *model_filename)

> >

> >  static void convolve(const float *input, float *output, const

> ConvolutionalParams *conv_params, int width, int height)

> >  {

> > -    int y, x, n_filter, ch, kernel_y, kernel_x;

> >      int radius = conv_params->kernel_size >> 1;

> >      int src_linesize = width * conv_params->input_num;

> >      int filter_linesize = conv_params->kernel_size *

> conv_params->input_num;

> >      int filter_size = conv_params->kernel_size * filter_linesize;

> > +    int pad_size = (conv_params->padding_method == VALID) ?

> (conv_params->kernel_size - 1) / 2 : 0;

> >

> > -    for (y = 0; y < height; ++y){

> > -        for (x = 0; x < width; ++x){

> > -            for (n_filter = 0; n_filter < conv_params->output_num;

> ++n_filter){

> > +    for (int y = pad_size; y < height - pad_size; ++y){

> > +        for (int x = pad_size; x < width - pad_size; ++x){

> > +            for (int n_filter = 0; n_filter < conv_params->output_num;

> ++n_filter){

> >                  output[n_filter] = conv_params->biases[n_filter];

> > -                for (ch = 0; ch < conv_params->input_num; ++ch){

> > -                    for (kernel_y = 0; kernel_y <

> conv_params->kernel_size; ++kernel_y){

> > -                        for (kernel_x = 0; kernel_x <

> conv_params->kernel_size; ++kernel_x){

> > -                            output[n_filter] +=

> input[CLAMP_TO_EDGE(y + kernel_y - radius, height) * src_linesize +

> > -

> CLAMP_TO_EDGE(x + kernel_x - radius, width) * conv_params->input_num + ch]

> *

> > -

> conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize +

> > -

> kernel_x * conv_params->input_num + ch];

> > +

> > +                for (int ch = 0; ch < conv_params->input_num; ++ch){

> > +                    for (int kernel_y = 0; kernel_y <

> conv_params->kernel_size; ++kernel_y){

> > +                        for (int kernel_x = 0; kernel_x <

> conv_params->kernel_size; ++kernel_x){

> > +                            float input_pel;

> > +                            if(conv_params->padding_method ==

> SAME_CLAMP_TO_EDGE){

> > +                                int y_pos = CLAMP_TO_EDGE(y +

> kernel_y - radius, height);

> > +                                int x_pos = CLAMP_TO_EDGE(x +

> kernel_x - radius, width);

> > +                                input_pel = input[y_pos *

> src_linesize + x_pos * conv_params->input_num + ch];

> > +                            }else{

> > +                                int y_pos = y + kernel_y - radius;

> > +                                int x_pos = x + kernel_x - radius;

> > +                                input_pel = (x_pos < 0 || x_pos >=

> width || y_pos < 0 || y_pos >= height) ? 0.0 :

> > +                                                   input[y_pos

> * src_linesize + x_pos * conv_params->input_num + ch];

> > +                            }

> > +

> > +

> > +                            output[n_filter] += input_pel *

> conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize +

> > +

> kernel_x * conv_params->input_num + ch];

> >                          }

> >                      }

> >                  }

> > @@ -305,6 +328,11 @@ DNNReturnType

> ff_dnn_execute_model_native(const DNNModel *model, DNNData *output

> >              conv_params = (ConvolutionalParams

> *)network->layers[layer].params;

> >              convolve(network->layers[layer - 1].output,

> network->layers[layer].output, conv_params, cur_width, cur_height);

> >              cur_channels = conv_params->output_num;

> > +            if(conv_params->padding_method == VALID){

> > +                int pad_size = conv_params->kernel_size - 1;

> > +                cur_height -= pad_size;

> > +                cur_width -= pad_size;

> > +            }

> >              break;

> >          case DEPTH_TO_SPACE:

> >              depth_to_space_params = (DepthToSpaceParams

> *)network->layers[layer].params;

> > diff --git a/libavfilter/dnn_backend_native.h

> b/libavfilter/dnn_backend_native.h

> > index e13a68a168..d70cd16387 100644

> > --- a/libavfilter/dnn_backend_native.h

> > +++ b/libavfilter/dnn_backend_native.h

> > @@ -34,6 +34,8 @@ typedef enum {INPUT, CONV, DEPTH_TO_SPACE}

> DNNLayerType;

> >

> >  typedef enum {RELU, TANH, SIGMOID} DNNActivationFunc;

> >

> > +typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE}

> DNNConvPaddingParam;

> > +

> >  typedef struct Layer{

> >      DNNLayerType type;

> >      float *output;

> > @@ -43,6 +45,7 @@ typedef struct Layer{

> >  typedef struct ConvolutionalParams{

> >      int32_t input_num, output_num, kernel_size;

> >      DNNActivationFunc activation;

> > +    DNNConvPaddingParam padding_method;

> >      float *kernel;

> >      float *biases;

> >  } ConvolutionalParams;

> > --

> > 2.17.1

> >

> > _______________________________________________

> > ffmpeg-devel mailing list

> > ffmpeg-devel@ffmpeg.org

> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

> >

> > To unsubscribe, visit link above, or email

> > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

> 

> 

> The https://github.com/HighVoltageRocknRoll/sr has  loss of

> communication,and the project

> https://github.com/HighVoltageRocknRoll/sr has no maintainer now, so i

> think the pull request cannot be merge.

> 1. So i recommend Xuewei fork the project to his github, and merge the

> pr to his fork project, and modify the sr document of

> libavfilter/vf_sr.c. makes GSoC derain mentor project continue.


I prefer this one.

> 

> 2. If 1st way cannot be acceptable, Xuewei should duplicate DNN code

> for the derain.

> 

> Comments welcome.

> 

> Thanks

> 

> Steven

> _______________________________________________

> ffmpeg-devel mailing list

> ffmpeg-devel@ffmpeg.org

> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

> 

> To unsubscribe, visit link above, or email

> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
Xuewei Meng May 15, 2019, 8:40 a.m.
Guo, Yejun <yejun.guo@intel.com> 于2019年5月15日周三 下午2:21写道:

>
>
> > -----Original Message-----
> > From: ffmpeg-devel [mailto:ffmpeg-devel-bounces@ffmpeg.org] On Behalf Of
> > Steven Liu
> > Sent: Wednesday, May 15, 2019 10:38 AM
> > To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org>
> > Cc: Xuewei Meng <xwmeng96@gmail.com>
> > Subject: Re: [FFmpeg-devel] [PATCH v2] Add multiple padding method in dnn
> > native
> >
> > Xuewei Meng <xwmeng96@gmail.com> 于2019年5月11日周六 上午11:11
> > 写道:
> > >
> > > ---
> > >  libavfilter/dnn_backend_native.c | 52 ++++++++++++++++++++++++--------
> > >  libavfilter/dnn_backend_native.h |  3 ++
> > >  2 files changed, 43 insertions(+), 12 deletions(-)
>
> @xuewei, we still need to mention the impact of sr filter, and explain why
> same_clamp_to_edge is needed.
>
> There are three padding methods in this patch, VALID, SAME and
SAME_CLAMP_TO_EDGE. The 'VALID' and 'SAME' options are tensorflow supported
padding methods. And the third one, 'SAME_CLAMP_TO_EDGE', is suggested by
sr filter. As this method can keep the output with the same size as
original input, and gives a slight better result as mentioned by Pedro
Arthur. So we keep this option in dnn native mode.


> > >
> > > diff --git a/libavfilter/dnn_backend_native.c
> > b/libavfilter/dnn_backend_native.c
> > > index 06fbdf368b..171a756385 100644
> > > --- a/libavfilter/dnn_backend_native.c
> > > +++ b/libavfilter/dnn_backend_native.c
> > > @@ -61,6 +61,12 @@ static DNNReturnType set_input_output_native(void
> > *model, DNNInputData *input, c
> > >                  return DNN_ERROR;
> > >              }
> > >              cur_channels = conv_params->output_num;
> > > +
> > > +            if(conv_params->padding_method == VALID){
> > > +                int pad_size = conv_params->kernel_size - 1;
> > > +                cur_height -= pad_size;
> > > +                cur_width -= pad_size;
> > > +            }
> > >              break;
> > >          case DEPTH_TO_SPACE:
> > >              depth_to_space_params = (DepthToSpaceParams
> > *)network->layers[layer].params;
> > > @@ -77,6 +83,10 @@ static DNNReturnType set_input_output_native(void
> > *model, DNNInputData *input, c
> > >          if (network->layers[layer].output){
> > >              av_freep(&network->layers[layer].output);
> > >          }
> > > +
> > > +        if(cur_height <= 0 || cur_width <= 0)
> > > +            return DNN_ERROR;
> > > +
> > >          network->layers[layer].output = av_malloc(cur_height *
> > cur_width * cur_channels * sizeof(float));
> > >          if (!network->layers[layer].output){
> > >              return DNN_ERROR;
> > > @@ -154,13 +164,14 @@ DNNModel *ff_dnn_load_model_native(const
> > char *model_filename)
> > >                  ff_dnn_free_model_native(&model);
> > >                  return NULL;
> > >              }
> > > +            conv_params->padding_method =
> > (int32_t)avio_rl32(model_file_context);
> > >              conv_params->activation =
> > (int32_t)avio_rl32(model_file_context);
> > >              conv_params->input_num =
> > (int32_t)avio_rl32(model_file_context);
> > >              conv_params->output_num =
> > (int32_t)avio_rl32(model_file_context);
> > >              conv_params->kernel_size =
> > (int32_t)avio_rl32(model_file_context);
> > >              kernel_size = conv_params->input_num *
> > conv_params->output_num *
> > >                            conv_params->kernel_size *
> > conv_params->kernel_size;
> > > -            dnn_size += 16 + (kernel_size + conv_params->output_num
> > << 2);
> > > +            dnn_size += 20 + (kernel_size + conv_params->output_num
> > << 2);
> > >              if (dnn_size > file_size || conv_params->input_num <= 0 ||
> > >                  conv_params->output_num <= 0 ||
> > conv_params->kernel_size <= 0){
> > >                  avio_closep(&model_file_context);
> > > @@ -218,23 +229,35 @@ DNNModel *ff_dnn_load_model_native(const
> > char *model_filename)
> > >
> > >  static void convolve(const float *input, float *output, const
> > ConvolutionalParams *conv_params, int width, int height)
> > >  {
> > > -    int y, x, n_filter, ch, kernel_y, kernel_x;
> > >      int radius = conv_params->kernel_size >> 1;
> > >      int src_linesize = width * conv_params->input_num;
> > >      int filter_linesize = conv_params->kernel_size *
> > conv_params->input_num;
> > >      int filter_size = conv_params->kernel_size * filter_linesize;
> > > +    int pad_size = (conv_params->padding_method == VALID) ?
> > (conv_params->kernel_size - 1) / 2 : 0;
> > >
> > > -    for (y = 0; y < height; ++y){
> > > -        for (x = 0; x < width; ++x){
> > > -            for (n_filter = 0; n_filter < conv_params->output_num;
> > ++n_filter){
> > > +    for (int y = pad_size; y < height - pad_size; ++y){
> > > +        for (int x = pad_size; x < width - pad_size; ++x){
> > > +            for (int n_filter = 0; n_filter < conv_params->output_num;
> > ++n_filter){
> > >                  output[n_filter] = conv_params->biases[n_filter];
> > > -                for (ch = 0; ch < conv_params->input_num; ++ch){
> > > -                    for (kernel_y = 0; kernel_y <
> > conv_params->kernel_size; ++kernel_y){
> > > -                        for (kernel_x = 0; kernel_x <
> > conv_params->kernel_size; ++kernel_x){
> > > -                            output[n_filter] +=
> > input[CLAMP_TO_EDGE(y + kernel_y - radius, height) * src_linesize +
> > > -
> > CLAMP_TO_EDGE(x + kernel_x - radius, width) * conv_params->input_num +
> ch]
> > *
> > > -
> > conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize +
> > > -
> > kernel_x * conv_params->input_num + ch];
> > > +
> > > +                for (int ch = 0; ch < conv_params->input_num; ++ch){
> > > +                    for (int kernel_y = 0; kernel_y <
> > conv_params->kernel_size; ++kernel_y){
> > > +                        for (int kernel_x = 0; kernel_x <
> > conv_params->kernel_size; ++kernel_x){
> > > +                            float input_pel;
> > > +                            if(conv_params->padding_method ==
> > SAME_CLAMP_TO_EDGE){
> > > +                                int y_pos = CLAMP_TO_EDGE(y +
> > kernel_y - radius, height);
> > > +                                int x_pos = CLAMP_TO_EDGE(x +
> > kernel_x - radius, width);
> > > +                                input_pel = input[y_pos *
> > src_linesize + x_pos * conv_params->input_num + ch];
> > > +                            }else{
> > > +                                int y_pos = y + kernel_y - radius;
> > > +                                int x_pos = x + kernel_x - radius;
> > > +                                input_pel = (x_pos < 0 || x_pos >=
> > width || y_pos < 0 || y_pos >= height) ? 0.0 :
> > > +                                                   input[y_pos
> > * src_linesize + x_pos * conv_params->input_num + ch];
> > > +                            }
> > > +
> > > +
> > > +                            output[n_filter] += input_pel *
> > conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize +
> > > +
> > kernel_x * conv_params->input_num + ch];
> > >                          }
> > >                      }
> > >                  }
> > > @@ -305,6 +328,11 @@ DNNReturnType
> > ff_dnn_execute_model_native(const DNNModel *model, DNNData *output
> > >              conv_params = (ConvolutionalParams
> > *)network->layers[layer].params;
> > >              convolve(network->layers[layer - 1].output,
> > network->layers[layer].output, conv_params, cur_width, cur_height);
> > >              cur_channels = conv_params->output_num;
> > > +            if(conv_params->padding_method == VALID){
> > > +                int pad_size = conv_params->kernel_size - 1;
> > > +                cur_height -= pad_size;
> > > +                cur_width -= pad_size;
> > > +            }
> > >              break;
> > >          case DEPTH_TO_SPACE:
> > >              depth_to_space_params = (DepthToSpaceParams
> > *)network->layers[layer].params;
> > > diff --git a/libavfilter/dnn_backend_native.h
> > b/libavfilter/dnn_backend_native.h
> > > index e13a68a168..d70cd16387 100644
> > > --- a/libavfilter/dnn_backend_native.h
> > > +++ b/libavfilter/dnn_backend_native.h
> > > @@ -34,6 +34,8 @@ typedef enum {INPUT, CONV, DEPTH_TO_SPACE}
> > DNNLayerType;
> > >
> > >  typedef enum {RELU, TANH, SIGMOID} DNNActivationFunc;
> > >
> > > +typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE}
> > DNNConvPaddingParam;
> > > +
> > >  typedef struct Layer{
> > >      DNNLayerType type;
> > >      float *output;
> > > @@ -43,6 +45,7 @@ typedef struct Layer{
> > >  typedef struct ConvolutionalParams{
> > >      int32_t input_num, output_num, kernel_size;
> > >      DNNActivationFunc activation;
> > > +    DNNConvPaddingParam padding_method;
> > >      float *kernel;
> > >      float *biases;
> > >  } ConvolutionalParams;
> > > --
> > > 2.17.1
> > >
> > > _______________________________________________
> > > ffmpeg-devel mailing list
> > > ffmpeg-devel@ffmpeg.org
> > > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> > >
> > > To unsubscribe, visit link above, or email
> > > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
> >
> >
> > The https://github.com/HighVoltageRocknRoll/sr has  loss of
> > communication,and the project
> > https://github.com/HighVoltageRocknRoll/sr has no maintainer now, so i
> > think the pull request cannot be merge.
> > 1. So i recommend Xuewei fork the project to his github, and merge the
> > pr to his fork project, and modify the sr document of
> > libavfilter/vf_sr.c. makes GSoC derain mentor project continue.
>
> I prefer this one.
>
> >
> > 2. If 1st way cannot be acceptable, Xuewei should duplicate DNN code
> > for the derain.
> >
> > Comments welcome.
> >
> > Thanks
> >
> > Steven
> > _______________________________________________
> > ffmpeg-devel mailing list
> > ffmpeg-devel@ffmpeg.org
> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> >
> > To unsubscribe, visit link above, or email
> > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
>
Guo, Yejun May 15, 2019, 10:53 a.m.
> 

> 

> From: Xuewei Meng [mailto:xwmeng96@gmail.com]

> Sent: Wednesday, May 15, 2019 4:41 PM

> To: Guo, Yejun <yejun.guo@intel.com>

> Cc: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org>

> Subject: Re: [FFmpeg-devel] [PATCH v2] Add multiple padding method in dnn

> native

> 

> 

> 

> Guo, Yejun <yejun.guo@intel.com> 于2019年5月15日周三 下午2:21写道:

> 

> 

> > -----Original Message-----

> > From: ffmpeg-devel [mailto:ffmpeg-devel-bounces@ffmpeg.org] On Behalf Of

> > Steven Liu

> > Sent: Wednesday, May 15, 2019 10:38 AM

> > To: FFmpeg development discussions and patches

> <ffmpeg-devel@ffmpeg.org>

> > Cc: Xuewei Meng <xwmeng96@gmail.com>

> > Subject: Re: [FFmpeg-devel] [PATCH v2] Add multiple padding method in dnn

> > native

> >

> > Xuewei Meng <xwmeng96@gmail.com> 于2019年5月11日周六 上午

> 11:11

> > 写道:

> > >

> > > ---

> > >  libavfilter/dnn_backend_native.c | 52 ++++++++++++++++++++++++--------

> > >  libavfilter/dnn_backend_native.h |  3 ++

> > >  2 files changed, 43 insertions(+), 12 deletions(-)

> 

> @xuewei, we still need to mention the impact of sr filter, and explain why

> same_clamp_to_edge is needed.

> There are three padding methods in this patch, VALID, SAME and

> SAME_CLAMP_TO_EDGE. The 'VALID' and 'SAME' options are tensorflow

> supported padding methods. And the third one, 'SAME_CLAMP_TO_EDGE', is

> suggested by sr filter. As this method can keep the output with the same size

> as original input, and gives a slight better result as mentioned by Pedro Arthur.

> So we keep this option in dnn native mode.


nice, please add them into commit log. And also the impact to sr filter.

> 

> > >

> > > diff --git a/libavfilter/dnn_backend_native.c

> > b/libavfilter/dnn_backend_native.c

> > > index 06fbdf368b..171a756385 100644

> > > --- a/libavfilter/dnn_backend_native.c

> > > +++ b/libavfilter/dnn_backend_native.c

> > > @@ -61,6 +61,12 @@ static DNNReturnType

> set_input_output_native(void

> > *model, DNNInputData *input, c

> > >                  return DNN_ERROR;

> > >              }

> > >              cur_channels = conv_params->output_num;

> > > +

> > > +            if(conv_params->padding_method == VALID){

> > > +                int pad_size = conv_params->kernel_size - 1;

> > > +                cur_height -= pad_size;

> > > +                cur_width -= pad_size;

> > > +            }

> > >              break;

> > >          case DEPTH_TO_SPACE:

> > >              depth_to_space_params = (DepthToSpaceParams

> > *)network->layers[layer].params;

> > > @@ -77,6 +83,10 @@ static DNNReturnType

> set_input_output_native(void

> > *model, DNNInputData *input, c

> > >          if (network->layers[layer].output){

> > >              av_freep(&network->layers[layer].output);

> > >          }

> > > +

> > > +        if(cur_height <= 0 || cur_width <= 0)

> > > +            return DNN_ERROR;

> > > +

> > >          network->layers[layer].output = av_malloc(cur_height *

> > cur_width * cur_channels * sizeof(float));

> > >          if (!network->layers[layer].output){

> > >              return DNN_ERROR;

> > > @@ -154,13 +164,14 @@ DNNModel *ff_dnn_load_model_native(const

> > char *model_filename)

> > >                  ff_dnn_free_model_native(&model);

> > >                  return NULL;

> > >              }

> > > +            conv_params->padding_method =

> > (int32_t)avio_rl32(model_file_context);

> > >              conv_params->activation =

> > (int32_t)avio_rl32(model_file_context);

> > >              conv_params->input_num =

> > (int32_t)avio_rl32(model_file_context);

> > >              conv_params->output_num =

> > (int32_t)avio_rl32(model_file_context);

> > >              conv_params->kernel_size =

> > (int32_t)avio_rl32(model_file_context);

> > >              kernel_size = conv_params->input_num *

> > conv_params->output_num *

> > >                            conv_params->kernel_size *

> > conv_params->kernel_size;

> > > -            dnn_size += 16 + (kernel_size + conv_params->output_num

> > << 2);

> > > +            dnn_size += 20 + (kernel_size +

> conv_params->output_num

> > << 2);

> > >              if (dnn_size > file_size || conv_params->input_num <= 0

> ||

> > >                  conv_params->output_num <= 0 ||

> > conv_params->kernel_size <= 0){

> > >                  avio_closep(&model_file_context);

> > > @@ -218,23 +229,35 @@ DNNModel *ff_dnn_load_model_native(const

> > char *model_filename)

> > >

> > >  static void convolve(const float *input, float *output, const

> > ConvolutionalParams *conv_params, int width, int height)

> > >  {

> > > -    int y, x, n_filter, ch, kernel_y, kernel_x;

> > >      int radius = conv_params->kernel_size >> 1;

> > >      int src_linesize = width * conv_params->input_num;

> > >      int filter_linesize = conv_params->kernel_size *

> > conv_params->input_num;

> > >      int filter_size = conv_params->kernel_size * filter_linesize;

> > > +    int pad_size = (conv_params->padding_method == VALID) ?

> > (conv_params->kernel_size - 1) / 2 : 0;

> > >

> > > -    for (y = 0; y < height; ++y){

> > > -        for (x = 0; x < width; ++x){

> > > -            for (n_filter = 0; n_filter < conv_params->output_num;

> > ++n_filter){

> > > +    for (int y = pad_size; y < height - pad_size; ++y){

> > > +        for (int x = pad_size; x < width - pad_size; ++x){

> > > +            for (int n_filter = 0; n_filter < conv_params->output_num;

> > ++n_filter){

> > >                  output[n_filter] = conv_params->biases[n_filter];

> > > -                for (ch = 0; ch < conv_params->input_num; ++ch){

> > > -                    for (kernel_y = 0; kernel_y <

> > conv_params->kernel_size; ++kernel_y){

> > > -                        for (kernel_x = 0; kernel_x <

> > conv_params->kernel_size; ++kernel_x){

> > > -                            output[n_filter] +=

> > input[CLAMP_TO_EDGE(y + kernel_y - radius, height) * src_linesize +

> > > -

> > CLAMP_TO_EDGE(x + kernel_x - radius, width) * conv_params->input_num +

> ch]

> > *

> > > -

> > conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize +

> > > -

> > kernel_x * conv_params->input_num + ch];

> > > +

> > > +                for (int ch = 0; ch < conv_params->input_num; ++ch){

> > > +                    for (int kernel_y = 0; kernel_y <

> > conv_params->kernel_size; ++kernel_y){

> > > +                        for (int kernel_x = 0; kernel_x <

> > conv_params->kernel_size; ++kernel_x){

> > > +                            float input_pel;

> > > +                            if(conv_params->padding_method ==

> > SAME_CLAMP_TO_EDGE){

> > > +                                int y_pos = CLAMP_TO_EDGE(y

> +

> > kernel_y - radius, height);

> > > +                                int x_pos = CLAMP_TO_EDGE(x

> +

> > kernel_x - radius, width);

> > > +                                input_pel = input[y_pos *

> > src_linesize + x_pos * conv_params->input_num + ch];

> > > +                            }else{

> > > +                                int y_pos = y + kernel_y - radius;

> > > +                                int x_pos = x + kernel_x - radius;

> > > +                                input_pel = (x_pos < 0 ||

> x_pos >=

> > width || y_pos < 0 || y_pos >= height) ? 0.0 :

> > >

> +                                                   input[y_p

> os

> > * src_linesize + x_pos * conv_params->input_num + ch];

> > > +                            }

> > > +

> > > +

> > > +                            output[n_filter] += input_pel *

> > conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize +

> > > +

> > kernel_x * conv_params->input_num + ch];

> > >                          }

> > >                      }

> > >                  }

> > > @@ -305,6 +328,11 @@ DNNReturnType

> > ff_dnn_execute_model_native(const DNNModel *model, DNNData *output

> > >              conv_params = (ConvolutionalParams

> > *)network->layers[layer].params;

> > >              convolve(network->layers[layer - 1].output,

> > network->layers[layer].output, conv_params, cur_width, cur_height);

> > >              cur_channels = conv_params->output_num;

> > > +            if(conv_params->padding_method == VALID){

> > > +                int pad_size = conv_params->kernel_size - 1;

> > > +                cur_height -= pad_size;

> > > +                cur_width -= pad_size;

> > > +            }

> > >              break;

> > >          case DEPTH_TO_SPACE:

> > >              depth_to_space_params = (DepthToSpaceParams

> > *)network->layers[layer].params;

> > > diff --git a/libavfilter/dnn_backend_native.h

> > b/libavfilter/dnn_backend_native.h

> > > index e13a68a168..d70cd16387 100644

> > > --- a/libavfilter/dnn_backend_native.h

> > > +++ b/libavfilter/dnn_backend_native.h

> > > @@ -34,6 +34,8 @@ typedef enum {INPUT, CONV, DEPTH_TO_SPACE}

> > DNNLayerType;

> > >

> > >  typedef enum {RELU, TANH, SIGMOID} DNNActivationFunc;

> > >

> > > +typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE}

> > DNNConvPaddingParam;

> > > +

> > >  typedef struct Layer{

> > >      DNNLayerType type;

> > >      float *output;

> > > @@ -43,6 +45,7 @@ typedef struct Layer{

> > >  typedef struct ConvolutionalParams{

> > >      int32_t input_num, output_num, kernel_size;

> > >      DNNActivationFunc activation;

> > > +    DNNConvPaddingParam padding_method;

> > >      float *kernel;

> > >      float *biases;

> > >  } ConvolutionalParams;

> > > --

> > > 2.17.1

> > >

> > > _______________________________________________

> > > ffmpeg-devel mailing list

> > > ffmpeg-devel@ffmpeg.org

> > > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

> > >

> > > To unsubscribe, visit link above, or email

> > > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

> >

> >

> > The https://github.com/HighVoltageRocknRoll/sr has  loss of

> > communication,and the project

> > https://github.com/HighVoltageRocknRoll/sr has no maintainer now, so i

> > think the pull request cannot be merge.

> > 1. So i recommend Xuewei fork the project to his github, and merge the

> > pr to his fork project, and modify the sr document of

> > libavfilter/vf_sr.c. makes GSoC derain mentor project continue.

> 

> I prefer this one.

> 

> >

> > 2. If 1st way cannot be acceptable, Xuewei should duplicate DNN code

> > for the derain.

> >

> > Comments welcome.

> >

> > Thanks

> >

> > Steven

> > _______________________________________________

> > ffmpeg-devel mailing list

> > ffmpeg-devel@ffmpeg.org

> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel

> >

> > To unsubscribe, visit link above, or email

> > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
Pedro Arthur May 15, 2019, 1:34 p.m.
Em qua, 15 de mai de 2019 às 04:44, Steven Liu
<lingjiujianke@gmail.com> escreveu:
>
> Xuewei Meng <xwmeng96@gmail.com> 于2019年5月11日周六 上午11:11写道:
> >
> > ---
> >  libavfilter/dnn_backend_native.c | 52 ++++++++++++++++++++++++--------
> >  libavfilter/dnn_backend_native.h |  3 ++
> >  2 files changed, 43 insertions(+), 12 deletions(-)
> >
> > diff --git a/libavfilter/dnn_backend_native.c b/libavfilter/dnn_backend_native.c
> > index 06fbdf368b..171a756385 100644
> > --- a/libavfilter/dnn_backend_native.c
> > +++ b/libavfilter/dnn_backend_native.c
> > @@ -61,6 +61,12 @@ static DNNReturnType set_input_output_native(void *model, DNNInputData *input, c
> >                  return DNN_ERROR;
> >              }
> >              cur_channels = conv_params->output_num;
> > +
> > +            if(conv_params->padding_method == VALID){
> > +                int pad_size = conv_params->kernel_size - 1;
> > +                cur_height -= pad_size;
> > +                cur_width -= pad_size;
> > +            }
> >              break;
> >          case DEPTH_TO_SPACE:
> >              depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params;
> > @@ -77,6 +83,10 @@ static DNNReturnType set_input_output_native(void *model, DNNInputData *input, c
> >          if (network->layers[layer].output){
> >              av_freep(&network->layers[layer].output);
> >          }
> > +
> > +        if(cur_height <= 0 || cur_width <= 0)
> > +            return DNN_ERROR;
> > +
> >          network->layers[layer].output = av_malloc(cur_height * cur_width * cur_channels * sizeof(float));
> >          if (!network->layers[layer].output){
> >              return DNN_ERROR;
> > @@ -154,13 +164,14 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
> >                  ff_dnn_free_model_native(&model);
> >                  return NULL;
> >              }
> > +            conv_params->padding_method = (int32_t)avio_rl32(model_file_context);
> >              conv_params->activation = (int32_t)avio_rl32(model_file_context);
> >              conv_params->input_num = (int32_t)avio_rl32(model_file_context);
> >              conv_params->output_num = (int32_t)avio_rl32(model_file_context);
> >              conv_params->kernel_size = (int32_t)avio_rl32(model_file_context);
> >              kernel_size = conv_params->input_num * conv_params->output_num *
> >                            conv_params->kernel_size * conv_params->kernel_size;
> > -            dnn_size += 16 + (kernel_size + conv_params->output_num << 2);
> > +            dnn_size += 20 + (kernel_size + conv_params->output_num << 2);
> >              if (dnn_size > file_size || conv_params->input_num <= 0 ||
> >                  conv_params->output_num <= 0 || conv_params->kernel_size <= 0){
> >                  avio_closep(&model_file_context);
> > @@ -218,23 +229,35 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename)
> >
> >  static void convolve(const float *input, float *output, const ConvolutionalParams *conv_params, int width, int height)
> >  {
> > -    int y, x, n_filter, ch, kernel_y, kernel_x;
> >      int radius = conv_params->kernel_size >> 1;
> >      int src_linesize = width * conv_params->input_num;
> >      int filter_linesize = conv_params->kernel_size * conv_params->input_num;
> >      int filter_size = conv_params->kernel_size * filter_linesize;
> > +    int pad_size = (conv_params->padding_method == VALID) ? (conv_params->kernel_size - 1) / 2 : 0;
> >
> > -    for (y = 0; y < height; ++y){
> > -        for (x = 0; x < width; ++x){
> > -            for (n_filter = 0; n_filter < conv_params->output_num; ++n_filter){
> > +    for (int y = pad_size; y < height - pad_size; ++y){
> > +        for (int x = pad_size; x < width - pad_size; ++x){
> > +            for (int n_filter = 0; n_filter < conv_params->output_num; ++n_filter){
> >                  output[n_filter] = conv_params->biases[n_filter];
> > -                for (ch = 0; ch < conv_params->input_num; ++ch){
> > -                    for (kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y){
> > -                        for (kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x){
> > -                            output[n_filter] += input[CLAMP_TO_EDGE(y + kernel_y - radius, height) * src_linesize +
> > -                                                      CLAMP_TO_EDGE(x + kernel_x - radius, width) * conv_params->input_num + ch] *
> > -                                                conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize +
> > -                                                                    kernel_x * conv_params->input_num + ch];
> > +
> > +                for (int ch = 0; ch < conv_params->input_num; ++ch){
> > +                    for (int kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y){
> > +                        for (int kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x){
> > +                            float input_pel;
> > +                            if(conv_params->padding_method == SAME_CLAMP_TO_EDGE){
> > +                                int y_pos = CLAMP_TO_EDGE(y + kernel_y - radius, height);
> > +                                int x_pos = CLAMP_TO_EDGE(x + kernel_x - radius, width);
> > +                                input_pel = input[y_pos * src_linesize + x_pos * conv_params->input_num + ch];
> > +                            }else{
> > +                                int y_pos = y + kernel_y - radius;
> > +                                int x_pos = x + kernel_x - radius;
> > +                                input_pel = (x_pos < 0 || x_pos >= width || y_pos < 0 || y_pos >= height) ? 0.0 :
> > +                                                   input[y_pos * src_linesize + x_pos * conv_params->input_num + ch];
> > +                            }
> > +
> > +
> > +                            output[n_filter] += input_pel * conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize +
> > +                                                                                kernel_x * conv_params->input_num + ch];
> >                          }
> >                      }
> >                  }
> > @@ -305,6 +328,11 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *output
> >              conv_params = (ConvolutionalParams *)network->layers[layer].params;
> >              convolve(network->layers[layer - 1].output, network->layers[layer].output, conv_params, cur_width, cur_height);
> >              cur_channels = conv_params->output_num;
> > +            if(conv_params->padding_method == VALID){
> > +                int pad_size = conv_params->kernel_size - 1;
> > +                cur_height -= pad_size;
> > +                cur_width -= pad_size;
> > +            }
> >              break;
> >          case DEPTH_TO_SPACE:
> >              depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params;
> > diff --git a/libavfilter/dnn_backend_native.h b/libavfilter/dnn_backend_native.h
> > index e13a68a168..d70cd16387 100644
> > --- a/libavfilter/dnn_backend_native.h
> > +++ b/libavfilter/dnn_backend_native.h
> > @@ -34,6 +34,8 @@ typedef enum {INPUT, CONV, DEPTH_TO_SPACE} DNNLayerType;
> >
> >  typedef enum {RELU, TANH, SIGMOID} DNNActivationFunc;
> >
> > +typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} DNNConvPaddingParam;
> > +
> >  typedef struct Layer{
> >      DNNLayerType type;
> >      float *output;
> > @@ -43,6 +45,7 @@ typedef struct Layer{
> >  typedef struct ConvolutionalParams{
> >      int32_t input_num, output_num, kernel_size;
> >      DNNActivationFunc activation;
> > +    DNNConvPaddingParam padding_method;
> >      float *kernel;
> >      float *biases;
> >  } ConvolutionalParams;
> > --
> > 2.17.1
> >
> > _______________________________________________
> > ffmpeg-devel mailing list
> > ffmpeg-devel@ffmpeg.org
> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> >
> > To unsubscribe, visit link above, or email
> > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
>
>
> The https://github.com/HighVoltageRocknRoll/sr has  loss of
> communication,and the project
> https://github.com/HighVoltageRocknRoll/sr has no maintainer now, so i
> think the pull request cannot be merge.
> 1. So i recommend Xuewei fork the project to his github, and merge the
> pr to his fork project, and modify the sr document of
> libavfilter/vf_sr.c. makes GSoC derain mentor project continue.
>
I see no problem with 1, anyway he already had to make a fork to
create the pr, he can continue developing it there.

> 2. If 1st way cannot be acceptable, Xuewei should duplicate DNN code
> for the derain.
>
> Comments welcome.
>
> Thanks
>
> Steven
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Patch hide | download patch | download mbox

diff --git a/libavfilter/dnn_backend_native.c b/libavfilter/dnn_backend_native.c
index 06fbdf368b..171a756385 100644
--- a/libavfilter/dnn_backend_native.c
+++ b/libavfilter/dnn_backend_native.c
@@ -61,6 +61,12 @@  static DNNReturnType set_input_output_native(void *model, DNNInputData *input, c
                 return DNN_ERROR;
             }
             cur_channels = conv_params->output_num;
+
+            if(conv_params->padding_method == VALID){
+                int pad_size = conv_params->kernel_size - 1;
+                cur_height -= pad_size;
+                cur_width -= pad_size;
+            }
             break;
         case DEPTH_TO_SPACE:
             depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params;
@@ -77,6 +83,10 @@  static DNNReturnType set_input_output_native(void *model, DNNInputData *input, c
         if (network->layers[layer].output){
             av_freep(&network->layers[layer].output);
         }
+
+        if(cur_height <= 0 || cur_width <= 0)
+            return DNN_ERROR; 
+        
         network->layers[layer].output = av_malloc(cur_height * cur_width * cur_channels * sizeof(float));
         if (!network->layers[layer].output){
             return DNN_ERROR;
@@ -154,13 +164,14 @@  DNNModel *ff_dnn_load_model_native(const char *model_filename)
                 ff_dnn_free_model_native(&model);
                 return NULL;
             }
+            conv_params->padding_method = (int32_t)avio_rl32(model_file_context);
             conv_params->activation = (int32_t)avio_rl32(model_file_context);
             conv_params->input_num = (int32_t)avio_rl32(model_file_context);
             conv_params->output_num = (int32_t)avio_rl32(model_file_context);
             conv_params->kernel_size = (int32_t)avio_rl32(model_file_context);
             kernel_size = conv_params->input_num * conv_params->output_num *
                           conv_params->kernel_size * conv_params->kernel_size;
-            dnn_size += 16 + (kernel_size + conv_params->output_num << 2);
+            dnn_size += 20 + (kernel_size + conv_params->output_num << 2);
             if (dnn_size > file_size || conv_params->input_num <= 0 ||
                 conv_params->output_num <= 0 || conv_params->kernel_size <= 0){
                 avio_closep(&model_file_context);
@@ -218,23 +229,35 @@  DNNModel *ff_dnn_load_model_native(const char *model_filename)
 
 static void convolve(const float *input, float *output, const ConvolutionalParams *conv_params, int width, int height)
 {
-    int y, x, n_filter, ch, kernel_y, kernel_x;
     int radius = conv_params->kernel_size >> 1;
     int src_linesize = width * conv_params->input_num;
     int filter_linesize = conv_params->kernel_size * conv_params->input_num;
     int filter_size = conv_params->kernel_size * filter_linesize;
+    int pad_size = (conv_params->padding_method == VALID) ? (conv_params->kernel_size - 1) / 2 : 0;
 
-    for (y = 0; y < height; ++y){
-        for (x = 0; x < width; ++x){
-            for (n_filter = 0; n_filter < conv_params->output_num; ++n_filter){
+    for (int y = pad_size; y < height - pad_size; ++y){
+        for (int x = pad_size; x < width - pad_size; ++x){
+            for (int n_filter = 0; n_filter < conv_params->output_num; ++n_filter){
                 output[n_filter] = conv_params->biases[n_filter];
-                for (ch = 0; ch < conv_params->input_num; ++ch){
-                    for (kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y){
-                        for (kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x){
-                            output[n_filter] += input[CLAMP_TO_EDGE(y + kernel_y - radius, height) * src_linesize +
-                                                      CLAMP_TO_EDGE(x + kernel_x - radius, width) * conv_params->input_num + ch] *
-                                                conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize +
-                                                                    kernel_x * conv_params->input_num + ch];
+
+                for (int ch = 0; ch < conv_params->input_num; ++ch){
+                    for (int kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y){
+                        for (int kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x){
+                            float input_pel;
+                            if(conv_params->padding_method == SAME_CLAMP_TO_EDGE){
+                                int y_pos = CLAMP_TO_EDGE(y + kernel_y - radius, height);
+                                int x_pos = CLAMP_TO_EDGE(x + kernel_x - radius, width);
+                                input_pel = input[y_pos * src_linesize + x_pos * conv_params->input_num + ch];
+                            }else{
+                                int y_pos = y + kernel_y - radius;
+                                int x_pos = x + kernel_x - radius;
+                                input_pel = (x_pos < 0 || x_pos >= width || y_pos < 0 || y_pos >= height) ? 0.0 : 
+                                                   input[y_pos * src_linesize + x_pos * conv_params->input_num + ch];
+                            }
+
+
+                            output[n_filter] += input_pel * conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize +
+                                                                                kernel_x * conv_params->input_num + ch];
                         }
                     }
                 }
@@ -305,6 +328,11 @@  DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *output
             conv_params = (ConvolutionalParams *)network->layers[layer].params;
             convolve(network->layers[layer - 1].output, network->layers[layer].output, conv_params, cur_width, cur_height);
             cur_channels = conv_params->output_num;
+            if(conv_params->padding_method == VALID){
+                int pad_size = conv_params->kernel_size - 1;
+                cur_height -= pad_size;
+                cur_width -= pad_size;
+            }
             break;
         case DEPTH_TO_SPACE:
             depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params;
diff --git a/libavfilter/dnn_backend_native.h b/libavfilter/dnn_backend_native.h
index e13a68a168..d70cd16387 100644
--- a/libavfilter/dnn_backend_native.h
+++ b/libavfilter/dnn_backend_native.h
@@ -34,6 +34,8 @@  typedef enum {INPUT, CONV, DEPTH_TO_SPACE} DNNLayerType;
 
 typedef enum {RELU, TANH, SIGMOID} DNNActivationFunc;
 
+typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} DNNConvPaddingParam;
+
 typedef struct Layer{
     DNNLayerType type;
     float *output;
@@ -43,6 +45,7 @@  typedef struct Layer{
 typedef struct ConvolutionalParams{
     int32_t input_num, output_num, kernel_size;
     DNNActivationFunc activation;
+    DNNConvPaddingParam padding_method;
     float *kernel;
     float *biases;
 } ConvolutionalParams;