| Message ID | 20190511031118.584-1-xwmeng96@gmail.com |
|---|---|
| State | New |
| Headers | show |
Xuewei Meng <xwmeng96@gmail.com> 于2019年5月11日周六 上午11:11写道： > > --- > libavfilter/dnn_backend_native.c | 52 ++++++++++++++++++++++++-------- > libavfilter/dnn_backend_native.h | 3 ++ > 2 files changed, 43 insertions(+), 12 deletions(-) > > diff --git a/libavfilter/dnn_backend_native.c b/libavfilter/dnn_backend_native.c > index 06fbdf368b..171a756385 100644 > --- a/libavfilter/dnn_backend_native.c > +++ b/libavfilter/dnn_backend_native.c > @@ -61,6 +61,12 @@ static DNNReturnType set_input_output_native(void *model, DNNInputData *input, c > return DNN_ERROR; > } > cur_channels = conv_params->output_num; > + > + if(conv_params->padding_method == VALID){ > + int pad_size = conv_params->kernel_size - 1; > + cur_height -= pad_size; > + cur_width -= pad_size; > + } > break; > case DEPTH_TO_SPACE: > depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params; > @@ -77,6 +83,10 @@ static DNNReturnType set_input_output_native(void *model, DNNInputData *input, c > if (network->layers[layer].output){ > av_freep(&network->layers[layer].output); > } > + > + if(cur_height <= 0 || cur_width <= 0) > + return DNN_ERROR; > + > network->layers[layer].output = av_malloc(cur_height * cur_width * cur_channels * sizeof(float)); > if (!network->layers[layer].output){ > return DNN_ERROR; > @@ -154,13 +164,14 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) > ff_dnn_free_model_native(&model); > return NULL; > } > + conv_params->padding_method = (int32_t)avio_rl32(model_file_context); > conv_params->activation = (int32_t)avio_rl32(model_file_context); > conv_params->input_num = (int32_t)avio_rl32(model_file_context); > conv_params->output_num = (int32_t)avio_rl32(model_file_context); > conv_params->kernel_size = (int32_t)avio_rl32(model_file_context); > kernel_size = conv_params->input_num * conv_params->output_num * > conv_params->kernel_size * conv_params->kernel_size; > - dnn_size += 16 + (kernel_size + conv_params->output_num << 2); > + dnn_size += 20 + (kernel_size + conv_params->output_num << 2); > if (dnn_size > file_size || conv_params->input_num <= 0 || > conv_params->output_num <= 0 || conv_params->kernel_size <= 0){ > avio_closep(&model_file_context); > @@ -218,23 +229,35 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) > > static void convolve(const float *input, float *output, const ConvolutionalParams *conv_params, int width, int height) > { > - int y, x, n_filter, ch, kernel_y, kernel_x; > int radius = conv_params->kernel_size >> 1; > int src_linesize = width * conv_params->input_num; > int filter_linesize = conv_params->kernel_size * conv_params->input_num; > int filter_size = conv_params->kernel_size * filter_linesize; > + int pad_size = (conv_params->padding_method == VALID) ? (conv_params->kernel_size - 1) / 2 : 0; > > - for (y = 0; y < height; ++y){ > - for (x = 0; x < width; ++x){ > - for (n_filter = 0; n_filter < conv_params->output_num; ++n_filter){ > + for (int y = pad_size; y < height - pad_size; ++y){ > + for (int x = pad_size; x < width - pad_size; ++x){ > + for (int n_filter = 0; n_filter < conv_params->output_num; ++n_filter){ > output[n_filter] = conv_params->biases[n_filter]; > - for (ch = 0; ch < conv_params->input_num; ++ch){ > - for (kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y){ > - for (kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x){ > - output[n_filter] += input[CLAMP_TO_EDGE(y + kernel_y - radius, height) * src_linesize + > - CLAMP_TO_EDGE(x + kernel_x - radius, width) * conv_params->input_num + ch] * > - conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize + > - kernel_x * conv_params->input_num + ch]; > + > + for (int ch = 0; ch < conv_params->input_num; ++ch){ > + for (int kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y){ > + for (int kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x){ > + float input_pel; > + if(conv_params->padding_method == SAME_CLAMP_TO_EDGE){ > + int y_pos = CLAMP_TO_EDGE(y + kernel_y - radius, height); > + int x_pos = CLAMP_TO_EDGE(x + kernel_x - radius, width); > + input_pel = input[y_pos * src_linesize + x_pos * conv_params->input_num + ch]; > + }else{ > + int y_pos = y + kernel_y - radius; > + int x_pos = x + kernel_x - radius; > + input_pel = (x_pos < 0 || x_pos >= width || y_pos < 0 || y_pos >= height) ? 0.0 : > + input[y_pos * src_linesize + x_pos * conv_params->input_num + ch]; > + } > + > + > + output[n_filter] += input_pel * conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize + > + kernel_x * conv_params->input_num + ch]; > } > } > } > @@ -305,6 +328,11 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *output > conv_params = (ConvolutionalParams *)network->layers[layer].params; > convolve(network->layers[layer - 1].output, network->layers[layer].output, conv_params, cur_width, cur_height); > cur_channels = conv_params->output_num; > + if(conv_params->padding_method == VALID){ > + int pad_size = conv_params->kernel_size - 1; > + cur_height -= pad_size; > + cur_width -= pad_size; > + } > break; > case DEPTH_TO_SPACE: > depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params; > diff --git a/libavfilter/dnn_backend_native.h b/libavfilter/dnn_backend_native.h > index e13a68a168..d70cd16387 100644 > --- a/libavfilter/dnn_backend_native.h > +++ b/libavfilter/dnn_backend_native.h > @@ -34,6 +34,8 @@ typedef enum {INPUT, CONV, DEPTH_TO_SPACE} DNNLayerType; > > typedef enum {RELU, TANH, SIGMOID} DNNActivationFunc; > > +typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} DNNConvPaddingParam; > + > typedef struct Layer{ > DNNLayerType type; > float *output; > @@ -43,6 +45,7 @@ typedef struct Layer{ > typedef struct ConvolutionalParams{ > int32_t input_num, output_num, kernel_size; > DNNActivationFunc activation; > + DNNConvPaddingParam padding_method; > float *kernel; > float *biases; > } ConvolutionalParams; > -- > 2.17.1 > > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". The https://github.com/HighVoltageRocknRoll/sr has loss of communication，and the project https://github.com/HighVoltageRocknRoll/sr has no maintainer now, so i think the pull request cannot be merge. 1. So i recommend Xuewei fork the project to his github, and merge the pr to his fork project, and modify the sr document of libavfilter/vf_sr.c. makes GSoC derain mentor project continue. 2. If 1st way cannot be acceptable, Xuewei should duplicate DNN code for the derain. Comments welcome. Thanks Steven
> -----Original Message----- > From: ffmpeg-devel [mailto:ffmpeg-devel-bounces@ffmpeg.org] On Behalf Of > Steven Liu > Sent: Wednesday, May 15, 2019 10:38 AM > To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org> > Cc: Xuewei Meng <xwmeng96@gmail.com> > Subject: Re: [FFmpeg-devel] [PATCH v2] Add multiple padding method in dnn > native > > Xuewei Meng <xwmeng96@gmail.com> 于2019年5月11日周六 上午11:11 > 写道： > > > > --- > > libavfilter/dnn_backend_native.c | 52 ++++++++++++++++++++++++-------- > > libavfilter/dnn_backend_native.h | 3 ++ > > 2 files changed, 43 insertions(+), 12 deletions(-) @xuewei, we still need to mention the impact of sr filter, and explain why same_clamp_to_edge is needed. > > > > diff --git a/libavfilter/dnn_backend_native.c > b/libavfilter/dnn_backend_native.c > > index 06fbdf368b..171a756385 100644 > > --- a/libavfilter/dnn_backend_native.c > > +++ b/libavfilter/dnn_backend_native.c > > @@ -61,6 +61,12 @@ static DNNReturnType set_input_output_native(void > *model, DNNInputData *input, c > > return DNN_ERROR; > > } > > cur_channels = conv_params->output_num; > > + > > + if(conv_params->padding_method == VALID){ > > + int pad_size = conv_params->kernel_size - 1; > > + cur_height -= pad_size; > > + cur_width -= pad_size; > > + } > > break; > > case DEPTH_TO_SPACE: > > depth_to_space_params = (DepthToSpaceParams > *)network->layers[layer].params; > > @@ -77,6 +83,10 @@ static DNNReturnType set_input_output_native(void > *model, DNNInputData *input, c > > if (network->layers[layer].output){ > > av_freep(&network->layers[layer].output); > > } > > + > > + if(cur_height <= 0 || cur_width <= 0) > > + return DNN_ERROR; > > + > > network->layers[layer].output = av_malloc(cur_height * > cur_width * cur_channels * sizeof(float)); > > if (!network->layers[layer].output){ > > return DNN_ERROR; > > @@ -154,13 +164,14 @@ DNNModel *ff_dnn_load_model_native(const > char *model_filename) > > ff_dnn_free_model_native(&model); > > return NULL; > > } > > + conv_params->padding_method = > (int32_t)avio_rl32(model_file_context); > > conv_params->activation = > (int32_t)avio_rl32(model_file_context); > > conv_params->input_num = > (int32_t)avio_rl32(model_file_context); > > conv_params->output_num = > (int32_t)avio_rl32(model_file_context); > > conv_params->kernel_size = > (int32_t)avio_rl32(model_file_context); > > kernel_size = conv_params->input_num * > conv_params->output_num * > > conv_params->kernel_size * > conv_params->kernel_size; > > - dnn_size += 16 + (kernel_size + conv_params->output_num > << 2); > > + dnn_size += 20 + (kernel_size + conv_params->output_num > << 2); > > if (dnn_size > file_size || conv_params->input_num <= 0 || > > conv_params->output_num <= 0 || > conv_params->kernel_size <= 0){ > > avio_closep(&model_file_context); > > @@ -218,23 +229,35 @@ DNNModel *ff_dnn_load_model_native(const > char *model_filename) > > > > static void convolve(const float *input, float *output, const > ConvolutionalParams *conv_params, int width, int height) > > { > > - int y, x, n_filter, ch, kernel_y, kernel_x; > > int radius = conv_params->kernel_size >> 1; > > int src_linesize = width * conv_params->input_num; > > int filter_linesize = conv_params->kernel_size * > conv_params->input_num; > > int filter_size = conv_params->kernel_size * filter_linesize; > > + int pad_size = (conv_params->padding_method == VALID) ? > (conv_params->kernel_size - 1) / 2 : 0; > > > > - for (y = 0; y < height; ++y){ > > - for (x = 0; x < width; ++x){ > > - for (n_filter = 0; n_filter < conv_params->output_num; > ++n_filter){ > > + for (int y = pad_size; y < height - pad_size; ++y){ > > + for (int x = pad_size; x < width - pad_size; ++x){ > > + for (int n_filter = 0; n_filter < conv_params->output_num; > ++n_filter){ > > output[n_filter] = conv_params->biases[n_filter]; > > - for (ch = 0; ch < conv_params->input_num; ++ch){ > > - for (kernel_y = 0; kernel_y < > conv_params->kernel_size; ++kernel_y){ > > - for (kernel_x = 0; kernel_x < > conv_params->kernel_size; ++kernel_x){ > > - output[n_filter] += > input[CLAMP_TO_EDGE(y + kernel_y - radius, height) * src_linesize + > > - > CLAMP_TO_EDGE(x + kernel_x - radius, width) * conv_params->input_num + ch] > * > > - > conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize + > > - > kernel_x * conv_params->input_num + ch]; > > + > > + for (int ch = 0; ch < conv_params->input_num; ++ch){ > > + for (int kernel_y = 0; kernel_y < > conv_params->kernel_size; ++kernel_y){ > > + for (int kernel_x = 0; kernel_x < > conv_params->kernel_size; ++kernel_x){ > > + float input_pel; > > + if(conv_params->padding_method == > SAME_CLAMP_TO_EDGE){ > > + int y_pos = CLAMP_TO_EDGE(y + > kernel_y - radius, height); > > + int x_pos = CLAMP_TO_EDGE(x + > kernel_x - radius, width); > > + input_pel = input[y_pos * > src_linesize + x_pos * conv_params->input_num + ch]; > > + }else{ > > + int y_pos = y + kernel_y - radius; > > + int x_pos = x + kernel_x - radius; > > + input_pel = (x_pos < 0 || x_pos >= > width || y_pos < 0 || y_pos >= height) ? 0.0 : > > + input[y_pos > * src_linesize + x_pos * conv_params->input_num + ch]; > > + } > > + > > + > > + output[n_filter] += input_pel * > conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize + > > + > kernel_x * conv_params->input_num + ch]; > > } > > } > > } > > @@ -305,6 +328,11 @@ DNNReturnType > ff_dnn_execute_model_native(const DNNModel *model, DNNData *output > > conv_params = (ConvolutionalParams > *)network->layers[layer].params; > > convolve(network->layers[layer - 1].output, > network->layers[layer].output, conv_params, cur_width, cur_height); > > cur_channels = conv_params->output_num; > > + if(conv_params->padding_method == VALID){ > > + int pad_size = conv_params->kernel_size - 1; > > + cur_height -= pad_size; > > + cur_width -= pad_size; > > + } > > break; > > case DEPTH_TO_SPACE: > > depth_to_space_params = (DepthToSpaceParams > *)network->layers[layer].params; > > diff --git a/libavfilter/dnn_backend_native.h > b/libavfilter/dnn_backend_native.h > > index e13a68a168..d70cd16387 100644 > > --- a/libavfilter/dnn_backend_native.h > > +++ b/libavfilter/dnn_backend_native.h > > @@ -34,6 +34,8 @@ typedef enum {INPUT, CONV, DEPTH_TO_SPACE} > DNNLayerType; > > > > typedef enum {RELU, TANH, SIGMOID} DNNActivationFunc; > > > > +typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} > DNNConvPaddingParam; > > + > > typedef struct Layer{ > > DNNLayerType type; > > float *output; > > @@ -43,6 +45,7 @@ typedef struct Layer{ > > typedef struct ConvolutionalParams{ > > int32_t input_num, output_num, kernel_size; > > DNNActivationFunc activation; > > + DNNConvPaddingParam padding_method; > > float *kernel; > > float *biases; > > } ConvolutionalParams; > > -- > > 2.17.1 > > > > _______________________________________________ > > ffmpeg-devel mailing list > > ffmpeg-devel@ffmpeg.org > > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > > > To unsubscribe, visit link above, or email > > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". > > > The https://github.com/HighVoltageRocknRoll/sr has loss of > communication，and the project > https://github.com/HighVoltageRocknRoll/sr has no maintainer now, so i > think the pull request cannot be merge. > 1. So i recommend Xuewei fork the project to his github, and merge the > pr to his fork project, and modify the sr document of > libavfilter/vf_sr.c. makes GSoC derain mentor project continue. I prefer this one. > > 2. If 1st way cannot be acceptable, Xuewei should duplicate DNN code > for the derain. > > Comments welcome. > > Thanks > > Steven > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
Guo, Yejun <yejun.guo@intel.com> 于2019年5月15日周三 下午2:21写道： > > > > -----Original Message----- > > From: ffmpeg-devel [mailto:ffmpeg-devel-bounces@ffmpeg.org] On Behalf Of > > Steven Liu > > Sent: Wednesday, May 15, 2019 10:38 AM > > To: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org> > > Cc: Xuewei Meng <xwmeng96@gmail.com> > > Subject: Re: [FFmpeg-devel] [PATCH v2] Add multiple padding method in dnn > > native > > > > Xuewei Meng <xwmeng96@gmail.com> 于2019年5月11日周六 上午11:11 > > 写道： > > > > > > --- > > > libavfilter/dnn_backend_native.c | 52 ++++++++++++++++++++++++-------- > > > libavfilter/dnn_backend_native.h | 3 ++ > > > 2 files changed, 43 insertions(+), 12 deletions(-) > > @xuewei, we still need to mention the impact of sr filter, and explain why > same_clamp_to_edge is needed. > > There are three padding methods in this patch, VALID, SAME and SAME_CLAMP_TO_EDGE. The 'VALID' and 'SAME' options are tensorflow supported padding methods. And the third one, 'SAME_CLAMP_TO_EDGE', is suggested by sr filter. As this method can keep the output with the same size as original input, and gives a slight better result as mentioned by Pedro Arthur. So we keep this option in dnn native mode. > > > > > > diff --git a/libavfilter/dnn_backend_native.c > > b/libavfilter/dnn_backend_native.c > > > index 06fbdf368b..171a756385 100644 > > > --- a/libavfilter/dnn_backend_native.c > > > +++ b/libavfilter/dnn_backend_native.c > > > @@ -61,6 +61,12 @@ static DNNReturnType set_input_output_native(void > > *model, DNNInputData *input, c > > > return DNN_ERROR; > > > } > > > cur_channels = conv_params->output_num; > > > + > > > + if(conv_params->padding_method == VALID){ > > > + int pad_size = conv_params->kernel_size - 1; > > > + cur_height -= pad_size; > > > + cur_width -= pad_size; > > > + } > > > break; > > > case DEPTH_TO_SPACE: > > > depth_to_space_params = (DepthToSpaceParams > > *)network->layers[layer].params; > > > @@ -77,6 +83,10 @@ static DNNReturnType set_input_output_native(void > > *model, DNNInputData *input, c > > > if (network->layers[layer].output){ > > > av_freep(&network->layers[layer].output); > > > } > > > + > > > + if(cur_height <= 0 || cur_width <= 0) > > > + return DNN_ERROR; > > > + > > > network->layers[layer].output = av_malloc(cur_height * > > cur_width * cur_channels * sizeof(float)); > > > if (!network->layers[layer].output){ > > > return DNN_ERROR; > > > @@ -154,13 +164,14 @@ DNNModel *ff_dnn_load_model_native(const > > char *model_filename) > > > ff_dnn_free_model_native(&model); > > > return NULL; > > > } > > > + conv_params->padding_method = > > (int32_t)avio_rl32(model_file_context); > > > conv_params->activation = > > (int32_t)avio_rl32(model_file_context); > > > conv_params->input_num = > > (int32_t)avio_rl32(model_file_context); > > > conv_params->output_num = > > (int32_t)avio_rl32(model_file_context); > > > conv_params->kernel_size = > > (int32_t)avio_rl32(model_file_context); > > > kernel_size = conv_params->input_num * > > conv_params->output_num * > > > conv_params->kernel_size * > > conv_params->kernel_size; > > > - dnn_size += 16 + (kernel_size + conv_params->output_num > > << 2); > > > + dnn_size += 20 + (kernel_size + conv_params->output_num > > << 2); > > > if (dnn_size > file_size || conv_params->input_num <= 0 || > > > conv_params->output_num <= 0 || > > conv_params->kernel_size <= 0){ > > > avio_closep(&model_file_context); > > > @@ -218,23 +229,35 @@ DNNModel *ff_dnn_load_model_native(const > > char *model_filename) > > > > > > static void convolve(const float *input, float *output, const > > ConvolutionalParams *conv_params, int width, int height) > > > { > > > - int y, x, n_filter, ch, kernel_y, kernel_x; > > > int radius = conv_params->kernel_size >> 1; > > > int src_linesize = width * conv_params->input_num; > > > int filter_linesize = conv_params->kernel_size * > > conv_params->input_num; > > > int filter_size = conv_params->kernel_size * filter_linesize; > > > + int pad_size = (conv_params->padding_method == VALID) ? > > (conv_params->kernel_size - 1) / 2 : 0; > > > > > > - for (y = 0; y < height; ++y){ > > > - for (x = 0; x < width; ++x){ > > > - for (n_filter = 0; n_filter < conv_params->output_num; > > ++n_filter){ > > > + for (int y = pad_size; y < height - pad_size; ++y){ > > > + for (int x = pad_size; x < width - pad_size; ++x){ > > > + for (int n_filter = 0; n_filter < conv_params->output_num; > > ++n_filter){ > > > output[n_filter] = conv_params->biases[n_filter]; > > > - for (ch = 0; ch < conv_params->input_num; ++ch){ > > > - for (kernel_y = 0; kernel_y < > > conv_params->kernel_size; ++kernel_y){ > > > - for (kernel_x = 0; kernel_x < > > conv_params->kernel_size; ++kernel_x){ > > > - output[n_filter] += > > input[CLAMP_TO_EDGE(y + kernel_y - radius, height) * src_linesize + > > > - > > CLAMP_TO_EDGE(x + kernel_x - radius, width) * conv_params->input_num + > ch] > > * > > > - > > conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize + > > > - > > kernel_x * conv_params->input_num + ch]; > > > + > > > + for (int ch = 0; ch < conv_params->input_num; ++ch){ > > > + for (int kernel_y = 0; kernel_y < > > conv_params->kernel_size; ++kernel_y){ > > > + for (int kernel_x = 0; kernel_x < > > conv_params->kernel_size; ++kernel_x){ > > > + float input_pel; > > > + if(conv_params->padding_method == > > SAME_CLAMP_TO_EDGE){ > > > + int y_pos = CLAMP_TO_EDGE(y + > > kernel_y - radius, height); > > > + int x_pos = CLAMP_TO_EDGE(x + > > kernel_x - radius, width); > > > + input_pel = input[y_pos * > > src_linesize + x_pos * conv_params->input_num + ch]; > > > + }else{ > > > + int y_pos = y + kernel_y - radius; > > > + int x_pos = x + kernel_x - radius; > > > + input_pel = (x_pos < 0 || x_pos >= > > width || y_pos < 0 || y_pos >= height) ? 0.0 : > > > + input[y_pos > > * src_linesize + x_pos * conv_params->input_num + ch]; > > > + } > > > + > > > + > > > + output[n_filter] += input_pel * > > conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize + > > > + > > kernel_x * conv_params->input_num + ch]; > > > } > > > } > > > } > > > @@ -305,6 +328,11 @@ DNNReturnType > > ff_dnn_execute_model_native(const DNNModel *model, DNNData *output > > > conv_params = (ConvolutionalParams > > *)network->layers[layer].params; > > > convolve(network->layers[layer - 1].output, > > network->layers[layer].output, conv_params, cur_width, cur_height); > > > cur_channels = conv_params->output_num; > > > + if(conv_params->padding_method == VALID){ > > > + int pad_size = conv_params->kernel_size - 1; > > > + cur_height -= pad_size; > > > + cur_width -= pad_size; > > > + } > > > break; > > > case DEPTH_TO_SPACE: > > > depth_to_space_params = (DepthToSpaceParams > > *)network->layers[layer].params; > > > diff --git a/libavfilter/dnn_backend_native.h > > b/libavfilter/dnn_backend_native.h > > > index e13a68a168..d70cd16387 100644 > > > --- a/libavfilter/dnn_backend_native.h > > > +++ b/libavfilter/dnn_backend_native.h > > > @@ -34,6 +34,8 @@ typedef enum {INPUT, CONV, DEPTH_TO_SPACE} > > DNNLayerType; > > > > > > typedef enum {RELU, TANH, SIGMOID} DNNActivationFunc; > > > > > > +typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} > > DNNConvPaddingParam; > > > + > > > typedef struct Layer{ > > > DNNLayerType type; > > > float *output; > > > @@ -43,6 +45,7 @@ typedef struct Layer{ > > > typedef struct ConvolutionalParams{ > > > int32_t input_num, output_num, kernel_size; > > > DNNActivationFunc activation; > > > + DNNConvPaddingParam padding_method; > > > float *kernel; > > > float *biases; > > > } ConvolutionalParams; > > > -- > > > 2.17.1 > > > > > > _______________________________________________ > > > ffmpeg-devel mailing list > > > ffmpeg-devel@ffmpeg.org > > > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > > > > > To unsubscribe, visit link above, or email > > > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". > > > > > > The https://github.com/HighVoltageRocknRoll/sr has loss of > > communication，and the project > > https://github.com/HighVoltageRocknRoll/sr has no maintainer now, so i > > think the pull request cannot be merge. > > 1. So i recommend Xuewei fork the project to his github, and merge the > > pr to his fork project, and modify the sr document of > > libavfilter/vf_sr.c. makes GSoC derain mentor project continue. > > I prefer this one. > > > > > 2. If 1st way cannot be acceptable, Xuewei should duplicate DNN code > > for the derain. > > > > Comments welcome. > > > > Thanks > > > > Steven > > _______________________________________________ > > ffmpeg-devel mailing list > > ffmpeg-devel@ffmpeg.org > > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > > > To unsubscribe, visit link above, or email > > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". >
> > > From: Xuewei Meng [mailto:xwmeng96@gmail.com] > Sent: Wednesday, May 15, 2019 4:41 PM > To: Guo, Yejun <yejun.guo@intel.com> > Cc: FFmpeg development discussions and patches <ffmpeg-devel@ffmpeg.org> > Subject: Re: [FFmpeg-devel] [PATCH v2] Add multiple padding method in dnn > native > > > > Guo, Yejun <yejun.guo@intel.com> 于2019年5月15日周三 下午2:21写道： > > > > -----Original Message----- > > From: ffmpeg-devel [mailto:ffmpeg-devel-bounces@ffmpeg.org] On Behalf Of > > Steven Liu > > Sent: Wednesday, May 15, 2019 10:38 AM > > To: FFmpeg development discussions and patches > <ffmpeg-devel@ffmpeg.org> > > Cc: Xuewei Meng <xwmeng96@gmail.com> > > Subject: Re: [FFmpeg-devel] [PATCH v2] Add multiple padding method in dnn > > native > > > > Xuewei Meng <xwmeng96@gmail.com> 于2019年5月11日周六 上午 > 11:11 > > 写道： > > > > > > --- > > > libavfilter/dnn_backend_native.c | 52 ++++++++++++++++++++++++-------- > > > libavfilter/dnn_backend_native.h | 3 ++ > > > 2 files changed, 43 insertions(+), 12 deletions(-) > > @xuewei, we still need to mention the impact of sr filter, and explain why > same_clamp_to_edge is needed. > There are three padding methods in this patch, VALID, SAME and > SAME_CLAMP_TO_EDGE. The 'VALID' and 'SAME' options are tensorflow > supported padding methods. And the third one, 'SAME_CLAMP_TO_EDGE', is > suggested by sr filter. As this method can keep the output with the same size > as original input, and gives a slight better result as mentioned by Pedro Arthur. > So we keep this option in dnn native mode. nice, please add them into commit log. And also the impact to sr filter. > > > > > > > diff --git a/libavfilter/dnn_backend_native.c > > b/libavfilter/dnn_backend_native.c > > > index 06fbdf368b..171a756385 100644 > > > --- a/libavfilter/dnn_backend_native.c > > > +++ b/libavfilter/dnn_backend_native.c > > > @@ -61,6 +61,12 @@ static DNNReturnType > set_input_output_native(void > > *model, DNNInputData *input, c > > > return DNN_ERROR; > > > } > > > cur_channels = conv_params->output_num; > > > + > > > + if(conv_params->padding_method == VALID){ > > > + int pad_size = conv_params->kernel_size - 1; > > > + cur_height -= pad_size; > > > + cur_width -= pad_size; > > > + } > > > break; > > > case DEPTH_TO_SPACE: > > > depth_to_space_params = (DepthToSpaceParams > > *)network->layers[layer].params; > > > @@ -77,6 +83,10 @@ static DNNReturnType > set_input_output_native(void > > *model, DNNInputData *input, c > > > if (network->layers[layer].output){ > > > av_freep(&network->layers[layer].output); > > > } > > > + > > > + if(cur_height <= 0 || cur_width <= 0) > > > + return DNN_ERROR; > > > + > > > network->layers[layer].output = av_malloc(cur_height * > > cur_width * cur_channels * sizeof(float)); > > > if (!network->layers[layer].output){ > > > return DNN_ERROR; > > > @@ -154,13 +164,14 @@ DNNModel *ff_dnn_load_model_native(const > > char *model_filename) > > > ff_dnn_free_model_native(&model); > > > return NULL; > > > } > > > + conv_params->padding_method = > > (int32_t)avio_rl32(model_file_context); > > > conv_params->activation = > > (int32_t)avio_rl32(model_file_context); > > > conv_params->input_num = > > (int32_t)avio_rl32(model_file_context); > > > conv_params->output_num = > > (int32_t)avio_rl32(model_file_context); > > > conv_params->kernel_size = > > (int32_t)avio_rl32(model_file_context); > > > kernel_size = conv_params->input_num * > > conv_params->output_num * > > > conv_params->kernel_size * > > conv_params->kernel_size; > > > - dnn_size += 16 + (kernel_size + conv_params->output_num > > << 2); > > > + dnn_size += 20 + (kernel_size + > conv_params->output_num > > << 2); > > > if (dnn_size > file_size || conv_params->input_num <= 0 > || > > > conv_params->output_num <= 0 || > > conv_params->kernel_size <= 0){ > > > avio_closep(&model_file_context); > > > @@ -218,23 +229,35 @@ DNNModel *ff_dnn_load_model_native(const > > char *model_filename) > > > > > > static void convolve(const float *input, float *output, const > > ConvolutionalParams *conv_params, int width, int height) > > > { > > > - int y, x, n_filter, ch, kernel_y, kernel_x; > > > int radius = conv_params->kernel_size >> 1; > > > int src_linesize = width * conv_params->input_num; > > > int filter_linesize = conv_params->kernel_size * > > conv_params->input_num; > > > int filter_size = conv_params->kernel_size * filter_linesize; > > > + int pad_size = (conv_params->padding_method == VALID) ? > > (conv_params->kernel_size - 1) / 2 : 0; > > > > > > - for (y = 0; y < height; ++y){ > > > - for (x = 0; x < width; ++x){ > > > - for (n_filter = 0; n_filter < conv_params->output_num; > > ++n_filter){ > > > + for (int y = pad_size; y < height - pad_size; ++y){ > > > + for (int x = pad_size; x < width - pad_size; ++x){ > > > + for (int n_filter = 0; n_filter < conv_params->output_num; > > ++n_filter){ > > > output[n_filter] = conv_params->biases[n_filter]; > > > - for (ch = 0; ch < conv_params->input_num; ++ch){ > > > - for (kernel_y = 0; kernel_y < > > conv_params->kernel_size; ++kernel_y){ > > > - for (kernel_x = 0; kernel_x < > > conv_params->kernel_size; ++kernel_x){ > > > - output[n_filter] += > > input[CLAMP_TO_EDGE(y + kernel_y - radius, height) * src_linesize + > > > - > > CLAMP_TO_EDGE(x + kernel_x - radius, width) * conv_params->input_num + > ch] > > * > > > - > > conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize + > > > - > > kernel_x * conv_params->input_num + ch]; > > > + > > > + for (int ch = 0; ch < conv_params->input_num; ++ch){ > > > + for (int kernel_y = 0; kernel_y < > > conv_params->kernel_size; ++kernel_y){ > > > + for (int kernel_x = 0; kernel_x < > > conv_params->kernel_size; ++kernel_x){ > > > + float input_pel; > > > + if(conv_params->padding_method == > > SAME_CLAMP_TO_EDGE){ > > > + int y_pos = CLAMP_TO_EDGE(y > + > > kernel_y - radius, height); > > > + int x_pos = CLAMP_TO_EDGE(x > + > > kernel_x - radius, width); > > > + input_pel = input[y_pos * > > src_linesize + x_pos * conv_params->input_num + ch]; > > > + }else{ > > > + int y_pos = y + kernel_y - radius; > > > + int x_pos = x + kernel_x - radius; > > > + input_pel = (x_pos < 0 || > x_pos >= > > width || y_pos < 0 || y_pos >= height) ? 0.0 : > > > > + input[y_p > os > > * src_linesize + x_pos * conv_params->input_num + ch]; > > > + } > > > + > > > + > > > + output[n_filter] += input_pel * > > conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize + > > > + > > kernel_x * conv_params->input_num + ch]; > > > } > > > } > > > } > > > @@ -305,6 +328,11 @@ DNNReturnType > > ff_dnn_execute_model_native(const DNNModel *model, DNNData *output > > > conv_params = (ConvolutionalParams > > *)network->layers[layer].params; > > > convolve(network->layers[layer - 1].output, > > network->layers[layer].output, conv_params, cur_width, cur_height); > > > cur_channels = conv_params->output_num; > > > + if(conv_params->padding_method == VALID){ > > > + int pad_size = conv_params->kernel_size - 1; > > > + cur_height -= pad_size; > > > + cur_width -= pad_size; > > > + } > > > break; > > > case DEPTH_TO_SPACE: > > > depth_to_space_params = (DepthToSpaceParams > > *)network->layers[layer].params; > > > diff --git a/libavfilter/dnn_backend_native.h > > b/libavfilter/dnn_backend_native.h > > > index e13a68a168..d70cd16387 100644 > > > --- a/libavfilter/dnn_backend_native.h > > > +++ b/libavfilter/dnn_backend_native.h > > > @@ -34,6 +34,8 @@ typedef enum {INPUT, CONV, DEPTH_TO_SPACE} > > DNNLayerType; > > > > > > typedef enum {RELU, TANH, SIGMOID} DNNActivationFunc; > > > > > > +typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} > > DNNConvPaddingParam; > > > + > > > typedef struct Layer{ > > > DNNLayerType type; > > > float *output; > > > @@ -43,6 +45,7 @@ typedef struct Layer{ > > > typedef struct ConvolutionalParams{ > > > int32_t input_num, output_num, kernel_size; > > > DNNActivationFunc activation; > > > + DNNConvPaddingParam padding_method; > > > float *kernel; > > > float *biases; > > > } ConvolutionalParams; > > > -- > > > 2.17.1 > > > > > > _______________________________________________ > > > ffmpeg-devel mailing list > > > ffmpeg-devel@ffmpeg.org > > > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > > > > > To unsubscribe, visit link above, or email > > > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". > > > > > > The https://github.com/HighVoltageRocknRoll/sr has loss of > > communication，and the project > > https://github.com/HighVoltageRocknRoll/sr has no maintainer now, so i > > think the pull request cannot be merge. > > 1. So i recommend Xuewei fork the project to his github, and merge the > > pr to his fork project, and modify the sr document of > > libavfilter/vf_sr.c. makes GSoC derain mentor project continue. > > I prefer this one. > > > > > 2. If 1st way cannot be acceptable, Xuewei should duplicate DNN code > > for the derain. > > > > Comments welcome. > > > > Thanks > > > > Steven > > _______________________________________________ > > ffmpeg-devel mailing list > > ffmpeg-devel@ffmpeg.org > > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > > > To unsubscribe, visit link above, or email > > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
Em qua, 15 de mai de 2019 às 04:44, Steven Liu <lingjiujianke@gmail.com> escreveu: > > Xuewei Meng <xwmeng96@gmail.com> 于2019年5月11日周六 上午11:11写道： > > > > --- > > libavfilter/dnn_backend_native.c | 52 ++++++++++++++++++++++++-------- > > libavfilter/dnn_backend_native.h | 3 ++ > > 2 files changed, 43 insertions(+), 12 deletions(-) > > > > diff --git a/libavfilter/dnn_backend_native.c b/libavfilter/dnn_backend_native.c > > index 06fbdf368b..171a756385 100644 > > --- a/libavfilter/dnn_backend_native.c > > +++ b/libavfilter/dnn_backend_native.c > > @@ -61,6 +61,12 @@ static DNNReturnType set_input_output_native(void *model, DNNInputData *input, c > > return DNN_ERROR; > > } > > cur_channels = conv_params->output_num; > > + > > + if(conv_params->padding_method == VALID){ > > + int pad_size = conv_params->kernel_size - 1; > > + cur_height -= pad_size; > > + cur_width -= pad_size; > > + } > > break; > > case DEPTH_TO_SPACE: > > depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params; > > @@ -77,6 +83,10 @@ static DNNReturnType set_input_output_native(void *model, DNNInputData *input, c > > if (network->layers[layer].output){ > > av_freep(&network->layers[layer].output); > > } > > + > > + if(cur_height <= 0 || cur_width <= 0) > > + return DNN_ERROR; > > + > > network->layers[layer].output = av_malloc(cur_height * cur_width * cur_channels * sizeof(float)); > > if (!network->layers[layer].output){ > > return DNN_ERROR; > > @@ -154,13 +164,14 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) > > ff_dnn_free_model_native(&model); > > return NULL; > > } > > + conv_params->padding_method = (int32_t)avio_rl32(model_file_context); > > conv_params->activation = (int32_t)avio_rl32(model_file_context); > > conv_params->input_num = (int32_t)avio_rl32(model_file_context); > > conv_params->output_num = (int32_t)avio_rl32(model_file_context); > > conv_params->kernel_size = (int32_t)avio_rl32(model_file_context); > > kernel_size = conv_params->input_num * conv_params->output_num * > > conv_params->kernel_size * conv_params->kernel_size; > > - dnn_size += 16 + (kernel_size + conv_params->output_num << 2); > > + dnn_size += 20 + (kernel_size + conv_params->output_num << 2); > > if (dnn_size > file_size || conv_params->input_num <= 0 || > > conv_params->output_num <= 0 || conv_params->kernel_size <= 0){ > > avio_closep(&model_file_context); > > @@ -218,23 +229,35 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) > > > > static void convolve(const float *input, float *output, const ConvolutionalParams *conv_params, int width, int height) > > { > > - int y, x, n_filter, ch, kernel_y, kernel_x; > > int radius = conv_params->kernel_size >> 1; > > int src_linesize = width * conv_params->input_num; > > int filter_linesize = conv_params->kernel_size * conv_params->input_num; > > int filter_size = conv_params->kernel_size * filter_linesize; > > + int pad_size = (conv_params->padding_method == VALID) ? (conv_params->kernel_size - 1) / 2 : 0; > > > > - for (y = 0; y < height; ++y){ > > - for (x = 0; x < width; ++x){ > > - for (n_filter = 0; n_filter < conv_params->output_num; ++n_filter){ > > + for (int y = pad_size; y < height - pad_size; ++y){ > > + for (int x = pad_size; x < width - pad_size; ++x){ > > + for (int n_filter = 0; n_filter < conv_params->output_num; ++n_filter){ > > output[n_filter] = conv_params->biases[n_filter]; > > - for (ch = 0; ch < conv_params->input_num; ++ch){ > > - for (kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y){ > > - for (kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x){ > > - output[n_filter] += input[CLAMP_TO_EDGE(y + kernel_y - radius, height) * src_linesize + > > - CLAMP_TO_EDGE(x + kernel_x - radius, width) * conv_params->input_num + ch] * > > - conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize + > > - kernel_x * conv_params->input_num + ch]; > > + > > + for (int ch = 0; ch < conv_params->input_num; ++ch){ > > + for (int kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y){ > > + for (int kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x){ > > + float input_pel; > > + if(conv_params->padding_method == SAME_CLAMP_TO_EDGE){ > > + int y_pos = CLAMP_TO_EDGE(y + kernel_y - radius, height); > > + int x_pos = CLAMP_TO_EDGE(x + kernel_x - radius, width); > > + input_pel = input[y_pos * src_linesize + x_pos * conv_params->input_num + ch]; > > + }else{ > > + int y_pos = y + kernel_y - radius; > > + int x_pos = x + kernel_x - radius; > > + input_pel = (x_pos < 0 || x_pos >= width || y_pos < 0 || y_pos >= height) ? 0.0 : > > + input[y_pos * src_linesize + x_pos * conv_params->input_num + ch]; > > + } > > + > > + > > + output[n_filter] += input_pel * conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize + > > + kernel_x * conv_params->input_num + ch]; > > } > > } > > } > > @@ -305,6 +328,11 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *output > > conv_params = (ConvolutionalParams *)network->layers[layer].params; > > convolve(network->layers[layer - 1].output, network->layers[layer].output, conv_params, cur_width, cur_height); > > cur_channels = conv_params->output_num; > > + if(conv_params->padding_method == VALID){ > > + int pad_size = conv_params->kernel_size - 1; > > + cur_height -= pad_size; > > + cur_width -= pad_size; > > + } > > break; > > case DEPTH_TO_SPACE: > > depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params; > > diff --git a/libavfilter/dnn_backend_native.h b/libavfilter/dnn_backend_native.h > > index e13a68a168..d70cd16387 100644 > > --- a/libavfilter/dnn_backend_native.h > > +++ b/libavfilter/dnn_backend_native.h > > @@ -34,6 +34,8 @@ typedef enum {INPUT, CONV, DEPTH_TO_SPACE} DNNLayerType; > > > > typedef enum {RELU, TANH, SIGMOID} DNNActivationFunc; > > > > +typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} DNNConvPaddingParam; > > + > > typedef struct Layer{ > > DNNLayerType type; > > float *output; > > @@ -43,6 +45,7 @@ typedef struct Layer{ > > typedef struct ConvolutionalParams{ > > int32_t input_num, output_num, kernel_size; > > DNNActivationFunc activation; > > + DNNConvPaddingParam padding_method; > > float *kernel; > > float *biases; > > } ConvolutionalParams; > > -- > > 2.17.1 > > > > _______________________________________________ > > ffmpeg-devel mailing list > > ffmpeg-devel@ffmpeg.org > > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > > > To unsubscribe, visit link above, or email > > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe". > > > The https://github.com/HighVoltageRocknRoll/sr has loss of > communication，and the project > https://github.com/HighVoltageRocknRoll/sr has no maintainer now, so i > think the pull request cannot be merge. > 1. So i recommend Xuewei fork the project to his github, and merge the > pr to his fork project, and modify the sr document of > libavfilter/vf_sr.c. makes GSoC derain mentor project continue. > I see no problem with 1, anyway he already had to make a fork to create the pr, he can continue developing it there. > 2. If 1st way cannot be acceptable, Xuewei should duplicate DNN code > for the derain. > > Comments welcome. > > Thanks > > Steven > _______________________________________________ > ffmpeg-devel mailing list > ffmpeg-devel@ffmpeg.org > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel > > To unsubscribe, visit link above, or email > ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
diff --git a/libavfilter/dnn_backend_native.c b/libavfilter/dnn_backend_native.c index 06fbdf368b..171a756385 100644 --- a/libavfilter/dnn_backend_native.c +++ b/libavfilter/dnn_backend_native.c @@ -61,6 +61,12 @@ static DNNReturnType set_input_output_native(void *model, DNNInputData *input, c return DNN_ERROR; } cur_channels = conv_params->output_num; + + if(conv_params->padding_method == VALID){ + int pad_size = conv_params->kernel_size - 1; + cur_height -= pad_size; + cur_width -= pad_size; + } break; case DEPTH_TO_SPACE: depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params; @@ -77,6 +83,10 @@ static DNNReturnType set_input_output_native(void *model, DNNInputData *input, c if (network->layers[layer].output){ av_freep(&network->layers[layer].output); } + + if(cur_height <= 0 || cur_width <= 0) + return DNN_ERROR; + network->layers[layer].output = av_malloc(cur_height * cur_width * cur_channels * sizeof(float)); if (!network->layers[layer].output){ return DNN_ERROR; @@ -154,13 +164,14 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) ff_dnn_free_model_native(&model); return NULL; } + conv_params->padding_method = (int32_t)avio_rl32(model_file_context); conv_params->activation = (int32_t)avio_rl32(model_file_context); conv_params->input_num = (int32_t)avio_rl32(model_file_context); conv_params->output_num = (int32_t)avio_rl32(model_file_context); conv_params->kernel_size = (int32_t)avio_rl32(model_file_context); kernel_size = conv_params->input_num * conv_params->output_num * conv_params->kernel_size * conv_params->kernel_size; - dnn_size += 16 + (kernel_size + conv_params->output_num << 2); + dnn_size += 20 + (kernel_size + conv_params->output_num << 2); if (dnn_size > file_size || conv_params->input_num <= 0 || conv_params->output_num <= 0 || conv_params->kernel_size <= 0){ avio_closep(&model_file_context); @@ -218,23 +229,35 @@ DNNModel *ff_dnn_load_model_native(const char *model_filename) static void convolve(const float *input, float *output, const ConvolutionalParams *conv_params, int width, int height) { - int y, x, n_filter, ch, kernel_y, kernel_x; int radius = conv_params->kernel_size >> 1; int src_linesize = width * conv_params->input_num; int filter_linesize = conv_params->kernel_size * conv_params->input_num; int filter_size = conv_params->kernel_size * filter_linesize; + int pad_size = (conv_params->padding_method == VALID) ? (conv_params->kernel_size - 1) / 2 : 0; - for (y = 0; y < height; ++y){ - for (x = 0; x < width; ++x){ - for (n_filter = 0; n_filter < conv_params->output_num; ++n_filter){ + for (int y = pad_size; y < height - pad_size; ++y){ + for (int x = pad_size; x < width - pad_size; ++x){ + for (int n_filter = 0; n_filter < conv_params->output_num; ++n_filter){ output[n_filter] = conv_params->biases[n_filter]; - for (ch = 0; ch < conv_params->input_num; ++ch){ - for (kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y){ - for (kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x){ - output[n_filter] += input[CLAMP_TO_EDGE(y + kernel_y - radius, height) * src_linesize + - CLAMP_TO_EDGE(x + kernel_x - radius, width) * conv_params->input_num + ch] * - conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize + - kernel_x * conv_params->input_num + ch]; + + for (int ch = 0; ch < conv_params->input_num; ++ch){ + for (int kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y){ + for (int kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x){ + float input_pel; + if(conv_params->padding_method == SAME_CLAMP_TO_EDGE){ + int y_pos = CLAMP_TO_EDGE(y + kernel_y - radius, height); + int x_pos = CLAMP_TO_EDGE(x + kernel_x - radius, width); + input_pel = input[y_pos * src_linesize + x_pos * conv_params->input_num + ch]; + }else{ + int y_pos = y + kernel_y - radius; + int x_pos = x + kernel_x - radius; + input_pel = (x_pos < 0 || x_pos >= width || y_pos < 0 || y_pos >= height) ? 0.0 : + input[y_pos * src_linesize + x_pos * conv_params->input_num + ch]; + } + + + output[n_filter] += input_pel * conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize + + kernel_x * conv_params->input_num + ch]; } } } @@ -305,6 +328,11 @@ DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *output conv_params = (ConvolutionalParams *)network->layers[layer].params; convolve(network->layers[layer - 1].output, network->layers[layer].output, conv_params, cur_width, cur_height); cur_channels = conv_params->output_num; + if(conv_params->padding_method == VALID){ + int pad_size = conv_params->kernel_size - 1; + cur_height -= pad_size; + cur_width -= pad_size; + } break; case DEPTH_TO_SPACE: depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params; diff --git a/libavfilter/dnn_backend_native.h b/libavfilter/dnn_backend_native.h index e13a68a168..d70cd16387 100644 --- a/libavfilter/dnn_backend_native.h +++ b/libavfilter/dnn_backend_native.h @@ -34,6 +34,8 @@ typedef enum {INPUT, CONV, DEPTH_TO_SPACE} DNNLayerType; typedef enum {RELU, TANH, SIGMOID} DNNActivationFunc; +typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} DNNConvPaddingParam; + typedef struct Layer{ DNNLayerType type; float *output; @@ -43,6 +45,7 @@ typedef struct Layer{ typedef struct ConvolutionalParams{ int32_t input_num, output_num, kernel_size; DNNActivationFunc activation; + DNNConvPaddingParam padding_method; float *kernel; float *biases; } ConvolutionalParams;