[FFmpeg-devel,v3,2/3] libavfilter/vf_cover_rect: support for cover image with more pixel format and different width and height

Submitted by lance.lmwang@gmail.com on June 11, 2019, 5:38 a.m.

Details

Message ID 20190611053844.21321-2-lance.lmwang@gmail.com
State New
Headers show

Commit Message

lance.lmwang@gmail.com June 11, 2019, 5:38 a.m.
From: Limin Wang <lance.lmwang@gmail.com>

Signed-off-by: Limin Wang <lance.lmwang@gmail.com>
---
 libavfilter/vf_cover_rect.c | 49 +++++++++++++++++++++++++++----------
 1 file changed, 36 insertions(+), 13 deletions(-)

Comments

Michael Niedermayer June 11, 2019, 8:51 p.m.
On Tue, Jun 11, 2019 at 01:38:43PM +0800, lance.lmwang@gmail.com wrote:
> From: Limin Wang <lance.lmwang@gmail.com>
> 
> Signed-off-by: Limin Wang <lance.lmwang@gmail.com>
> ---
>  libavfilter/vf_cover_rect.c | 49 +++++++++++++++++++++++++++----------
>  1 file changed, 36 insertions(+), 13 deletions(-)
> 
> diff --git a/libavfilter/vf_cover_rect.c b/libavfilter/vf_cover_rect.c
> index 898debf09d..22389b79d2 100644
> --- a/libavfilter/vf_cover_rect.c
> +++ b/libavfilter/vf_cover_rect.c
> @@ -28,6 +28,7 @@
>  #include "internal.h"
>  
>  #include "lavfutils.h"
> +#include "lswsutils.h"
>  
>  enum mode {
>      MODE_COVER,
> @@ -40,6 +41,7 @@ typedef struct CoverContext {
>      int mode;
>      char *cover_filename;
>      AVFrame *cover_frame;
> +    AVFrame *match_frame;
>      int width, height;
>  } CoverContext;
>  
> @@ -71,21 +73,21 @@ static int config_input(AVFilterLink *inlink)
>      return 0;
>  }
>  
> -static void cover_rect(CoverContext *cover, AVFrame *in, int offx, int offy)
> +static void cover_rect(AVFrame *cover_frame, AVFrame *in, int offx, int offy)
>  {
>      int x, y, p;
>  
>      for (p = 0; p < 3; p++) {
>          uint8_t *data = in->data[p] + (offx>>!!p) + (offy>>!!p) * in->linesize[p];
> -        const uint8_t *src = cover->cover_frame->data[p];
> -        int w = AV_CEIL_RSHIFT(cover->cover_frame->width , !!p);
> -        int h = AV_CEIL_RSHIFT(cover->cover_frame->height, !!p);
> +        const uint8_t *src = cover_frame->data[p];
> +        int w = AV_CEIL_RSHIFT(cover_frame->width , !!p);
> +        int h = AV_CEIL_RSHIFT(cover_frame->height, !!p);
>          for (y = 0; y < h; y++) {
>              for (x = 0; x < w; x++) {
>                  data[x] = src[x];
>              }
>              data += in->linesize[p];
> -            src += cover->cover_frame->linesize[p];
> +            src += cover_frame->linesize[p];
>          }
>      }
>  }
> @@ -138,7 +140,10 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in)
>      CoverContext *cover = ctx->priv;
>      AVDictionaryEntry *ex, *ey, *ew, *eh;
>      int x = -1, y = -1, w = -1, h = -1;
> +    enum AVPixelFormat in_format;
>      char *xendptr = NULL, *yendptr = NULL, *wendptr = NULL, *hendptr = NULL;
> +    AVFrame *cover_frame = NULL;
> +    int ret;
>  
>      ex = av_dict_get(in->metadata, "lavfi.rect.x", NULL, AV_DICT_MATCH_CASE);
>      ey = av_dict_get(in->metadata, "lavfi.rect.y", NULL, AV_DICT_MATCH_CASE);
> @@ -167,13 +172,30 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in)
>      }
>      w = FFMIN(w, in->width  - x);
>      h = FFMIN(h, in->height - y);
> +    in_format = in->format;
>  
>      if (w > in->width || h > in->height || w <= 0 || h <= 0)
>          return AVERROR(EINVAL);
>  
>      if (cover->cover_frame) {
> -        if (w != cover->cover_frame->width || h != cover->cover_frame->height)
> -            return AVERROR(EINVAL);
> +        if (w != cover->cover_frame->width || h != cover->cover_frame->height ||
> +                in_format != cover->cover_frame->format) {
> +            if (!cover->match_frame && !(cover->match_frame = av_frame_alloc()))
> +                return AVERROR(EINVAL);
> +
> +            if ((ret = ff_scale_image(cover->match_frame->data, cover->match_frame->linesize,
> +                            w, h, in_format, cover->cover_frame->data, cover->cover_frame->linesize,
> +                            cover->cover_frame->width, cover->cover_frame->height,
> +                            cover->cover_frame->format, ctx)) < 0)
> +                return AVERROR(EINVAL);

This looks like the same static cover image would be converted again for each input frame
which if iam not misreading this would be doing the exact same operation over and over
again, thats same wastefull

[....]
lance.lmwang@gmail.com June 12, 2019, 1:25 a.m.
On Wed, Jun 12, 2019 at 4:51 AM Michael Niedermayer <michael@niedermayer.cc>
wrote:

> On Tue, Jun 11, 2019 at 01:38:43PM +0800, lance.lmwang@gmail.com wrote:
> > From: Limin Wang <lance.lmwang@gmail.com>
> >
> > Signed-off-by: Limin Wang <lance.lmwang@gmail.com>
> > ---
> >  libavfilter/vf_cover_rect.c | 49 +++++++++++++++++++++++++++----------
> >  1 file changed, 36 insertions(+), 13 deletions(-)
> >
> > diff --git a/libavfilter/vf_cover_rect.c b/libavfilter/vf_cover_rect.c
> > index 898debf09d..22389b79d2 100644
> > --- a/libavfilter/vf_cover_rect.c
> > +++ b/libavfilter/vf_cover_rect.c
> > @@ -28,6 +28,7 @@
> >  #include "internal.h"
> >
> >  #include "lavfutils.h"
> > +#include "lswsutils.h"
> >
> >  enum mode {
> >      MODE_COVER,
> > @@ -40,6 +41,7 @@ typedef struct CoverContext {
> >      int mode;
> >      char *cover_filename;
> >      AVFrame *cover_frame;
> > +    AVFrame *match_frame;
> >      int width, height;
> >  } CoverContext;
> >
> > @@ -71,21 +73,21 @@ static int config_input(AVFilterLink *inlink)
> >      return 0;
> >  }
> >
> > -static void cover_rect(CoverContext *cover, AVFrame *in, int offx, int
> offy)
> > +static void cover_rect(AVFrame *cover_frame, AVFrame *in, int offx, int
> offy)
> >  {
> >      int x, y, p;
> >
> >      for (p = 0; p < 3; p++) {
> >          uint8_t *data = in->data[p] + (offx>>!!p) + (offy>>!!p) *
> in->linesize[p];
> > -        const uint8_t *src = cover->cover_frame->data[p];
> > -        int w = AV_CEIL_RSHIFT(cover->cover_frame->width , !!p);
> > -        int h = AV_CEIL_RSHIFT(cover->cover_frame->height, !!p);
> > +        const uint8_t *src = cover_frame->data[p];
> > +        int w = AV_CEIL_RSHIFT(cover_frame->width , !!p);
> > +        int h = AV_CEIL_RSHIFT(cover_frame->height, !!p);
> >          for (y = 0; y < h; y++) {
> >              for (x = 0; x < w; x++) {
> >                  data[x] = src[x];
> >              }
> >              data += in->linesize[p];
> > -            src += cover->cover_frame->linesize[p];
> > +            src += cover_frame->linesize[p];
> >          }
> >      }
> >  }
> > @@ -138,7 +140,10 @@ static int filter_frame(AVFilterLink *inlink,
> AVFrame *in)
> >      CoverContext *cover = ctx->priv;
> >      AVDictionaryEntry *ex, *ey, *ew, *eh;
> >      int x = -1, y = -1, w = -1, h = -1;
> > +    enum AVPixelFormat in_format;
> >      char *xendptr = NULL, *yendptr = NULL, *wendptr = NULL, *hendptr =
> NULL;
> > +    AVFrame *cover_frame = NULL;
> > +    int ret;
> >
> >      ex = av_dict_get(in->metadata, "lavfi.rect.x", NULL,
> AV_DICT_MATCH_CASE);
> >      ey = av_dict_get(in->metadata, "lavfi.rect.y", NULL,
> AV_DICT_MATCH_CASE);
> > @@ -167,13 +172,30 @@ static int filter_frame(AVFilterLink *inlink,
> AVFrame *in)
> >      }
> >      w = FFMIN(w, in->width  - x);
> >      h = FFMIN(h, in->height - y);
> > +    in_format = in->format;
> >
> >      if (w > in->width || h > in->height || w <= 0 || h <= 0)
> >          return AVERROR(EINVAL);
> >
> >      if (cover->cover_frame) {
> > -        if (w != cover->cover_frame->width || h !=
> cover->cover_frame->height)
> > -            return AVERROR(EINVAL);
> > +        if (w != cover->cover_frame->width || h !=
> cover->cover_frame->height ||
> > +                in_format != cover->cover_frame->format) {
> > +            if (!cover->match_frame && !(cover->match_frame =
> av_frame_alloc()))
> > +                return AVERROR(EINVAL);
> > +
> > +            if ((ret = ff_scale_image(cover->match_frame->data,
> cover->match_frame->linesize,
> > +                            w, h, in_format, cover->cover_frame->data,
> cover->cover_frame->linesize,
> > +                            cover->cover_frame->width,
> cover->cover_frame->height,
> > +                            cover->cover_frame->format, ctx)) < 0)
> > +                return AVERROR(EINVAL);
>
> This looks like the same static cover image would be converted again for
> each input frame
> which if iam not misreading this would be doing the exact same operation
> over and over
> again, thats same wastefull
>
> I plan to add code t to find more different object with different width
and height, however the cover image is same.
So it's necessary to convert it after get the real find object width and
height.  If you use the same width and height with
current code, it'll do the same function.  After the patch applied, I'll
send with another patch to avoid conversion every time
 if width and height, format  is same.



> [....]
> --
> Michael     GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB
>
> Why not whip the teacher when the pupil misbehaves? -- Diogenes of Sinope
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
Moritz Barsnick June 12, 2019, 9:43 a.m.
On Tue, Jun 11, 2019 at 13:38:43 +0800, lance.lmwang@gmail.com wrote:
> @@ -220,11 +248,6 @@ static av_cold int init(AVFilterContext *ctx)
>                                  &cover->cover_frame->width, &cover->cover_frame->height,
>                                  &cover->cover_frame->format, cover->cover_filename, ctx)) < 0)
>              return ret;
> -
> -        if (cover->cover_frame->format != AV_PIX_FMT_YUV420P && cover->cover_frame->format != AV_PIX_FMT_YUVJ420P) {
> -            av_log(ctx, AV_LOG_ERROR, "cover image is not a YUV420 image\n");
> -            return AVERROR(EINVAL);
> -        }
>      }

Don't you also need to update query_formats()? Just wondering,
untested.

Moritz
lance.lmwang@gmail.com June 12, 2019, 10:52 a.m.
On Wed, Jun 12, 2019 at 5:43 PM Moritz Barsnick <barsnick@gmx.net> wrote:

> On Tue, Jun 11, 2019 at 13:38:43 +0800, lance.lmwang@gmail.com wrote:
> > @@ -220,11 +248,6 @@ static av_cold int init(AVFilterContext *ctx)
> >                                  &cover->cover_frame->width,
> &cover->cover_frame->height,
> >                                  &cover->cover_frame->format,
> cover->cover_filename, ctx)) < 0)
> >              return ret;
> > -
> > -        if (cover->cover_frame->format != AV_PIX_FMT_YUV420P &&
> cover->cover_frame->format != AV_PIX_FMT_YUVJ420P) {
> > -            av_log(ctx, AV_LOG_ERROR, "cover image is not a YUV420
> image\n");
> > -            return AVERROR(EINVAL);
> > -        }
> >      }
>
> Don't you also need to update query_formats()? Just wondering,
> untested.
>
> It's image format only, the patch will convert it to same format as input
frame.  I have tested with the new patch and haven't see any issue.




> Moritz
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
lance.lmwang@gmail.com June 12, 2019, 10:54 a.m.
On Wed, Jun 12, 2019 at 4:51 AM Michael Niedermayer <michael@niedermayer.cc>
wrote:

> On Tue, Jun 11, 2019 at 01:38:43PM +0800, lance.lmwang@gmail.com wrote:
> > From: Limin Wang <lance.lmwang@gmail.com>
> >
> > Signed-off-by: Limin Wang <lance.lmwang@gmail.com>
> > ---
> >  libavfilter/vf_cover_rect.c | 49 +++++++++++++++++++++++++++----------
> >  1 file changed, 36 insertions(+), 13 deletions(-)
> >
> > diff --git a/libavfilter/vf_cover_rect.c b/libavfilter/vf_cover_rect.c
> > index 898debf09d..22389b79d2 100644
> > --- a/libavfilter/vf_cover_rect.c
> > +++ b/libavfilter/vf_cover_rect.c
> > @@ -28,6 +28,7 @@
> >  #include "internal.h"
> >
> >  #include "lavfutils.h"
> > +#include "lswsutils.h"
> >
> >  enum mode {
> >      MODE_COVER,
> > @@ -40,6 +41,7 @@ typedef struct CoverContext {
> >      int mode;
> >      char *cover_filename;
> >      AVFrame *cover_frame;
> > +    AVFrame *match_frame;
> >      int width, height;
> >  } CoverContext;
> >
> > @@ -71,21 +73,21 @@ static int config_input(AVFilterLink *inlink)
> >      return 0;
> >  }
> >
> > -static void cover_rect(CoverContext *cover, AVFrame *in, int offx, int
> offy)
> > +static void cover_rect(AVFrame *cover_frame, AVFrame *in, int offx, int
> offy)
> >  {
> >      int x, y, p;
> >
> >      for (p = 0; p < 3; p++) {
> >          uint8_t *data = in->data[p] + (offx>>!!p) + (offy>>!!p) *
> in->linesize[p];
> > -        const uint8_t *src = cover->cover_frame->data[p];
> > -        int w = AV_CEIL_RSHIFT(cover->cover_frame->width , !!p);
> > -        int h = AV_CEIL_RSHIFT(cover->cover_frame->height, !!p);
> > +        const uint8_t *src = cover_frame->data[p];
> > +        int w = AV_CEIL_RSHIFT(cover_frame->width , !!p);
> > +        int h = AV_CEIL_RSHIFT(cover_frame->height, !!p);
> >          for (y = 0; y < h; y++) {
> >              for (x = 0; x < w; x++) {
> >                  data[x] = src[x];
> >              }
> >              data += in->linesize[p];
> > -            src += cover->cover_frame->linesize[p];
> > +            src += cover_frame->linesize[p];
> >          }
> >      }
> >  }
> > @@ -138,7 +140,10 @@ static int filter_frame(AVFilterLink *inlink,
> AVFrame *in)
> >      CoverContext *cover = ctx->priv;
> >      AVDictionaryEntry *ex, *ey, *ew, *eh;
> >      int x = -1, y = -1, w = -1, h = -1;
> > +    enum AVPixelFormat in_format;
> >      char *xendptr = NULL, *yendptr = NULL, *wendptr = NULL, *hendptr =
> NULL;
> > +    AVFrame *cover_frame = NULL;
> > +    int ret;
> >
> >      ex = av_dict_get(in->metadata, "lavfi.rect.x", NULL,
> AV_DICT_MATCH_CASE);
> >      ey = av_dict_get(in->metadata, "lavfi.rect.y", NULL,
> AV_DICT_MATCH_CASE);
> > @@ -167,13 +172,30 @@ static int filter_frame(AVFilterLink *inlink,
> AVFrame *in)
> >      }
> >      w = FFMIN(w, in->width  - x);
> >      h = FFMIN(h, in->height - y);
> > +    in_format = in->format;
> >
> >      if (w > in->width || h > in->height || w <= 0 || h <= 0)
> >          return AVERROR(EINVAL);
> >
> >      if (cover->cover_frame) {
> > -        if (w != cover->cover_frame->width || h !=
> cover->cover_frame->height)
> > -            return AVERROR(EINVAL);
> > +        if (w != cover->cover_frame->width || h !=
> cover->cover_frame->height ||
> > +                in_format != cover->cover_frame->format) {
> > +            if (!cover->match_frame && !(cover->match_frame =
> av_frame_alloc()))
> > +                return AVERROR(EINVAL);
> > +
> > +            if ((ret = ff_scale_image(cover->match_frame->data,
> cover->match_frame->linesize,
> > +                            w, h, in_format, cover->cover_frame->data,
> cover->cover_frame->linesize,
> > +                            cover->cover_frame->width,
> cover->cover_frame->height,
> > +                            cover->cover_frame->format, ctx)) < 0)
> > +                return AVERROR(EINVAL);
>
> This looks like the same static cover image would be converted again for
> each input frame
> which if iam not misreading this would be doing the exact same operation
> over and over
> again, thats same wastefull
>
>
I have fixed the issue in the updated patch, please help to review again.
https://patchwork.ffmpeg.org/patch/13510/


> [....]
> --
> Michael     GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB
>
> Why not whip the teacher when the pupil misbehaves? -- Diogenes of Sinope
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Patch hide | download patch | download mbox

diff --git a/libavfilter/vf_cover_rect.c b/libavfilter/vf_cover_rect.c
index 898debf09d..22389b79d2 100644
--- a/libavfilter/vf_cover_rect.c
+++ b/libavfilter/vf_cover_rect.c
@@ -28,6 +28,7 @@ 
 #include "internal.h"
 
 #include "lavfutils.h"
+#include "lswsutils.h"
 
 enum mode {
     MODE_COVER,
@@ -40,6 +41,7 @@  typedef struct CoverContext {
     int mode;
     char *cover_filename;
     AVFrame *cover_frame;
+    AVFrame *match_frame;
     int width, height;
 } CoverContext;
 
@@ -71,21 +73,21 @@  static int config_input(AVFilterLink *inlink)
     return 0;
 }
 
-static void cover_rect(CoverContext *cover, AVFrame *in, int offx, int offy)
+static void cover_rect(AVFrame *cover_frame, AVFrame *in, int offx, int offy)
 {
     int x, y, p;
 
     for (p = 0; p < 3; p++) {
         uint8_t *data = in->data[p] + (offx>>!!p) + (offy>>!!p) * in->linesize[p];
-        const uint8_t *src = cover->cover_frame->data[p];
-        int w = AV_CEIL_RSHIFT(cover->cover_frame->width , !!p);
-        int h = AV_CEIL_RSHIFT(cover->cover_frame->height, !!p);
+        const uint8_t *src = cover_frame->data[p];
+        int w = AV_CEIL_RSHIFT(cover_frame->width , !!p);
+        int h = AV_CEIL_RSHIFT(cover_frame->height, !!p);
         for (y = 0; y < h; y++) {
             for (x = 0; x < w; x++) {
                 data[x] = src[x];
             }
             data += in->linesize[p];
-            src += cover->cover_frame->linesize[p];
+            src += cover_frame->linesize[p];
         }
     }
 }
@@ -138,7 +140,10 @@  static int filter_frame(AVFilterLink *inlink, AVFrame *in)
     CoverContext *cover = ctx->priv;
     AVDictionaryEntry *ex, *ey, *ew, *eh;
     int x = -1, y = -1, w = -1, h = -1;
+    enum AVPixelFormat in_format;
     char *xendptr = NULL, *yendptr = NULL, *wendptr = NULL, *hendptr = NULL;
+    AVFrame *cover_frame = NULL;
+    int ret;
 
     ex = av_dict_get(in->metadata, "lavfi.rect.x", NULL, AV_DICT_MATCH_CASE);
     ey = av_dict_get(in->metadata, "lavfi.rect.y", NULL, AV_DICT_MATCH_CASE);
@@ -167,13 +172,30 @@  static int filter_frame(AVFilterLink *inlink, AVFrame *in)
     }
     w = FFMIN(w, in->width  - x);
     h = FFMIN(h, in->height - y);
+    in_format = in->format;
 
     if (w > in->width || h > in->height || w <= 0 || h <= 0)
         return AVERROR(EINVAL);
 
     if (cover->cover_frame) {
-        if (w != cover->cover_frame->width || h != cover->cover_frame->height)
-            return AVERROR(EINVAL);
+        if (w != cover->cover_frame->width || h != cover->cover_frame->height ||
+                in_format != cover->cover_frame->format) {
+            if (!cover->match_frame && !(cover->match_frame = av_frame_alloc()))
+                return AVERROR(EINVAL);
+
+            if ((ret = ff_scale_image(cover->match_frame->data, cover->match_frame->linesize,
+                            w, h, in_format, cover->cover_frame->data, cover->cover_frame->linesize,
+                            cover->cover_frame->width, cover->cover_frame->height,
+                            cover->cover_frame->format, ctx)) < 0)
+                return AVERROR(EINVAL);
+
+            cover->match_frame->width  = w;
+            cover->match_frame->height = h;
+            cover->match_frame->format = in_format;
+
+            cover_frame = cover->match_frame;
+        } else
+            cover_frame = cover->cover_frame;
     }
 
     cover->width  = w;
@@ -187,8 +209,12 @@  static int filter_frame(AVFilterLink *inlink, AVFrame *in)
     if (cover->mode == MODE_BLUR) {
         blur (cover, in, x, y);
     } else {
-        cover_rect(cover, in, x, y);
+        cover_rect(cover_frame, in, x, y);
+
+        if (cover->match_frame)
+            av_freep(&cover->match_frame->data[0]);
     }
+
     return ff_filter_frame(ctx->outputs[0], in);
 }
 
@@ -199,6 +225,8 @@  static av_cold void uninit(AVFilterContext *ctx)
     if (cover->cover_frame)
         av_freep(&cover->cover_frame->data[0]);
     av_frame_free(&cover->cover_frame);
+
+    av_frame_free(&cover->match_frame);
 }
 
 static av_cold int init(AVFilterContext *ctx)
@@ -220,11 +248,6 @@  static av_cold int init(AVFilterContext *ctx)
                                 &cover->cover_frame->width, &cover->cover_frame->height,
                                 &cover->cover_frame->format, cover->cover_filename, ctx)) < 0)
             return ret;
-
-        if (cover->cover_frame->format != AV_PIX_FMT_YUV420P && cover->cover_frame->format != AV_PIX_FMT_YUVJ420P) {
-            av_log(ctx, AV_LOG_ERROR, "cover image is not a YUV420 image\n");
-            return AVERROR(EINVAL);
-        }
     }
 
     return 0;