[FFmpeg-devel,v3,2/2] fftools/ffmpeg: add exif orientation support per frame's metadata

Submitted by Jun Li on May 28, 2019, 6:18 a.m.

Details

Message ID 20190528061826.22537-2-junli1026@gmail.com
State New
Headers show

Commit Message

Jun Li May 28, 2019, 6:18 a.m.
Fix #6945
Rotate or/and flip frame according to frame's metadata orientation
---
 fftools/ffmpeg.c        | 16 +++++++++++++++-
 fftools/ffmpeg.h        |  3 ++-
 fftools/ffmpeg_filter.c | 28 +++++++++++++++++++++++-----
 3 files changed, 40 insertions(+), 7 deletions(-)

Comments

Paul B Mahol May 28, 2019, 2:24 p.m.
On 5/28/19, Jun Li <junli1026@gmail.com> wrote:
> Fix #6945
> Rotate or/and flip frame according to frame's metadata orientation
> ---
>  fftools/ffmpeg.c        | 16 +++++++++++++++-
>  fftools/ffmpeg.h        |  3 ++-
>  fftools/ffmpeg_filter.c | 28 +++++++++++++++++++++++-----
>  3 files changed, 40 insertions(+), 7 deletions(-)
>
> diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c
> index 01f04103cf..2f4229a9d0 100644
> --- a/fftools/ffmpeg.c
> +++ b/fftools/ffmpeg.c
> @@ -2126,6 +2126,19 @@ static int ifilter_has_all_input_formats(FilterGraph
> *fg)
>      return 1;
>  }
>
> +static int orientation_need_update(InputFilter *ifilter, AVFrame *frame)
> +{
> +    int orientaion = get_frame_orientation(frame);
> +    int filterst = ifilter->orientation <= 1 ? 0 : // not set
> +                   ifilter->orientation <= 4 ? 1 : // auto flip
> +                   2; // auto transpose
> +    int framest = orientaion <= 1 ? 0 : // not set
> +                  orientaion <= 4 ? 1 : // auto flip
> +                  2; // auto transpose
> +
> +    return filterst != framest;
> +}
> +
>  static int ifilter_send_frame(InputFilter *ifilter, AVFrame *frame)
>  {
>      FilterGraph *fg = ifilter->graph;
> @@ -2142,7 +2155,8 @@ static int ifilter_send_frame(InputFilter *ifilter,
> AVFrame *frame)
>          break;
>      case AVMEDIA_TYPE_VIDEO:
>          need_reinit |= ifilter->width  != frame->width ||
> -                       ifilter->height != frame->height;
> +                       ifilter->height != frame->height ||
> +                       orientation_need_update(ifilter, frame);
>          break;
>      }
>
> diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
> index eb1eaf6363..54532ef0eb 100644
> --- a/fftools/ffmpeg.h
> +++ b/fftools/ffmpeg.h
> @@ -244,7 +244,7 @@ typedef struct InputFilter {
>      // parameters configured for this input
>      int format;
>
> -    int width, height;
> +    int width, height, orientation;
>      AVRational sample_aspect_ratio;
>
>      int sample_rate;
> @@ -649,6 +649,7 @@ int init_complex_filtergraph(FilterGraph *fg);
>  void sub2video_update(InputStream *ist, AVSubtitle *sub);
>
>  int ifilter_parameters_from_frame(InputFilter *ifilter, const AVFrame
> *frame);
> +int get_frame_orientation(const AVFrame* frame);
>
>  int ffmpeg_parse_options(int argc, char **argv);
>
> diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c
> index 72838de1e2..ff63540906 100644
> --- a/fftools/ffmpeg_filter.c
> +++ b/fftools/ffmpeg_filter.c
> @@ -743,6 +743,18 @@ static int sub2video_prepare(InputStream *ist,
> InputFilter *ifilter)
>      return 0;
>  }
>
> +int get_frame_orientation(const AVFrame *frame)
> +{
> +    AVDictionaryEntry *entry = NULL;
> +    int orientation = 0;
> +
> +    // read exif orientation data
> +    entry = av_dict_get(frame->metadata, "Orientation", NULL, 0);
> +    if (entry)
> +        orientation = atoi(entry->value);
> +    return orientation;
> +}
> +
>  static int configure_input_video_filter(FilterGraph *fg, InputFilter
> *ifilter,
>                                          AVFilterInOut *in)
>  {
> @@ -809,13 +821,18 @@ static int configure_input_video_filter(FilterGraph
> *fg, InputFilter *ifilter,
>      if (ist->autorotate) {
>          double theta = get_rotation(ist->st);
>
> -        if (fabs(theta - 90) < 1.0) {
> +        if (fabs(theta) < 1.0) { // no rotation info in stream meta
> +            if (ifilter->orientation < 0 || ifilter->orientation > 8) {
> +                av_log(NULL, AV_LOG_ERROR, "Invalid frame orientation:
> %i\n", ifilter->orientation);
> +            } else if (ifilter->orientation > 1 && ifilter->orientation <=
> 4) { // skip 0 (not set) and 1 (orientaion 'Normal')
> +                ret = insert_filter(&last_filter, &pad_idx, "transpose",
> "orientation=-1");
> +            } else if (ifilter->orientation > 4) {
> +                ret = insert_filter(&last_filter, &pad_idx, "transpose",
> "orientation=-2");

Usning non-named option values is bad and not user-friendly.

> +            }
> +        } else if (fabs(theta - 90) < 1.0) {
>              ret = insert_filter(&last_filter, &pad_idx, "transpose",
> "clock");
>          } else if (fabs(theta - 180) < 1.0) {
> -            ret = insert_filter(&last_filter, &pad_idx, "hflip", NULL);
> -            if (ret < 0)
> -                return ret;
> -            ret = insert_filter(&last_filter, &pad_idx, "vflip", NULL);
> +            ret = insert_filter(&last_filter, &pad_idx, "transpose",
> "orientation=3");

ditto

>          } else if (fabs(theta - 270) < 1.0) {
>              ret = insert_filter(&last_filter, &pad_idx, "transpose",
> "cclock");
>          } else if (fabs(theta) > 1.0) {
> @@ -1191,6 +1208,7 @@ int ifilter_parameters_from_frame(InputFilter
> *ifilter, const AVFrame *frame)
>      ifilter->width               = frame->width;
>      ifilter->height              = frame->height;
>      ifilter->sample_aspect_ratio = frame->sample_aspect_ratio;
> +    ifilter->orientation         = get_frame_orientation(frame);
>
>      ifilter->sample_rate         = frame->sample_rate;
>      ifilter->channels            = frame->channels;
> --
> 2.17.1
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".
Michael Niedermayer May 28, 2019, 8:50 p.m.
On Mon, May 27, 2019 at 11:18:26PM -0700, Jun Li wrote:
> Fix #6945
> Rotate or/and flip frame according to frame's metadata orientation
> ---
>  fftools/ffmpeg.c        | 16 +++++++++++++++-
>  fftools/ffmpeg.h        |  3 ++-
>  fftools/ffmpeg_filter.c | 28 +++++++++++++++++++++++-----
>  3 files changed, 40 insertions(+), 7 deletions(-)
> 
> diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c
> index 01f04103cf..2f4229a9d0 100644
> --- a/fftools/ffmpeg.c
> +++ b/fftools/ffmpeg.c
> @@ -2126,6 +2126,19 @@ static int ifilter_has_all_input_formats(FilterGraph *fg)
>      return 1;
>  }
>  
> +static int orientation_need_update(InputFilter *ifilter, AVFrame *frame)
> +{
> +    int orientaion = get_frame_orientation(frame);
> +    int filterst = ifilter->orientation <= 1 ? 0 : // not set
> +                   ifilter->orientation <= 4 ? 1 : // auto flip
> +                   2; // auto transpose
> +    int framest = orientaion <= 1 ? 0 : // not set
> +                  orientaion <= 4 ? 1 : // auto flip
> +                  2; // auto transpose
> +
> +    return filterst != framest;
> +}
> +
>  static int ifilter_send_frame(InputFilter *ifilter, AVFrame *frame)
>  {
>      FilterGraph *fg = ifilter->graph;
> @@ -2142,7 +2155,8 @@ static int ifilter_send_frame(InputFilter *ifilter, AVFrame *frame)
>          break;
>      case AVMEDIA_TYPE_VIDEO:
>          need_reinit |= ifilter->width  != frame->width ||
> -                       ifilter->height != frame->height;
> +                       ifilter->height != frame->height ||
> +                       orientation_need_update(ifilter, frame);
>          break;
>      }
>  
> diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
> index eb1eaf6363..54532ef0eb 100644
> --- a/fftools/ffmpeg.h
> +++ b/fftools/ffmpeg.h
> @@ -244,7 +244,7 @@ typedef struct InputFilter {
>      // parameters configured for this input
>      int format;
>  
> -    int width, height;
> +    int width, height, orientation;
>      AVRational sample_aspect_ratio;
>  
>      int sample_rate;
> @@ -649,6 +649,7 @@ int init_complex_filtergraph(FilterGraph *fg);
>  void sub2video_update(InputStream *ist, AVSubtitle *sub);
>  
>  int ifilter_parameters_from_frame(InputFilter *ifilter, const AVFrame *frame);
> +int get_frame_orientation(const AVFrame* frame);
>  
>  int ffmpeg_parse_options(int argc, char **argv);
>  
> diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c
> index 72838de1e2..ff63540906 100644
> --- a/fftools/ffmpeg_filter.c
> +++ b/fftools/ffmpeg_filter.c
> @@ -743,6 +743,18 @@ static int sub2video_prepare(InputStream *ist, InputFilter *ifilter)
>      return 0;
>  }
>  
> +int get_frame_orientation(const AVFrame *frame)
> +{
> +    AVDictionaryEntry *entry = NULL;
> +    int orientation = 0;
> +    
> +    // read exif orientation data
> +    entry = av_dict_get(frame->metadata, "Orientation", NULL, 0);

> +    if (entry)
> +        orientation = atoi(entry->value);

this probably should be checking the validity of the string unless
it has been checked already elsewhere

thx

[...]
Jun Li May 29, 2019, 5:09 a.m.
On Tue, May 28, 2019 at 1:50 PM Michael Niedermayer <michael@niedermayer.cc>
wrote:

> On Mon, May 27, 2019 at 11:18:26PM -0700, Jun Li wrote:
> > Fix #6945
> > Rotate or/and flip frame according to frame's metadata orientation
> > ---
> >  fftools/ffmpeg.c        | 16 +++++++++++++++-
> >  fftools/ffmpeg.h        |  3 ++-
> >  fftools/ffmpeg_filter.c | 28 +++++++++++++++++++++++-----
> >  3 files changed, 40 insertions(+), 7 deletions(-)
> >
> > diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c
> > index 01f04103cf..2f4229a9d0 100644
> > --- a/fftools/ffmpeg.c
> > +++ b/fftools/ffmpeg.c
> > @@ -2126,6 +2126,19 @@ static int
> ifilter_has_all_input_formats(FilterGraph *fg)
> >      return 1;
> >  }
> >
> > +static int orientation_need_update(InputFilter *ifilter, AVFrame *frame)
> > +{
> > +    int orientaion = get_frame_orientation(frame);
> > +    int filterst = ifilter->orientation <= 1 ? 0 : // not set
> > +                   ifilter->orientation <= 4 ? 1 : // auto flip
> > +                   2; // auto transpose
> > +    int framest = orientaion <= 1 ? 0 : // not set
> > +                  orientaion <= 4 ? 1 : // auto flip
> > +                  2; // auto transpose
> > +
> > +    return filterst != framest;
> > +}
> > +
> >  static int ifilter_send_frame(InputFilter *ifilter, AVFrame *frame)
> >  {
> >      FilterGraph *fg = ifilter->graph;
> > @@ -2142,7 +2155,8 @@ static int ifilter_send_frame(InputFilter
> *ifilter, AVFrame *frame)
> >          break;
> >      case AVMEDIA_TYPE_VIDEO:
> >          need_reinit |= ifilter->width  != frame->width ||
> > -                       ifilter->height != frame->height;
> > +                       ifilter->height != frame->height ||
> > +                       orientation_need_update(ifilter, frame);
> >          break;
> >      }
> >
> > diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
> > index eb1eaf6363..54532ef0eb 100644
> > --- a/fftools/ffmpeg.h
> > +++ b/fftools/ffmpeg.h
> > @@ -244,7 +244,7 @@ typedef struct InputFilter {
> >      // parameters configured for this input
> >      int format;
> >
> > -    int width, height;
> > +    int width, height, orientation;
> >      AVRational sample_aspect_ratio;
> >
> >      int sample_rate;
> > @@ -649,6 +649,7 @@ int init_complex_filtergraph(FilterGraph *fg);
> >  void sub2video_update(InputStream *ist, AVSubtitle *sub);
> >
> >  int ifilter_parameters_from_frame(InputFilter *ifilter, const AVFrame
> *frame);
> > +int get_frame_orientation(const AVFrame* frame);
> >
> >  int ffmpeg_parse_options(int argc, char **argv);
> >
> > diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c
> > index 72838de1e2..ff63540906 100644
> > --- a/fftools/ffmpeg_filter.c
> > +++ b/fftools/ffmpeg_filter.c
> > @@ -743,6 +743,18 @@ static int sub2video_prepare(InputStream *ist,
> InputFilter *ifilter)
> >      return 0;
> >  }
> >
> > +int get_frame_orientation(const AVFrame *frame)
> > +{
> > +    AVDictionaryEntry *entry = NULL;
> > +    int orientation = 0;
> > +
> > +    // read exif orientation data
> > +    entry = av_dict_get(frame->metadata, "Orientation", NULL, 0);
>
> > +    if (entry)
> > +        orientation = atoi(entry->value);
>
> this probably should be checking the validity of the string unless
> it has been checked already elsewhere


Thanks Michael for the catch. I updated to version 5:
https://patchwork.ffmpeg.org/patch/13321/
https://patchwork.ffmpeg.org/patch/13322/

Best Regards,
Jun


> thx
>
> [...]
> --
> Michael     GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB
>
> I have never wished to cater to the crowd; for what I know they do not
> approve, and what they approve I do not know. -- Epicurus
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request@ffmpeg.org with subject "unsubscribe".

Patch hide | download patch | download mbox

diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c
index 01f04103cf..2f4229a9d0 100644
--- a/fftools/ffmpeg.c
+++ b/fftools/ffmpeg.c
@@ -2126,6 +2126,19 @@  static int ifilter_has_all_input_formats(FilterGraph *fg)
     return 1;
 }
 
+static int orientation_need_update(InputFilter *ifilter, AVFrame *frame)
+{
+    int orientaion = get_frame_orientation(frame);
+    int filterst = ifilter->orientation <= 1 ? 0 : // not set
+                   ifilter->orientation <= 4 ? 1 : // auto flip
+                   2; // auto transpose
+    int framest = orientaion <= 1 ? 0 : // not set
+                  orientaion <= 4 ? 1 : // auto flip
+                  2; // auto transpose
+
+    return filterst != framest;
+}
+
 static int ifilter_send_frame(InputFilter *ifilter, AVFrame *frame)
 {
     FilterGraph *fg = ifilter->graph;
@@ -2142,7 +2155,8 @@  static int ifilter_send_frame(InputFilter *ifilter, AVFrame *frame)
         break;
     case AVMEDIA_TYPE_VIDEO:
         need_reinit |= ifilter->width  != frame->width ||
-                       ifilter->height != frame->height;
+                       ifilter->height != frame->height ||
+                       orientation_need_update(ifilter, frame);
         break;
     }
 
diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
index eb1eaf6363..54532ef0eb 100644
--- a/fftools/ffmpeg.h
+++ b/fftools/ffmpeg.h
@@ -244,7 +244,7 @@  typedef struct InputFilter {
     // parameters configured for this input
     int format;
 
-    int width, height;
+    int width, height, orientation;
     AVRational sample_aspect_ratio;
 
     int sample_rate;
@@ -649,6 +649,7 @@  int init_complex_filtergraph(FilterGraph *fg);
 void sub2video_update(InputStream *ist, AVSubtitle *sub);
 
 int ifilter_parameters_from_frame(InputFilter *ifilter, const AVFrame *frame);
+int get_frame_orientation(const AVFrame* frame);
 
 int ffmpeg_parse_options(int argc, char **argv);
 
diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c
index 72838de1e2..ff63540906 100644
--- a/fftools/ffmpeg_filter.c
+++ b/fftools/ffmpeg_filter.c
@@ -743,6 +743,18 @@  static int sub2video_prepare(InputStream *ist, InputFilter *ifilter)
     return 0;
 }
 
+int get_frame_orientation(const AVFrame *frame)
+{
+    AVDictionaryEntry *entry = NULL;
+    int orientation = 0;
+    
+    // read exif orientation data
+    entry = av_dict_get(frame->metadata, "Orientation", NULL, 0);
+    if (entry)
+        orientation = atoi(entry->value);
+    return orientation;
+}
+
 static int configure_input_video_filter(FilterGraph *fg, InputFilter *ifilter,
                                         AVFilterInOut *in)
 {
@@ -809,13 +821,18 @@  static int configure_input_video_filter(FilterGraph *fg, InputFilter *ifilter,
     if (ist->autorotate) {
         double theta = get_rotation(ist->st);
 
-        if (fabs(theta - 90) < 1.0) {
+        if (fabs(theta) < 1.0) { // no rotation info in stream meta
+            if (ifilter->orientation < 0 || ifilter->orientation > 8) {
+                av_log(NULL, AV_LOG_ERROR, "Invalid frame orientation: %i\n", ifilter->orientation);
+            } else if (ifilter->orientation > 1 && ifilter->orientation <= 4) { // skip 0 (not set) and 1 (orientaion 'Normal') 
+                ret = insert_filter(&last_filter, &pad_idx, "transpose", "orientation=-1");
+            } else if (ifilter->orientation > 4) {
+                ret = insert_filter(&last_filter, &pad_idx, "transpose", "orientation=-2");
+            }
+        } else if (fabs(theta - 90) < 1.0) {
             ret = insert_filter(&last_filter, &pad_idx, "transpose", "clock");
         } else if (fabs(theta - 180) < 1.0) {
-            ret = insert_filter(&last_filter, &pad_idx, "hflip", NULL);
-            if (ret < 0)
-                return ret;
-            ret = insert_filter(&last_filter, &pad_idx, "vflip", NULL);
+            ret = insert_filter(&last_filter, &pad_idx, "transpose", "orientation=3");
         } else if (fabs(theta - 270) < 1.0) {
             ret = insert_filter(&last_filter, &pad_idx, "transpose", "cclock");
         } else if (fabs(theta) > 1.0) {
@@ -1191,6 +1208,7 @@  int ifilter_parameters_from_frame(InputFilter *ifilter, const AVFrame *frame)
     ifilter->width               = frame->width;
     ifilter->height              = frame->height;
     ifilter->sample_aspect_ratio = frame->sample_aspect_ratio;
+    ifilter->orientation         = get_frame_orientation(frame);
 
     ifilter->sample_rate         = frame->sample_rate;
     ifilter->channels            = frame->channels;