diff mbox

[FFmpeg-devel,1/4] lavu/frame: add mb_types side data

Message ID 20180617042117.3844-1-ramiro.polla@gmail.com
State New
Headers show

Commit Message

Ramiro Polla June 17, 2018, 4:21 a.m. UTC 
---
 libavcodec/avcodec.h       |  4 ++++
 libavcodec/mpegutils.c     | 20 ++++++++++++++++++++
 libavcodec/options_table.h |  1 +
 libavutil/frame.c          |  1 +
 libavutil/frame.h          |  9 +++++++++
 5 files changed, 35 insertions(+)

Comments

Rostislav Pehlivanov June 17, 2018, 12:45 p.m. UTC | #1 
On 17 June 2018 at 05:21, Ramiro Polla <ramiro.polla@gmail.com> wrote:

> ---
>  libavcodec/avcodec.h       |  4 ++++
>  libavcodec/mpegutils.c     | 20 ++++++++++++++++++++
>  libavcodec/options_table.h |  1 +
>  libavutil/frame.c          |  1 +
>  libavutil/frame.h          |  9 +++++++++
>  5 files changed, 35 insertions(+)
>
> diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h
> index c90166deb6..7fe4fc9347 100644
> --- a/libavcodec/avcodec.h
> +++ b/libavcodec/avcodec.h
> @@ -929,6 +929,10 @@ typedef struct RcOverride{
>   */
>  #define AV_CODEC_FLAG2_SHOW_ALL       (1 << 22)
>  /**
> + * Export macroblock types through frame side data
> + */
> +#define AV_CODEC_FLAG2_EXPORT_MB_TYPES (1 << 27)
> +/**
>   * Export motion vectors through frame side data
>   */
>  #define AV_CODEC_FLAG2_EXPORT_MVS     (1 << 28)
> diff --git a/libavcodec/mpegutils.c b/libavcodec/mpegutils.c
> index 3f94540616..0fbe5f8c9d 100644
> --- a/libavcodec/mpegutils.c
> +++ b/libavcodec/mpegutils.c
> @@ -188,6 +188,26 @@ void ff_print_debug_info2(AVCodecContext *avctx,
> AVFrame *pict, uint8_t *mbskip_
>          av_freep(&mvs);
>      }
>
> +    if ((avctx->flags2 & AV_CODEC_FLAG2_EXPORT_MB_TYPES) &&
> mbtype_table) {
> +        int size = (2 + mb_height * mb_width) * sizeof(uint32_t);
> +        int mb_x, mb_y;
> +
> +        AVFrameSideData *sd;
> +        uint32_t *out;
> +
> +        sd = av_frame_new_side_data(pict, AV_FRAME_DATA_MB_TYPES, size);
> +        if (!sd)
> +            return;
> +
> +        out = (uint32_t *) sd->data;
> +        *out++ = mb_height;
> +        *out++ = mb_width;
> +
> +        for (mb_y = 0; mb_y < mb_height; mb_y++)
> +            for (mb_x = 0; mb_x < mb_width; mb_x++)
> +                *out++ = mbtype_table[mb_x + mb_y * mb_stride];
> +    }
> +
>      /* TODO: export all the following to make them accessible for users
> (and filters) */
>      if (avctx->hwaccel || !mbtype_table)
>          return;
> diff --git a/libavcodec/options_table.h b/libavcodec/options_table.h
> index 099261e168..25c84de321 100644
> --- a/libavcodec/options_table.h
> +++ b/libavcodec/options_table.h
> @@ -76,6 +76,7 @@ static const AVOption avcodec_options[] = {
>  {"export_mvs", "export motion vectors through frame side data", 0,
> AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG2_EXPORT_MVS}, INT_MIN, INT_MAX,
> V|D, "flags2"},
>  {"skip_manual", "do not skip samples and export skip information as frame
> side data", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG2_SKIP_MANUAL},
> INT_MIN, INT_MAX, V|D, "flags2"},
>  {"ass_ro_flush_noop", "do not reset ASS ReadOrder field on flush", 0,
> AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG2_RO_FLUSH_NOOP}, INT_MIN, INT_MAX,
> S|D, "flags2"},
> +{"export_mb_types", "export macroblock types through frame side data", 0,
> AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG2_EXPORT_MB_TYPES}, INT_MIN,
> INT_MAX, V|D, "flags2"},
>  {"time_base", NULL, OFFSET(time_base), AV_OPT_TYPE_RATIONAL, {.dbl = 0},
> 0, INT_MAX},
>  {"g", "set the group of picture (GOP) size", OFFSET(gop_size),
> AV_OPT_TYPE_INT, {.i64 = 12 }, INT_MIN, INT_MAX, V|E},
>  {"ar", "set audio sampling rate (in Hz)", OFFSET(sample_rate),
> AV_OPT_TYPE_INT, {.i64 = DEFAULT }, 0, INT_MAX, A|D|E},
> diff --git a/libavutil/frame.c b/libavutil/frame.c
> index deb9b6f334..577d4f6e6d 100644
> --- a/libavutil/frame.c
> +++ b/libavutil/frame.c
> @@ -834,6 +834,7 @@ const char *av_frame_side_data_name(enum
> AVFrameSideDataType type)
>      case AV_FRAME_DATA_ICC_PROFILE:                 return "ICC profile";
>      case AV_FRAME_DATA_QP_TABLE_PROPERTIES:         return "QP table
> properties";
>      case AV_FRAME_DATA_QP_TABLE_DATA:               return "QP table
> data";
> +    case AV_FRAME_DATA_MB_TYPES:                    return "Macroblock
> types";
>      }
>      return NULL;
>  }
> diff --git a/libavutil/frame.h b/libavutil/frame.h
> index 9d57d6ce66..ce1231b03b 100644
> --- a/libavutil/frame.h
> +++ b/libavutil/frame.h
> @@ -158,6 +158,15 @@ enum AVFrameSideDataType {
>       */
>      AV_FRAME_DATA_QP_TABLE_DATA,
>  #endif
> +
> +    /**
> +     * Macroblock types exported by some codecs (on demand through the
> +     * export_mb_types flag set in the libavcodec AVCodecContext flags2
> option).
> +     * The data is composed by a header consisting of uint32_t mb_height
> and
> +     * uint32_t mb_width, followed by a uint32_t
> mb_types[mb_height][mb_width]
> +     * array.
> +     */
> +    AV_FRAME_DATA_MB_TYPES,
>  };
>
>  enum AVActiveFormatDescription {
> --
> 2.11.0
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> http://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>

NAK

I'd really rather have a json or xml output which would be codec
independent rather than this codec specific format. Which could then be fed
into an analyzer (like the AV1 analyzer). We discussed this before.
Hendrik Leppkes June 17, 2018, 2:47 p.m. UTC | #2 
On Sun, Jun 17, 2018 at 2:45 PM Rostislav Pehlivanov
<atomnuker@gmail.com> wrote:
>
> On 17 June 2018 at 05:21, Ramiro Polla <ramiro.polla@gmail.com> wrote:
>
> > ---
> >  libavcodec/avcodec.h       |  4 ++++
> >  libavcodec/mpegutils.c     | 20 ++++++++++++++++++++
> >  libavcodec/options_table.h |  1 +
> >  libavutil/frame.c          |  1 +
> >  libavutil/frame.h          |  9 +++++++++
> >  5 files changed, 35 insertions(+)
> >
> > diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h
> > index c90166deb6..7fe4fc9347 100644
> > --- a/libavcodec/avcodec.h
> > +++ b/libavcodec/avcodec.h
> > @@ -929,6 +929,10 @@ typedef struct RcOverride{
> >   */
> >  #define AV_CODEC_FLAG2_SHOW_ALL       (1 << 22)
> >  /**
> > + * Export macroblock types through frame side data
> > + */
> > +#define AV_CODEC_FLAG2_EXPORT_MB_TYPES (1 << 27)
> > +/**
> >   * Export motion vectors through frame side data
> >   */
> >  #define AV_CODEC_FLAG2_EXPORT_MVS     (1 << 28)
> > diff --git a/libavcodec/mpegutils.c b/libavcodec/mpegutils.c
> > index 3f94540616..0fbe5f8c9d 100644
> > --- a/libavcodec/mpegutils.c
> > +++ b/libavcodec/mpegutils.c
> > @@ -188,6 +188,26 @@ void ff_print_debug_info2(AVCodecContext *avctx,
> > AVFrame *pict, uint8_t *mbskip_
> >          av_freep(&mvs);
> >      }
> >
> > +    if ((avctx->flags2 & AV_CODEC_FLAG2_EXPORT_MB_TYPES) &&
> > mbtype_table) {
> > +        int size = (2 + mb_height * mb_width) * sizeof(uint32_t);
> > +        int mb_x, mb_y;
> > +
> > +        AVFrameSideData *sd;
> > +        uint32_t *out;
> > +
> > +        sd = av_frame_new_side_data(pict, AV_FRAME_DATA_MB_TYPES, size);
> > +        if (!sd)
> > +            return;
> > +
> > +        out = (uint32_t *) sd->data;
> > +        *out++ = mb_height;
> > +        *out++ = mb_width;
> > +
> > +        for (mb_y = 0; mb_y < mb_height; mb_y++)
> > +            for (mb_x = 0; mb_x < mb_width; mb_x++)
> > +                *out++ = mbtype_table[mb_x + mb_y * mb_stride];
> > +    }
> > +
> >      /* TODO: export all the following to make them accessible for users
> > (and filters) */
> >      if (avctx->hwaccel || !mbtype_table)
> >          return;
> > diff --git a/libavcodec/options_table.h b/libavcodec/options_table.h
> > index 099261e168..25c84de321 100644
> > --- a/libavcodec/options_table.h
> > +++ b/libavcodec/options_table.h
> > @@ -76,6 +76,7 @@ static const AVOption avcodec_options[] = {
> >  {"export_mvs", "export motion vectors through frame side data", 0,
> > AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG2_EXPORT_MVS}, INT_MIN, INT_MAX,
> > V|D, "flags2"},
> >  {"skip_manual", "do not skip samples and export skip information as frame
> > side data", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG2_SKIP_MANUAL},
> > INT_MIN, INT_MAX, V|D, "flags2"},
> >  {"ass_ro_flush_noop", "do not reset ASS ReadOrder field on flush", 0,
> > AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG2_RO_FLUSH_NOOP}, INT_MIN, INT_MAX,
> > S|D, "flags2"},
> > +{"export_mb_types", "export macroblock types through frame side data", 0,
> > AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG2_EXPORT_MB_TYPES}, INT_MIN,
> > INT_MAX, V|D, "flags2"},
> >  {"time_base", NULL, OFFSET(time_base), AV_OPT_TYPE_RATIONAL, {.dbl = 0},
> > 0, INT_MAX},
> >  {"g", "set the group of picture (GOP) size", OFFSET(gop_size),
> > AV_OPT_TYPE_INT, {.i64 = 12 }, INT_MIN, INT_MAX, V|E},
> >  {"ar", "set audio sampling rate (in Hz)", OFFSET(sample_rate),
> > AV_OPT_TYPE_INT, {.i64 = DEFAULT }, 0, INT_MAX, A|D|E},
> > diff --git a/libavutil/frame.c b/libavutil/frame.c
> > index deb9b6f334..577d4f6e6d 100644
> > --- a/libavutil/frame.c
> > +++ b/libavutil/frame.c
> > @@ -834,6 +834,7 @@ const char *av_frame_side_data_name(enum
> > AVFrameSideDataType type)
> >      case AV_FRAME_DATA_ICC_PROFILE:                 return "ICC profile";
> >      case AV_FRAME_DATA_QP_TABLE_PROPERTIES:         return "QP table
> > properties";
> >      case AV_FRAME_DATA_QP_TABLE_DATA:               return "QP table
> > data";
> > +    case AV_FRAME_DATA_MB_TYPES:                    return "Macroblock
> > types";
> >      }
> >      return NULL;
> >  }
> > diff --git a/libavutil/frame.h b/libavutil/frame.h
> > index 9d57d6ce66..ce1231b03b 100644
> > --- a/libavutil/frame.h
> > +++ b/libavutil/frame.h
> > @@ -158,6 +158,15 @@ enum AVFrameSideDataType {
> >       */
> >      AV_FRAME_DATA_QP_TABLE_DATA,
> >  #endif
> > +
> > +    /**
> > +     * Macroblock types exported by some codecs (on demand through the
> > +     * export_mb_types flag set in the libavcodec AVCodecContext flags2
> > option).
> > +     * The data is composed by a header consisting of uint32_t mb_height
> > and
> > +     * uint32_t mb_width, followed by a uint32_t
> > mb_types[mb_height][mb_width]
> > +     * array.
> > +     */
> > +    AV_FRAME_DATA_MB_TYPES,
> >  };
> >
> >  enum AVActiveFormatDescription {
> > --
> > 2.11.0
> >
> > _______________________________________________
> > ffmpeg-devel mailing list
> > ffmpeg-devel@ffmpeg.org
> > http://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> >
>
> NAK
>
> I'd really rather have a json or xml output which would be codec
> independent rather than this codec specific format. Which could then be fed
> into an analyzer (like the AV1 analyzer). We discussed this before.

Producing some human/tool readable textual format is not something
that should be put on the decoders to do, its something a tool like
ffprobe could do, or at worst put as a formatting helper function in
avutil.
Internally, we should have a common binary format (because thats the
easiest to handle from C) that expresses the MB types. Does our
current format qualify for that?  I don't know, maybe not, can it deal
with varying block sizes modern codecs use?

- Hendrik
diff mbox

Patch

diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h
index c90166deb6..7fe4fc9347 100644
--- a/libavcodec/avcodec.h
+++ b/libavcodec/avcodec.h
@@ -929,6 +929,10 @@  typedef struct RcOverride{
  */
 #define AV_CODEC_FLAG2_SHOW_ALL       (1 << 22)
 /**
+ * Export macroblock types through frame side data
+ */
+#define AV_CODEC_FLAG2_EXPORT_MB_TYPES (1 << 27)
+/**
  * Export motion vectors through frame side data
  */
 #define AV_CODEC_FLAG2_EXPORT_MVS     (1 << 28)
diff --git a/libavcodec/mpegutils.c b/libavcodec/mpegutils.c
index 3f94540616..0fbe5f8c9d 100644
--- a/libavcodec/mpegutils.c
+++ b/libavcodec/mpegutils.c
@@ -188,6 +188,26 @@  void ff_print_debug_info2(AVCodecContext *avctx, AVFrame *pict, uint8_t *mbskip_
         av_freep(&mvs);
     }
 
+    if ((avctx->flags2 & AV_CODEC_FLAG2_EXPORT_MB_TYPES) && mbtype_table) {
+        int size = (2 + mb_height * mb_width) * sizeof(uint32_t);
+        int mb_x, mb_y;
+
+        AVFrameSideData *sd;
+        uint32_t *out;
+
+        sd = av_frame_new_side_data(pict, AV_FRAME_DATA_MB_TYPES, size);
+        if (!sd)
+            return;
+
+        out = (uint32_t *) sd->data;
+        *out++ = mb_height;
+        *out++ = mb_width;
+
+        for (mb_y = 0; mb_y < mb_height; mb_y++)
+            for (mb_x = 0; mb_x < mb_width; mb_x++)
+                *out++ = mbtype_table[mb_x + mb_y * mb_stride];
+    }
+
     /* TODO: export all the following to make them accessible for users (and filters) */
     if (avctx->hwaccel || !mbtype_table)
         return;
diff --git a/libavcodec/options_table.h b/libavcodec/options_table.h
index 099261e168..25c84de321 100644
--- a/libavcodec/options_table.h
+++ b/libavcodec/options_table.h
@@ -76,6 +76,7 @@  static const AVOption avcodec_options[] = {
 {"export_mvs", "export motion vectors through frame side data", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG2_EXPORT_MVS}, INT_MIN, INT_MAX, V|D, "flags2"},
 {"skip_manual", "do not skip samples and export skip information as frame side data", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG2_SKIP_MANUAL}, INT_MIN, INT_MAX, V|D, "flags2"},
 {"ass_ro_flush_noop", "do not reset ASS ReadOrder field on flush", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG2_RO_FLUSH_NOOP}, INT_MIN, INT_MAX, S|D, "flags2"},
+{"export_mb_types", "export macroblock types through frame side data", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG2_EXPORT_MB_TYPES}, INT_MIN, INT_MAX, V|D, "flags2"},
 {"time_base", NULL, OFFSET(time_base), AV_OPT_TYPE_RATIONAL, {.dbl = 0}, 0, INT_MAX},
 {"g", "set the group of picture (GOP) size", OFFSET(gop_size), AV_OPT_TYPE_INT, {.i64 = 12 }, INT_MIN, INT_MAX, V|E},
 {"ar", "set audio sampling rate (in Hz)", OFFSET(sample_rate), AV_OPT_TYPE_INT, {.i64 = DEFAULT }, 0, INT_MAX, A|D|E},
diff --git a/libavutil/frame.c b/libavutil/frame.c
index deb9b6f334..577d4f6e6d 100644
--- a/libavutil/frame.c
+++ b/libavutil/frame.c
@@ -834,6 +834,7 @@  const char *av_frame_side_data_name(enum AVFrameSideDataType type)
     case AV_FRAME_DATA_ICC_PROFILE:                 return "ICC profile";
     case AV_FRAME_DATA_QP_TABLE_PROPERTIES:         return "QP table properties";
     case AV_FRAME_DATA_QP_TABLE_DATA:               return "QP table data";
+    case AV_FRAME_DATA_MB_TYPES:                    return "Macroblock types";
     }
     return NULL;
 }
diff --git a/libavutil/frame.h b/libavutil/frame.h
index 9d57d6ce66..ce1231b03b 100644
--- a/libavutil/frame.h
+++ b/libavutil/frame.h
@@ -158,6 +158,15 @@  enum AVFrameSideDataType {
      */
     AV_FRAME_DATA_QP_TABLE_DATA,
 #endif
+
+    /**
+     * Macroblock types exported by some codecs (on demand through the
+     * export_mb_types flag set in the libavcodec AVCodecContext flags2 option).
+     * The data is composed by a header consisting of uint32_t mb_height and
+     * uint32_t mb_width, followed by a uint32_t mb_types[mb_height][mb_width]
+     * array.
+     */
+    AV_FRAME_DATA_MB_TYPES,
 };
 
 enum AVActiveFormatDescription {