diff mbox series

[FFmpeg-devel,08/10] libavutil: add side data AVDnnBoundingBox for dnn based detect/classify filters

Message ID 20210210045854.14964-1-yejun.guo@intel.com
State Superseded
Headers show
Series [FFmpeg-devel,01/10] dnn_backend_openvino.c: fix mismatch between ffmpeg(NHWC) and openvino(NCHW) | expand

Checks

Context Check Description
andriy/x86_make success Make finished
andriy/x86_make_fate fail Make fate failed
andriy/PPC64_make success Make finished
andriy/PPC64_make_fate warning Make fate failed

Commit Message

Guo, Yejun Feb. 10, 2021, 4:58 a.m. UTC
Signed-off-by: Guo, Yejun <yejun.guo@intel.com>
---
 doc/APIchanges       |  2 ++
 libavutil/Makefile   |  1 +
 libavutil/dnn_bbox.h | 68 ++++++++++++++++++++++++++++++++++++++++++++
 libavutil/frame.c    |  1 +
 libavutil/frame.h    |  7 +++++
 libavutil/version.h  |  2 +-
 6 files changed, 80 insertions(+), 1 deletion(-)
 create mode 100644 libavutil/dnn_bbox.h

Comments

Guo, Yejun Feb. 10, 2021, 9:32 a.m. UTC | #1
> -----Original Message-----
> From: Guo, Yejun <yejun.guo@intel.com>
> Sent: 2021年2月10日 12:59
> To: ffmpeg-devel@ffmpeg.org
> Cc: Guo, Yejun <yejun.guo@intel.com>
> Subject: [PATCH 08/10] libavutil: add side data AVDnnBoundingBox for dnn
> based detect/classify filters
> 
> Signed-off-by: Guo, Yejun <yejun.guo@intel.com>
> ---
>  doc/APIchanges       |  2 ++
>  libavutil/Makefile   |  1 +
>  libavutil/dnn_bbox.h | 68
> ++++++++++++++++++++++++++++++++++++++++++++
>  libavutil/frame.c    |  1 +
>  libavutil/frame.h    |  7 +++++
>  libavutil/version.h  |  2 +-
>  6 files changed, 80 insertions(+), 1 deletion(-)  create mode 100644
> libavutil/dnn_bbox.h
> 
> diff --git a/doc/APIchanges b/doc/APIchanges index 1332694820..7cbfa9fafa
> 100644
> --- a/doc/APIchanges
> +++ b/doc/APIchanges
> @@ -14,6 +14,8 @@ libavutil:     2017-10-21
> 
> 
>  API changes, most recent first:
> +2021-02-08 - xxxxxxxxxx - lavu 56.65.100 - frame.h
> +  Add AV_FRAME_DATA_DNN_BBOXES
> 
>  2021-01-26 - xxxxxxxxxx - lavu 56.64.100 - common.h
>    Add FFABSU()
> diff --git a/libavutil/Makefile b/libavutil/Makefile index
> 27bafe9e12..b99cb7084f 100644
> --- a/libavutil/Makefile
> +++ b/libavutil/Makefile
> @@ -23,6 +23,7 @@ HEADERS = adler32.h
> \
>            des.h
> \
>            dict.h
> \
>            display.h
> \
> +          dnn_bbox.h
> \
>            dovi_meta.h
> \
>            downmix_info.h
> \
>            encryption_info.h
> \
> diff --git a/libavutil/dnn_bbox.h b/libavutil/dnn_bbox.h new file mode 100644
> index 0000000000..ae68f9e2e2
> --- /dev/null
> +++ b/libavutil/dnn_bbox.h
> @@ -0,0 +1,68 @@
> +/*
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
> +02110-1301 USA  */
> +
> +#ifndef AVFILTER_DNN_BBOX_H
> +#define AVFILTER_DNN_BBOX_H

the macro name is not correct and caught by patchwork,
will fix it and send the whole patch set as V2, thanks.
diff mbox series

Patch

diff --git a/doc/APIchanges b/doc/APIchanges
index 1332694820..7cbfa9fafa 100644
--- a/doc/APIchanges
+++ b/doc/APIchanges
@@ -14,6 +14,8 @@  libavutil:     2017-10-21
 
 
 API changes, most recent first:
+2021-02-08 - xxxxxxxxxx - lavu 56.65.100 - frame.h
+  Add AV_FRAME_DATA_DNN_BBOXES
 
 2021-01-26 - xxxxxxxxxx - lavu 56.64.100 - common.h
   Add FFABSU()
diff --git a/libavutil/Makefile b/libavutil/Makefile
index 27bafe9e12..b99cb7084f 100644
--- a/libavutil/Makefile
+++ b/libavutil/Makefile
@@ -23,6 +23,7 @@  HEADERS = adler32.h                                                     \
           des.h                                                         \
           dict.h                                                        \
           display.h                                                     \
+          dnn_bbox.h                                                    \
           dovi_meta.h                                                   \
           downmix_info.h                                                \
           encryption_info.h                                             \
diff --git a/libavutil/dnn_bbox.h b/libavutil/dnn_bbox.h
new file mode 100644
index 0000000000..ae68f9e2e2
--- /dev/null
+++ b/libavutil/dnn_bbox.h
@@ -0,0 +1,68 @@ 
+/*
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVFILTER_DNN_BBOX_H
+#define AVFILTER_DNN_BBOX_H
+
+#include "rational.h"
+
+typedef struct AVDnnBoundingBox {
+    /**
+     * Must be set to the size of this data structure (that is,
+     * sizeof(AVDnnBoundingBox)).
+     */
+    uint32_t self_size;
+
+    /**
+     * Object detection is usually applied to a smaller image that
+     * is scaled down from the original frame.
+     * width and height are attributes of the scaled image, in pixel.
+     */
+    int model_input_width;
+    int model_input_height;
+
+    /**
+     * Distance in pixels from the top edge of the scaled image to top
+     * and bottom, and from the left edge of the scaled image to left and
+     * right, defining the bounding box.
+     */
+    int top;
+    int left;
+    int bottom;
+    int right;
+
+    /**
+     * Detect result
+     */
+    int detect_label;
+    AVRational detect_conf;
+
+    /**
+     * At most 4 classifications based on the detected bounding box.
+     * For example, we can get max 4 different attributes with 4 different
+     * DNN models on one bounding box.
+     * classify_count is zero if no classification.
+     */
+#define AV_NUM_BBOX_CLASSIFY 4
+    uint32_t classify_count;
+    int classify_labels[AV_NUM_BBOX_CLASSIFY];
+    AVRational classify_confs[AV_NUM_BBOX_CLASSIFY];
+} AVDnnBoundingBox;
+
+#endif
diff --git a/libavutil/frame.c b/libavutil/frame.c
index eab51b6a32..4308507827 100644
--- a/libavutil/frame.c
+++ b/libavutil/frame.c
@@ -852,6 +852,7 @@  const char *av_frame_side_data_name(enum AVFrameSideDataType type)
     case AV_FRAME_DATA_VIDEO_ENC_PARAMS:            return "Video encoding parameters";
     case AV_FRAME_DATA_SEI_UNREGISTERED:            return "H.26[45] User Data Unregistered SEI message";
     case AV_FRAME_DATA_FILM_GRAIN_PARAMS:           return "Film grain parameters";
+    case AV_FRAME_DATA_DNN_BBOXES:                  return "DNN bounding boxes";
     }
     return NULL;
 }
diff --git a/libavutil/frame.h b/libavutil/frame.h
index 1aeafef6de..a4dcfd27c9 100644
--- a/libavutil/frame.h
+++ b/libavutil/frame.h
@@ -198,6 +198,13 @@  enum AVFrameSideDataType {
      * Must be present for every frame which should have film grain applied.
      */
     AV_FRAME_DATA_FILM_GRAIN_PARAMS,
+
+    /**
+     * Bounding box generated by dnn based filters for object detection and classification,
+     * the data is an array of AVDnnBoudingBox, the number of array element is implied by
+     * AVFrameSideData.size / AVDnnBoudingBox.self_size.
+     */
+    AV_FRAME_DATA_DNN_BBOXES,
 };
 
 enum AVActiveFormatDescription {
diff --git a/libavutil/version.h b/libavutil/version.h
index 21136e6cb7..b2165754f9 100644
--- a/libavutil/version.h
+++ b/libavutil/version.h
@@ -79,7 +79,7 @@ 
  */
 
 #define LIBAVUTIL_VERSION_MAJOR  56
-#define LIBAVUTIL_VERSION_MINOR  64
+#define LIBAVUTIL_VERSION_MINOR  65
 #define LIBAVUTIL_VERSION_MICRO 100
 
 #define LIBAVUTIL_VERSION_INT   AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \