diff mbox

[FFmpeg-devel,v1,3/3] libavutil: Add thumbnail output to vaapi_h264 decoder

Message ID 20190408071010.12955-3-zachary.zhou@intel.com
State Superseded
Headers show

Commit Message

Zachary Zhou April 8, 2019, 7:10 a.m. UTC 
This is sample code for reference

HW support for decode+scaling in a single HW command (VDBOX+SFC).
The primary target usage is video analytics, but can be used playback,
transcoding, etc.

For VAAPI -
https://github.com/intel/libva
basically, it allows multiple outputs (in different resolutions) using the decode context in a single call (you can search for “additional_outputs” in va.h).

VAAPI sample code -
https://github.com/intel/libva-utils/commit/957a269f02b00760b7e807643c821ee26abc529b
---
 libavutil/frame.c           |  1 +
 libavutil/frame.h           | 13 +++++++
 libavutil/hwcontext_vaapi.c | 73 ++++++++++++++++++++++++++++++++++++-
 libavutil/hwcontext_vaapi.h |  6 +++
 4 files changed, 92 insertions(+), 1 deletion(-)

Comments

Zachary Zhou April 8, 2019, 7:48 a.m. UTC | #1 
> -----Original Message-----

> From: Zhou, Zachary

> Sent: Monday, April 8, 2019 3:10 PM

> To: ffmpeg-devel@ffmpeg.org

> Cc: Zhou, Zachary <zachary.zhou@intel.com>

> Subject: [PATCH v1 3/3] libavutil: Add thumbnail output to vaapi_h264

> decoder

> 

> This is sample code for reference

> 

> HW support for decode+scaling in a single HW command (VDBOX+SFC).

> The primary target usage is video analytics, but can be used playback,

> transcoding, etc.

> 

> For VAAPI -

> https://github.com/intel/libva

> basically, it allows multiple outputs (in different resolutions) using the decode

> context in a single call (you can search for “additional_outputs” in va.h).

> 

> VAAPI sample code -

> https://github.com/intel/libva-

> utils/commit/957a269f02b00760b7e807643c821ee26abc529b

> ---

>  libavutil/frame.c           |  1 +

>  libavutil/frame.h           | 13 +++++++

>  libavutil/hwcontext_vaapi.c | 73 ++++++++++++++++++++++++++++++++++++-

>  libavutil/hwcontext_vaapi.h |  6 +++

>  4 files changed, 92 insertions(+), 1 deletion(-)

> 

> diff --git a/libavutil/frame.c b/libavutil/frame.c index dcf1fc3d17..aa9cd744ae

> 100644

> --- a/libavutil/frame.c

> +++ b/libavutil/frame.c

> @@ -842,6 +842,7 @@ const char *av_frame_side_data_name(enum

> AVFrameSideDataType type)  #endif

>      case AV_FRAME_DATA_DYNAMIC_HDR_PLUS: return "HDR Dynamic

> Metadata SMPTE2094-40 (HDR10+)";

>      case AV_FRAME_DATA_REGIONS_OF_INTEREST: return "Regions Of

> Interest";

> +    case AV_FRAME_DATA_SFC_INFO: return "SFC Information";

>      }

>      return NULL;

>  }

> diff --git a/libavutil/frame.h b/libavutil/frame.h index

> 8aa3e88367..616e7d9d4e 100644

> --- a/libavutil/frame.h

> +++ b/libavutil/frame.h

> @@ -179,6 +179,11 @@ enum AVFrameSideDataType {

>       * array element is implied by AVFrameSideData.size /

> AVRegionOfInterest.self_size.

>       */

>      AV_FRAME_DATA_REGIONS_OF_INTEREST,

> +

> +    /**

> +     * VDENC SFC information.

> +     */

> +    AV_FRAME_DATA_SFC_INFO,

>  };

> 

>  enum AVActiveFormatDescription {

> @@ -235,6 +240,14 @@ typedef struct AVRegionOfInterest {

>      AVRational qoffset;

>  } AVRegionOfInterest;

> 

> +/**

> + * Structure to hold Information of SFC.

> + *

> + */

> +typedef struct AVSFCInfo {

> +    uint8_t *data;

> +} AVSFCInfo;

> +

>  /**

>   * This structure describes decoded (raw) audio or video data.

>   *

> diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c index

> 8624369bb9..caa6d02b15 100644

> --- a/libavutil/hwcontext_vaapi.c

> +++ b/libavutil/hwcontext_vaapi.c

> @@ -642,8 +642,74 @@ static void

> vaapi_frames_uninit(AVHWFramesContext *hwfc)

>      av_freep(&ctx->attributes);

>  }

> 

> +static int vaapi_get_sfc_buffer(AVHWFramesContext *hwfc, AVFrame

> +*frame) {

> +    AVVAAPIFramesContext *avfc = hwfc->hwctx;

> +    AVFrameSideData *frame_sfc_sd = av_frame_get_side_data(frame,

> +                                                           AV_FRAME_DATA_SFC_INFO);

> +    if (frame_sfc_sd) {

> +        AVVAAPIDeviceContext *hwctx = hwfc->device_ctx->hwctx;

> +        VASurfaceID sfc_surface_id;

> +        VASurfaceAttrib attrib;

> +        unsigned int rt_format;

> +        AVBufferRef *ref;

> +        VAStatus vas;

> +

> +        memset(&attrib, 0, sizeof(attrib));

> +        attrib.type = VASurfaceAttribPixelFormat;

> +        attrib.flags = VA_SURFACE_ATTRIB_SETTABLE;

> +        attrib.value.type = VAGenericValueTypeInteger;

> +

> +        switch (avfc->sfc_format) {

> +        case AV_PIX_FMT_NV12:

> +            attrib.value.value.i = VA_FOURCC_NV12;

> +            rt_format = VA_RT_FORMAT_YUV420;

> +            break;

> +        case AV_PIX_FMT_ARGB:

> +            attrib.value.value.i = VA_FOURCC_ARGB;

> +            rt_format = VA_RT_FORMAT_RGB32;

> +            break;

> +        default:

> +            av_log(hwfc, AV_LOG_ERROR, "Unknown sfc format: %d\n", avfc-

> >sfc_format);

> +            return AVERROR(EINVAL);

> +        }

> +

> +        vas = vaCreateSurfaces(hwctx->display, rt_format,

> +                               avfc->sfc_width, avfc->sfc_height,

> +                               &sfc_surface_id, 1,

> +                               &attrib, 1);

> +        if (vas != VA_STATUS_SUCCESS) {

> +            av_log(hwfc, AV_LOG_ERROR, "Failed to create sfc surface: "

> +                   "%d (%s).\n", vas, vaErrorStr(vas));

> +            return AVERROR(EIO);

> +        }

> +

> +        av_log(hwfc, AV_LOG_DEBUG, "Created sfc surface %#x. sfc_format(%d),

> w(%d), h(%d)\n",

> +               sfc_surface_id, avfc->sfc_format, avfc->sfc_width,

> + avfc->sfc_height);

> +

> +        ref = av_buffer_create((uint8_t*)(uintptr_t)sfc_surface_id,

> +                               sizeof(sfc_surface_id), &vaapi_buffer_free,

> +                               hwfc, AV_BUFFER_FLAG_READONLY);

> +        if (!ref) {

> +            vaDestroySurfaces(hwctx->display, &sfc_surface_id, 1);

> +            return AVERROR(EIO);

> +        }

> +

> +        frame_sfc_sd->buf = ref;

> +

> +        av_log(hwfc, AV_LOG_DEBUG, "Created sfc av buffer ref: %p\n",

> + ref);

> +

> +    } else {

> +        av_log(hwfc, AV_LOG_DEBUG, "VDSFC frame_sfc_sd is null\n");

> +    }

> +

> +    return 0;

> +}

> +

>  static int vaapi_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)  {

> +    int err;

> +

>      frame->buf[0] = av_buffer_pool_get(hwfc->pool);

>      if (!frame->buf[0])

>          return AVERROR(ENOMEM);

> @@ -653,6 +719,11 @@ static int vaapi_get_buffer(AVHWFramesContext

> *hwfc, AVFrame *frame)

>      frame->width   = hwfc->width;

>      frame->height  = hwfc->height;

> 

> +    //create sfc surface, put it in side data

> +    err = vaapi_get_sfc_buffer(hwfc, frame);

> +    if (err)

> +        return err;

> +

>      return 0;

>  }

> 

> @@ -784,7 +855,7 @@ static int vaapi_map_frame(AVHWFramesContext

> *hwfc,

>      // assume for now that the user is not aware of that and would therefore

>      // prefer not to be given direct-mapped memory if they request read access.

>      if (ctx->derive_works && dst->format == hwfc->sw_format &&

> -        ((flags & AV_HWFRAME_MAP_DIRECT) || !(flags &

> AV_HWFRAME_MAP_READ))) {

> +        ((flags & AV_HWFRAME_MAP_DIRECT) || (flags &

> + AV_HWFRAME_MAP_READ))) {

>          vas = vaDeriveImage(hwctx->display, surface_id, &map->image);

>          if (vas != VA_STATUS_SUCCESS) {

>              av_log(hwfc, AV_LOG_ERROR, "Failed to derive image from "

> diff --git a/libavutil/hwcontext_vaapi.h b/libavutil/hwcontext_vaapi.h index

> 0b2e071cb3..e2b26ac1da 100644

> --- a/libavutil/hwcontext_vaapi.h

> +++ b/libavutil/hwcontext_vaapi.h

> @@ -100,6 +100,12 @@ typedef struct AVVAAPIFramesContext {

>       */

>      VASurfaceID     *surface_ids;

>      int           nb_surfaces;

> +

> +    // VSFC info

> +    int sfc_flags;

> +    int sfc_format;

> +    int sfc_width;

> +    int sfc_height;

>  } AVVAAPIFramesContext;

> 

>  /**


Please ignore this patch, new version will be send out soon.
Sorry for inconvenience.

> --

> 2.17.1
diff mbox

Patch

diff --git a/libavutil/frame.c b/libavutil/frame.c
index dcf1fc3d17..aa9cd744ae 100644
--- a/libavutil/frame.c
+++ b/libavutil/frame.c
@@ -842,6 +842,7 @@  const char *av_frame_side_data_name(enum AVFrameSideDataType type)
 #endif
     case AV_FRAME_DATA_DYNAMIC_HDR_PLUS: return "HDR Dynamic Metadata SMPTE2094-40 (HDR10+)";
     case AV_FRAME_DATA_REGIONS_OF_INTEREST: return "Regions Of Interest";
+    case AV_FRAME_DATA_SFC_INFO: return "SFC Information";
     }
     return NULL;
 }
diff --git a/libavutil/frame.h b/libavutil/frame.h
index 8aa3e88367..616e7d9d4e 100644
--- a/libavutil/frame.h
+++ b/libavutil/frame.h
@@ -179,6 +179,11 @@  enum AVFrameSideDataType {
      * array element is implied by AVFrameSideData.size / AVRegionOfInterest.self_size.
      */
     AV_FRAME_DATA_REGIONS_OF_INTEREST,
+
+    /**
+     * VDENC SFC information.
+     */
+    AV_FRAME_DATA_SFC_INFO,
 };
 
 enum AVActiveFormatDescription {
@@ -235,6 +240,14 @@  typedef struct AVRegionOfInterest {
     AVRational qoffset;
 } AVRegionOfInterest;
 
+/**
+ * Structure to hold Information of SFC.
+ *
+ */
+typedef struct AVSFCInfo {
+    uint8_t *data;
+} AVSFCInfo;
+
 /**
  * This structure describes decoded (raw) audio or video data.
  *
diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c
index 8624369bb9..caa6d02b15 100644
--- a/libavutil/hwcontext_vaapi.c
+++ b/libavutil/hwcontext_vaapi.c
@@ -642,8 +642,74 @@  static void vaapi_frames_uninit(AVHWFramesContext *hwfc)
     av_freep(&ctx->attributes);
 }
 
+static int vaapi_get_sfc_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
+{
+    AVVAAPIFramesContext *avfc = hwfc->hwctx;
+    AVFrameSideData *frame_sfc_sd = av_frame_get_side_data(frame,
+                                                           AV_FRAME_DATA_SFC_INFO);
+    if (frame_sfc_sd) {
+        AVVAAPIDeviceContext *hwctx = hwfc->device_ctx->hwctx;
+        VASurfaceID sfc_surface_id;
+        VASurfaceAttrib attrib;
+        unsigned int rt_format;
+        AVBufferRef *ref;
+        VAStatus vas;
+
+        memset(&attrib, 0, sizeof(attrib));
+        attrib.type = VASurfaceAttribPixelFormat;
+        attrib.flags = VA_SURFACE_ATTRIB_SETTABLE;
+        attrib.value.type = VAGenericValueTypeInteger;
+
+        switch (avfc->sfc_format) {
+        case AV_PIX_FMT_NV12:
+            attrib.value.value.i = VA_FOURCC_NV12;
+            rt_format = VA_RT_FORMAT_YUV420;
+            break;
+        case AV_PIX_FMT_ARGB:
+            attrib.value.value.i = VA_FOURCC_ARGB;
+            rt_format = VA_RT_FORMAT_RGB32;
+            break;
+        default:
+            av_log(hwfc, AV_LOG_ERROR, "Unknown sfc format: %d\n", avfc->sfc_format);
+            return AVERROR(EINVAL);
+        }
+
+        vas = vaCreateSurfaces(hwctx->display, rt_format,
+                               avfc->sfc_width, avfc->sfc_height,
+                               &sfc_surface_id, 1,
+                               &attrib, 1);
+        if (vas != VA_STATUS_SUCCESS) {
+            av_log(hwfc, AV_LOG_ERROR, "Failed to create sfc surface: "
+                   "%d (%s).\n", vas, vaErrorStr(vas));
+            return AVERROR(EIO);
+        }
+
+        av_log(hwfc, AV_LOG_DEBUG, "Created sfc surface %#x. sfc_format(%d), w(%d), h(%d)\n",
+               sfc_surface_id, avfc->sfc_format, avfc->sfc_width, avfc->sfc_height);
+
+        ref = av_buffer_create((uint8_t*)(uintptr_t)sfc_surface_id,
+                               sizeof(sfc_surface_id), &vaapi_buffer_free,
+                               hwfc, AV_BUFFER_FLAG_READONLY);
+        if (!ref) {
+            vaDestroySurfaces(hwctx->display, &sfc_surface_id, 1);
+            return AVERROR(EIO);
+        }
+
+        frame_sfc_sd->buf = ref;
+
+        av_log(hwfc, AV_LOG_DEBUG, "Created sfc av buffer ref: %p\n", ref);
+
+    } else {
+        av_log(hwfc, AV_LOG_DEBUG, "VDSFC frame_sfc_sd is null\n");
+    }
+
+    return 0;
+}
+
 static int vaapi_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
 {
+    int err;
+
     frame->buf[0] = av_buffer_pool_get(hwfc->pool);
     if (!frame->buf[0])
         return AVERROR(ENOMEM);
@@ -653,6 +719,11 @@  static int vaapi_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
     frame->width   = hwfc->width;
     frame->height  = hwfc->height;
 
+    //create sfc surface, put it in side data
+    err = vaapi_get_sfc_buffer(hwfc, frame);
+    if (err)
+        return err;
+
     return 0;
 }
 
@@ -784,7 +855,7 @@  static int vaapi_map_frame(AVHWFramesContext *hwfc,
     // assume for now that the user is not aware of that and would therefore
     // prefer not to be given direct-mapped memory if they request read access.
     if (ctx->derive_works && dst->format == hwfc->sw_format &&
-        ((flags & AV_HWFRAME_MAP_DIRECT) || !(flags & AV_HWFRAME_MAP_READ))) {
+        ((flags & AV_HWFRAME_MAP_DIRECT) || (flags & AV_HWFRAME_MAP_READ))) {
         vas = vaDeriveImage(hwctx->display, surface_id, &map->image);
         if (vas != VA_STATUS_SUCCESS) {
             av_log(hwfc, AV_LOG_ERROR, "Failed to derive image from "
diff --git a/libavutil/hwcontext_vaapi.h b/libavutil/hwcontext_vaapi.h
index 0b2e071cb3..e2b26ac1da 100644
--- a/libavutil/hwcontext_vaapi.h
+++ b/libavutil/hwcontext_vaapi.h
@@ -100,6 +100,12 @@  typedef struct AVVAAPIFramesContext {
      */
     VASurfaceID     *surface_ids;
     int           nb_surfaces;
+
+    // VSFC info
+    int sfc_flags;
+    int sfc_format;
+    int sfc_width;
+    int sfc_height;
 } AVVAAPIFramesContext;
 
 /**