Message ID | 20190408071010.12955-3-zachary.zhou@intel.com |
---|---|
State | Superseded |
Headers | show |
> -----Original Message----- > From: Zhou, Zachary > Sent: Monday, April 8, 2019 3:10 PM > To: ffmpeg-devel@ffmpeg.org > Cc: Zhou, Zachary <zachary.zhou@intel.com> > Subject: [PATCH v1 3/3] libavutil: Add thumbnail output to vaapi_h264 > decoder > > This is sample code for reference > > HW support for decode+scaling in a single HW command (VDBOX+SFC). > The primary target usage is video analytics, but can be used playback, > transcoding, etc. > > For VAAPI - > https://github.com/intel/libva > basically, it allows multiple outputs (in different resolutions) using the decode > context in a single call (you can search for “additional_outputs” in va.h). > > VAAPI sample code - > https://github.com/intel/libva- > utils/commit/957a269f02b00760b7e807643c821ee26abc529b > --- > libavutil/frame.c | 1 + > libavutil/frame.h | 13 +++++++ > libavutil/hwcontext_vaapi.c | 73 ++++++++++++++++++++++++++++++++++++- > libavutil/hwcontext_vaapi.h | 6 +++ > 4 files changed, 92 insertions(+), 1 deletion(-) > > diff --git a/libavutil/frame.c b/libavutil/frame.c index dcf1fc3d17..aa9cd744ae > 100644 > --- a/libavutil/frame.c > +++ b/libavutil/frame.c > @@ -842,6 +842,7 @@ const char *av_frame_side_data_name(enum > AVFrameSideDataType type) #endif > case AV_FRAME_DATA_DYNAMIC_HDR_PLUS: return "HDR Dynamic > Metadata SMPTE2094-40 (HDR10+)"; > case AV_FRAME_DATA_REGIONS_OF_INTEREST: return "Regions Of > Interest"; > + case AV_FRAME_DATA_SFC_INFO: return "SFC Information"; > } > return NULL; > } > diff --git a/libavutil/frame.h b/libavutil/frame.h index > 8aa3e88367..616e7d9d4e 100644 > --- a/libavutil/frame.h > +++ b/libavutil/frame.h > @@ -179,6 +179,11 @@ enum AVFrameSideDataType { > * array element is implied by AVFrameSideData.size / > AVRegionOfInterest.self_size. > */ > AV_FRAME_DATA_REGIONS_OF_INTEREST, > + > + /** > + * VDENC SFC information. > + */ > + AV_FRAME_DATA_SFC_INFO, > }; > > enum AVActiveFormatDescription { > @@ -235,6 +240,14 @@ typedef struct AVRegionOfInterest { > AVRational qoffset; > } AVRegionOfInterest; > > +/** > + * Structure to hold Information of SFC. > + * > + */ > +typedef struct AVSFCInfo { > + uint8_t *data; > +} AVSFCInfo; > + > /** > * This structure describes decoded (raw) audio or video data. > * > diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c index > 8624369bb9..caa6d02b15 100644 > --- a/libavutil/hwcontext_vaapi.c > +++ b/libavutil/hwcontext_vaapi.c > @@ -642,8 +642,74 @@ static void > vaapi_frames_uninit(AVHWFramesContext *hwfc) > av_freep(&ctx->attributes); > } > > +static int vaapi_get_sfc_buffer(AVHWFramesContext *hwfc, AVFrame > +*frame) { > + AVVAAPIFramesContext *avfc = hwfc->hwctx; > + AVFrameSideData *frame_sfc_sd = av_frame_get_side_data(frame, > + AV_FRAME_DATA_SFC_INFO); > + if (frame_sfc_sd) { > + AVVAAPIDeviceContext *hwctx = hwfc->device_ctx->hwctx; > + VASurfaceID sfc_surface_id; > + VASurfaceAttrib attrib; > + unsigned int rt_format; > + AVBufferRef *ref; > + VAStatus vas; > + > + memset(&attrib, 0, sizeof(attrib)); > + attrib.type = VASurfaceAttribPixelFormat; > + attrib.flags = VA_SURFACE_ATTRIB_SETTABLE; > + attrib.value.type = VAGenericValueTypeInteger; > + > + switch (avfc->sfc_format) { > + case AV_PIX_FMT_NV12: > + attrib.value.value.i = VA_FOURCC_NV12; > + rt_format = VA_RT_FORMAT_YUV420; > + break; > + case AV_PIX_FMT_ARGB: > + attrib.value.value.i = VA_FOURCC_ARGB; > + rt_format = VA_RT_FORMAT_RGB32; > + break; > + default: > + av_log(hwfc, AV_LOG_ERROR, "Unknown sfc format: %d\n", avfc- > >sfc_format); > + return AVERROR(EINVAL); > + } > + > + vas = vaCreateSurfaces(hwctx->display, rt_format, > + avfc->sfc_width, avfc->sfc_height, > + &sfc_surface_id, 1, > + &attrib, 1); > + if (vas != VA_STATUS_SUCCESS) { > + av_log(hwfc, AV_LOG_ERROR, "Failed to create sfc surface: " > + "%d (%s).\n", vas, vaErrorStr(vas)); > + return AVERROR(EIO); > + } > + > + av_log(hwfc, AV_LOG_DEBUG, "Created sfc surface %#x. sfc_format(%d), > w(%d), h(%d)\n", > + sfc_surface_id, avfc->sfc_format, avfc->sfc_width, > + avfc->sfc_height); > + > + ref = av_buffer_create((uint8_t*)(uintptr_t)sfc_surface_id, > + sizeof(sfc_surface_id), &vaapi_buffer_free, > + hwfc, AV_BUFFER_FLAG_READONLY); > + if (!ref) { > + vaDestroySurfaces(hwctx->display, &sfc_surface_id, 1); > + return AVERROR(EIO); > + } > + > + frame_sfc_sd->buf = ref; > + > + av_log(hwfc, AV_LOG_DEBUG, "Created sfc av buffer ref: %p\n", > + ref); > + > + } else { > + av_log(hwfc, AV_LOG_DEBUG, "VDSFC frame_sfc_sd is null\n"); > + } > + > + return 0; > +} > + > static int vaapi_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame) { > + int err; > + > frame->buf[0] = av_buffer_pool_get(hwfc->pool); > if (!frame->buf[0]) > return AVERROR(ENOMEM); > @@ -653,6 +719,11 @@ static int vaapi_get_buffer(AVHWFramesContext > *hwfc, AVFrame *frame) > frame->width = hwfc->width; > frame->height = hwfc->height; > > + //create sfc surface, put it in side data > + err = vaapi_get_sfc_buffer(hwfc, frame); > + if (err) > + return err; > + > return 0; > } > > @@ -784,7 +855,7 @@ static int vaapi_map_frame(AVHWFramesContext > *hwfc, > // assume for now that the user is not aware of that and would therefore > // prefer not to be given direct-mapped memory if they request read access. > if (ctx->derive_works && dst->format == hwfc->sw_format && > - ((flags & AV_HWFRAME_MAP_DIRECT) || !(flags & > AV_HWFRAME_MAP_READ))) { > + ((flags & AV_HWFRAME_MAP_DIRECT) || (flags & > + AV_HWFRAME_MAP_READ))) { > vas = vaDeriveImage(hwctx->display, surface_id, &map->image); > if (vas != VA_STATUS_SUCCESS) { > av_log(hwfc, AV_LOG_ERROR, "Failed to derive image from " > diff --git a/libavutil/hwcontext_vaapi.h b/libavutil/hwcontext_vaapi.h index > 0b2e071cb3..e2b26ac1da 100644 > --- a/libavutil/hwcontext_vaapi.h > +++ b/libavutil/hwcontext_vaapi.h > @@ -100,6 +100,12 @@ typedef struct AVVAAPIFramesContext { > */ > VASurfaceID *surface_ids; > int nb_surfaces; > + > + // VSFC info > + int sfc_flags; > + int sfc_format; > + int sfc_width; > + int sfc_height; > } AVVAAPIFramesContext; > > /** Please ignore this patch, new version will be send out soon. Sorry for inconvenience. > -- > 2.17.1
diff --git a/libavutil/frame.c b/libavutil/frame.c index dcf1fc3d17..aa9cd744ae 100644 --- a/libavutil/frame.c +++ b/libavutil/frame.c @@ -842,6 +842,7 @@ const char *av_frame_side_data_name(enum AVFrameSideDataType type) #endif case AV_FRAME_DATA_DYNAMIC_HDR_PLUS: return "HDR Dynamic Metadata SMPTE2094-40 (HDR10+)"; case AV_FRAME_DATA_REGIONS_OF_INTEREST: return "Regions Of Interest"; + case AV_FRAME_DATA_SFC_INFO: return "SFC Information"; } return NULL; } diff --git a/libavutil/frame.h b/libavutil/frame.h index 8aa3e88367..616e7d9d4e 100644 --- a/libavutil/frame.h +++ b/libavutil/frame.h @@ -179,6 +179,11 @@ enum AVFrameSideDataType { * array element is implied by AVFrameSideData.size / AVRegionOfInterest.self_size. */ AV_FRAME_DATA_REGIONS_OF_INTEREST, + + /** + * VDENC SFC information. + */ + AV_FRAME_DATA_SFC_INFO, }; enum AVActiveFormatDescription { @@ -235,6 +240,14 @@ typedef struct AVRegionOfInterest { AVRational qoffset; } AVRegionOfInterest; +/** + * Structure to hold Information of SFC. + * + */ +typedef struct AVSFCInfo { + uint8_t *data; +} AVSFCInfo; + /** * This structure describes decoded (raw) audio or video data. * diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c index 8624369bb9..caa6d02b15 100644 --- a/libavutil/hwcontext_vaapi.c +++ b/libavutil/hwcontext_vaapi.c @@ -642,8 +642,74 @@ static void vaapi_frames_uninit(AVHWFramesContext *hwfc) av_freep(&ctx->attributes); } +static int vaapi_get_sfc_buffer(AVHWFramesContext *hwfc, AVFrame *frame) +{ + AVVAAPIFramesContext *avfc = hwfc->hwctx; + AVFrameSideData *frame_sfc_sd = av_frame_get_side_data(frame, + AV_FRAME_DATA_SFC_INFO); + if (frame_sfc_sd) { + AVVAAPIDeviceContext *hwctx = hwfc->device_ctx->hwctx; + VASurfaceID sfc_surface_id; + VASurfaceAttrib attrib; + unsigned int rt_format; + AVBufferRef *ref; + VAStatus vas; + + memset(&attrib, 0, sizeof(attrib)); + attrib.type = VASurfaceAttribPixelFormat; + attrib.flags = VA_SURFACE_ATTRIB_SETTABLE; + attrib.value.type = VAGenericValueTypeInteger; + + switch (avfc->sfc_format) { + case AV_PIX_FMT_NV12: + attrib.value.value.i = VA_FOURCC_NV12; + rt_format = VA_RT_FORMAT_YUV420; + break; + case AV_PIX_FMT_ARGB: + attrib.value.value.i = VA_FOURCC_ARGB; + rt_format = VA_RT_FORMAT_RGB32; + break; + default: + av_log(hwfc, AV_LOG_ERROR, "Unknown sfc format: %d\n", avfc->sfc_format); + return AVERROR(EINVAL); + } + + vas = vaCreateSurfaces(hwctx->display, rt_format, + avfc->sfc_width, avfc->sfc_height, + &sfc_surface_id, 1, + &attrib, 1); + if (vas != VA_STATUS_SUCCESS) { + av_log(hwfc, AV_LOG_ERROR, "Failed to create sfc surface: " + "%d (%s).\n", vas, vaErrorStr(vas)); + return AVERROR(EIO); + } + + av_log(hwfc, AV_LOG_DEBUG, "Created sfc surface %#x. sfc_format(%d), w(%d), h(%d)\n", + sfc_surface_id, avfc->sfc_format, avfc->sfc_width, avfc->sfc_height); + + ref = av_buffer_create((uint8_t*)(uintptr_t)sfc_surface_id, + sizeof(sfc_surface_id), &vaapi_buffer_free, + hwfc, AV_BUFFER_FLAG_READONLY); + if (!ref) { + vaDestroySurfaces(hwctx->display, &sfc_surface_id, 1); + return AVERROR(EIO); + } + + frame_sfc_sd->buf = ref; + + av_log(hwfc, AV_LOG_DEBUG, "Created sfc av buffer ref: %p\n", ref); + + } else { + av_log(hwfc, AV_LOG_DEBUG, "VDSFC frame_sfc_sd is null\n"); + } + + return 0; +} + static int vaapi_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame) { + int err; + frame->buf[0] = av_buffer_pool_get(hwfc->pool); if (!frame->buf[0]) return AVERROR(ENOMEM); @@ -653,6 +719,11 @@ static int vaapi_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame) frame->width = hwfc->width; frame->height = hwfc->height; + //create sfc surface, put it in side data + err = vaapi_get_sfc_buffer(hwfc, frame); + if (err) + return err; + return 0; } @@ -784,7 +855,7 @@ static int vaapi_map_frame(AVHWFramesContext *hwfc, // assume for now that the user is not aware of that and would therefore // prefer not to be given direct-mapped memory if they request read access. if (ctx->derive_works && dst->format == hwfc->sw_format && - ((flags & AV_HWFRAME_MAP_DIRECT) || !(flags & AV_HWFRAME_MAP_READ))) { + ((flags & AV_HWFRAME_MAP_DIRECT) || (flags & AV_HWFRAME_MAP_READ))) { vas = vaDeriveImage(hwctx->display, surface_id, &map->image); if (vas != VA_STATUS_SUCCESS) { av_log(hwfc, AV_LOG_ERROR, "Failed to derive image from " diff --git a/libavutil/hwcontext_vaapi.h b/libavutil/hwcontext_vaapi.h index 0b2e071cb3..e2b26ac1da 100644 --- a/libavutil/hwcontext_vaapi.h +++ b/libavutil/hwcontext_vaapi.h @@ -100,6 +100,12 @@ typedef struct AVVAAPIFramesContext { */ VASurfaceID *surface_ids; int nb_surfaces; + + // VSFC info + int sfc_flags; + int sfc_format; + int sfc_width; + int sfc_height; } AVVAAPIFramesContext; /**