Message ID | 20210629220816.97586-1-ffmpeg@tmm1.net |
---|---|
State | Accepted |
Headers | show |
Series | [FFmpeg-devel] avutil/hwcontext_videotoolbox: implement hwupload to convert AVFrame to CVPixelBuffer | expand |
Context | Check | Description |
---|---|---|
andriy/x86_make | fail | Make failed |
andriy/PPC64_make | success | Make finished |
andriy/PPC64_make_fate | success | Make fate finished |
On Tue, Jun 29, 2021 at 3:08 PM Aman Karmani <ffmpeg@tmm1.net> wrote: > From: Aman Karmani <aman@tmm1.net> > > Teach AV_HWDEVICE_TYPE_VIDEOTOOLBOX to be able to create AVFrames of type > AV_PIX_FMT_VIDEOTOOLBOX. This can be used to hwupload a regular AVFrame > into its CVPixelBuffer equivalent. > > ffmpeg -init_hw_device videotoolbox -f lavfi -i color=black:640x480 > -vf hwupload -c:v h264_videotoolbox -f null -y /dev/null Will merge this in a few days if no one objects. Do I need to bump avutil patch or minor? Aman > > Signed-off-by: Aman Karmani <aman@tmm1.net> > --- > libavutil/hwcontext_videotoolbox.c | 418 ++++++++++++++++++++++++++++- > libavutil/hwcontext_videotoolbox.h | 7 +- > 2 files changed, 420 insertions(+), 5 deletions(-) > > diff --git a/libavutil/hwcontext_videotoolbox.c > b/libavutil/hwcontext_videotoolbox.c > index bded9873fe..58095a1fc9 100644 > --- a/libavutil/hwcontext_videotoolbox.c > +++ b/libavutil/hwcontext_videotoolbox.c > @@ -24,6 +24,7 @@ > #include <VideoToolbox/VideoToolbox.h> > > #include "buffer.h" > +#include "buffer_internal.h" > #include "common.h" > #include "hwcontext.h" > #include "hwcontext_internal.h" > @@ -32,6 +33,10 @@ > #include "pixfmt.h" > #include "pixdesc.h" > > +typedef struct VTFramesContext { > + CVPixelBufferPoolRef pool; > +} VTFramesContext; > + > static const struct { > uint32_t cv_fmt; > bool full_range; > @@ -50,6 +55,39 @@ static const struct { > #endif > }; > > +static const enum AVPixelFormat supported_formats[] = { > + AV_PIX_FMT_NV12, > + AV_PIX_FMT_YUV420P, > + AV_PIX_FMT_UYVY422, > + AV_PIX_FMT_P010, > + AV_PIX_FMT_BGRA, > +}; > + > +static int vt_frames_get_constraints(AVHWDeviceContext *ctx, > + const void *hwconfig, > + AVHWFramesConstraints *constraints) > +{ > + int i; > + > + constraints->valid_sw_formats = > av_malloc_array(FF_ARRAY_ELEMS(supported_formats) + 1, > + > sizeof(*constraints->valid_sw_formats)); > + if (!constraints->valid_sw_formats) > + return AVERROR(ENOMEM); > + > + for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++) > + constraints->valid_sw_formats[i] = supported_formats[i]; > + constraints->valid_sw_formats[FF_ARRAY_ELEMS(supported_formats)] = > AV_PIX_FMT_NONE; > + > + constraints->valid_hw_formats = av_malloc_array(2, > sizeof(*constraints->valid_hw_formats)); > + if (!constraints->valid_hw_formats) > + return AVERROR(ENOMEM); > + > + constraints->valid_hw_formats[0] = AV_PIX_FMT_VIDEOTOOLBOX; > + constraints->valid_hw_formats[1] = AV_PIX_FMT_NONE; > + > + return 0; > +} > + > enum AVPixelFormat av_map_videotoolbox_format_to_pixfmt(uint32_t cv_fmt) > { > int i; > @@ -75,11 +113,134 @@ uint32_t > av_map_videotoolbox_format_from_pixfmt2(enum AVPixelFormat pix_fmt, boo > return 0; > } > > +static int vt_pool_alloc(AVHWFramesContext *ctx) > +{ > + VTFramesContext *fctx = ctx->internal->priv; > + CVReturn err; > + CFNumberRef w, h, pixfmt; > + uint32_t cv_pixfmt; > + CFMutableDictionaryRef attributes, iosurface_properties; > + > + attributes = CFDictionaryCreateMutable( > + NULL, > + 2, > + &kCFTypeDictionaryKeyCallBacks, > + &kCFTypeDictionaryValueCallBacks); > + > + cv_pixfmt = av_map_videotoolbox_format_from_pixfmt(ctx->sw_format); > + pixfmt = CFNumberCreate(NULL, kCFNumberSInt32Type, &cv_pixfmt); > + CFDictionarySetValue( > + attributes, > + kCVPixelBufferPixelFormatTypeKey, > + pixfmt); > + CFRelease(pixfmt); > + > + iosurface_properties = CFDictionaryCreateMutable( > + NULL, > + 0, > + &kCFTypeDictionaryKeyCallBacks, > + &kCFTypeDictionaryValueCallBacks); > + CFDictionarySetValue(attributes, > kCVPixelBufferIOSurfacePropertiesKey, iosurface_properties); > + CFRelease(iosurface_properties); > + > + w = CFNumberCreate(NULL, kCFNumberSInt32Type, &ctx->width); > + h = CFNumberCreate(NULL, kCFNumberSInt32Type, &ctx->height); > + CFDictionarySetValue(attributes, kCVPixelBufferWidthKey, w); > + CFDictionarySetValue(attributes, kCVPixelBufferHeightKey, h); > + CFRelease(w); > + CFRelease(h); > + > + err = CVPixelBufferPoolCreate( > + NULL, > + NULL, > + attributes, > + &fctx->pool); > + CFRelease(attributes); > + > + if (err == kCVReturnSuccess) > + return 0; > + > + av_log(ctx, AV_LOG_ERROR, "Error creating CVPixelBufferPool: %d\n", > err); > + return AVERROR_EXTERNAL; > +} > + > +static AVBufferRef *vt_dummy_pool_alloc(void *opaque, size_t size) > +{ > + return NULL; > +} > + > +static void vt_frames_uninit(AVHWFramesContext *ctx) > +{ > + VTFramesContext *fctx = ctx->internal->priv; > + if (fctx->pool) { > + CVPixelBufferPoolRelease(fctx->pool); > + fctx->pool = NULL; > + } > +} > + > +static int vt_frames_init(AVHWFramesContext *ctx) > +{ > + int i, ret; > + > + for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++) { > + if (ctx->sw_format == supported_formats[i]) > + break; > + } > + if (i == FF_ARRAY_ELEMS(supported_formats)) { > + av_log(ctx, AV_LOG_ERROR, "Pixel format '%s' is not supported\n", > + av_get_pix_fmt_name(ctx->sw_format)); > + return AVERROR(ENOSYS); > + } > + > + // create a dummy pool so av_hwframe_get_buffer doesn't EINVAL > + if (!ctx->pool) { > + ctx->internal->pool_internal = av_buffer_pool_init2(0, ctx, > vt_dummy_pool_alloc, NULL); > + if (!ctx->internal->pool_internal) > + return AVERROR(ENOMEM); > + } > + > + ret = vt_pool_alloc(ctx); > + if (ret < 0) > + return ret; > + > + return 0; > +} > + > +static void videotoolbox_buffer_release(void *opaque, uint8_t *data) > +{ > + CVPixelBufferRelease((CVPixelBufferRef)data); > +} > + > static int vt_get_buffer(AVHWFramesContext *ctx, AVFrame *frame) > { > - frame->buf[0] = av_buffer_pool_get(ctx->pool); > - if (!frame->buf[0]) > - return AVERROR(ENOMEM); > + VTFramesContext *fctx = ctx->internal->priv; > + > + if (ctx->pool && ctx->pool->size != 0) { > + frame->buf[0] = av_buffer_pool_get(ctx->pool); > + if (!frame->buf[0]) > + return AVERROR(ENOMEM); > + } else { > + CVPixelBufferRef pixbuf; > + AVBufferRef *buf = NULL; > + CVReturn err; > + > + err = CVPixelBufferPoolCreatePixelBuffer( > + NULL, > + fctx->pool, > + &pixbuf > + ); > + if (err != kCVReturnSuccess) { > + av_log(ctx, AV_LOG_ERROR, "Failed to create pixel buffer from > pool: %d\n", err); > + return AVERROR_EXTERNAL; > + } > + > + buf = av_buffer_create((uint8_t *)pixbuf, 1, > videotoolbox_buffer_release, NULL, 0); > + if (!buf) { > + CVPixelBufferRelease(pixbuf); > + return AVERROR(ENOMEM); > + } > + frame->buf[0] = buf; > + } > > frame->data[3] = frame->buf[0]->data; > frame->format = AV_PIX_FMT_VIDEOTOOLBOX; > @@ -111,6 +272,248 @@ static void vt_unmap(AVHWFramesContext *ctx, > HWMapDescriptor *hwmap) > CVPixelBufferUnlockBaseAddress(pixbuf, (uintptr_t)hwmap->priv); > } > > +static int vt_pixbuf_set_par(AVHWFramesContext *hwfc, > + CVPixelBufferRef pixbuf, const AVFrame *src) > +{ > + CFMutableDictionaryRef par = NULL; > + CFNumberRef num = NULL, den = NULL; > + AVRational avpar = src->sample_aspect_ratio; > + > + if (avpar.num == 0) > + return 0; > + > + av_reduce(&avpar.num, &avpar.den, > + avpar.num, avpar.den, > + 0xFFFFFFFF); > + > + num = CFNumberCreate(kCFAllocatorDefault, > + kCFNumberIntType, > + &avpar.num); > + > + den = CFNumberCreate(kCFAllocatorDefault, > + kCFNumberIntType, > + &avpar.den); > + > + par = CFDictionaryCreateMutable(kCFAllocatorDefault, > + 2, > + &kCFCopyStringDictionaryKeyCallBacks, > + &kCFTypeDictionaryValueCallBacks); > + > + if (!par || !num || !den) { > + if (par) CFRelease(par); > + if (num) CFRelease(num); > + if (den) CFRelease(den); > + return AVERROR(ENOMEM); > + } > + > + CFDictionarySetValue( > + par, > + kCVImageBufferPixelAspectRatioHorizontalSpacingKey, > + num); > + CFDictionarySetValue( > + par, > + kCVImageBufferPixelAspectRatioVerticalSpacingKey, > + den); > + > + CVBufferSetAttachment( > + pixbuf, > + kCVImageBufferPixelAspectRatioKey, > + par, > + kCVAttachmentMode_ShouldPropagate > + ); > + > + CFRelease(par); > + CFRelease(num); > + CFRelease(den); > + > + return 0; > +} > + > +static int vt_pixbuf_set_chromaloc(AVHWFramesContext *hwfc, > + CVPixelBufferRef pixbuf, const AVFrame > *src) > +{ > + CFStringRef loc = NULL; > + > + switch (src->chroma_location) { > + case AVCHROMA_LOC_LEFT: > + loc = kCVImageBufferChromaLocation_Left; > + break; > + case AVCHROMA_LOC_CENTER: > + loc = kCVImageBufferChromaLocation_Center; > + break; > + case AVCHROMA_LOC_TOP: > + loc = kCVImageBufferChromaLocation_Top; > + break; > + case AVCHROMA_LOC_BOTTOM: > + loc = kCVImageBufferChromaLocation_Bottom; > + break; > + case AVCHROMA_LOC_TOPLEFT: > + loc = kCVImageBufferChromaLocation_TopLeft; > + break; > + case AVCHROMA_LOC_BOTTOMLEFT: > + loc = kCVImageBufferChromaLocation_BottomLeft; > + break; > + } > + > + if (loc) { > + CVBufferSetAttachment( > + pixbuf, > + kCVImageBufferChromaLocationTopFieldKey, > + loc, > + kCVAttachmentMode_ShouldPropagate); > + } > + > + return 0; > +} > + > +static int vt_pixbuf_set_colorspace(AVHWFramesContext *hwfc, > + CVPixelBufferRef pixbuf, const > AVFrame *src) > +{ > + CFStringRef colormatrix = NULL, colorpri = NULL, colortrc = NULL; > + Float32 gamma = 0; > + > + switch (src->colorspace) { > + case AVCOL_SPC_BT2020_CL: > + case AVCOL_SPC_BT2020_NCL: > + if (__builtin_available(macOS 10.11, *)) > + colormatrix = kCVImageBufferYCbCrMatrix_ITU_R_2020; > + else > + colormatrix = CFSTR("ITU_R_2020"); > + break; > + case AVCOL_SPC_BT470BG: > + case AVCOL_SPC_SMPTE170M: > + colormatrix = kCVImageBufferYCbCrMatrix_ITU_R_601_4; > + break; > + case AVCOL_SPC_BT709: > + colormatrix = kCVImageBufferYCbCrMatrix_ITU_R_709_2; > + break; > + case AVCOL_SPC_SMPTE240M: > + colormatrix = kCVImageBufferYCbCrMatrix_SMPTE_240M_1995; > + break; > + case AVCOL_SPC_UNSPECIFIED: > + break; > + default: > + av_log(hwfc, AV_LOG_WARNING, "Color space %s is not > supported.\n", av_color_space_name(src->colorspace)); > + } > + > + switch (src->color_primaries) { > + case AVCOL_PRI_BT2020: > + if (__builtin_available(macOS 10.11, *)) > + colorpri = kCVImageBufferColorPrimaries_ITU_R_2020; > + else > + colorpri = CFSTR("ITU_R_2020"); > + break; > + case AVCOL_PRI_BT709: > + colorpri = kCVImageBufferColorPrimaries_ITU_R_709_2; > + break; > + case AVCOL_PRI_SMPTE170M: > + colorpri = kCVImageBufferColorPrimaries_SMPTE_C; > + break; > + case AVCOL_PRI_BT470BG: > + colorpri = kCVImageBufferColorPrimaries_EBU_3213; > + break; > + case AVCOL_PRI_UNSPECIFIED: > + break; > + default: > + av_log(hwfc, AV_LOG_WARNING, "Color primaries %s is not > supported.\n", av_color_primaries_name(src->color_primaries)); > + } > + > + switch (src->color_trc) { > + case AVCOL_TRC_SMPTE2084: > + if (__builtin_available(macOS 10.13, *)) > + colortrc = kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ; > + else > + colortrc = CFSTR("SMPTE_ST_2084_PQ"); > + break; > + case AVCOL_TRC_BT2020_10: > + case AVCOL_TRC_BT2020_12: > + if (__builtin_available(macOS 10.11, *)) > + colortrc = kCVImageBufferTransferFunction_ITU_R_2020; > + else > + colortrc = CFSTR("ITU_R_2020"); > + break; > + case AVCOL_TRC_BT709: > + colortrc = kCVImageBufferTransferFunction_ITU_R_709_2; > + break; > + case AVCOL_TRC_SMPTE240M: > + colortrc = kCVImageBufferTransferFunction_SMPTE_240M_1995; > + break; > + case AVCOL_TRC_SMPTE428: > + if (__builtin_available(macOS 10.12, *)) > + colortrc = kCVImageBufferTransferFunction_SMPTE_ST_428_1; > + else > + colortrc = CFSTR("SMPTE_ST_428_1"); > + break; > + case AVCOL_TRC_ARIB_STD_B67: > + if (__builtin_available(macOS 10.13, *)) > + colortrc = kCVImageBufferTransferFunction_ITU_R_2100_HLG; > + else > + colortrc = CFSTR("ITU_R_2100_HLG"); > + break; > + case AVCOL_TRC_GAMMA22: > + gamma = 2.2; > + colortrc = kCVImageBufferTransferFunction_UseGamma; > + break; > + case AVCOL_TRC_GAMMA28: > + gamma = 2.8; > + colortrc = kCVImageBufferTransferFunction_UseGamma; > + break; > + case AVCOL_TRC_UNSPECIFIED: > + break; > + default: > + av_log(hwfc, AV_LOG_WARNING, "Color transfer function %s is not > supported.\n", av_color_transfer_name(src->color_trc)); > + } > + > + if (colormatrix) { > + CVBufferSetAttachment( > + pixbuf, > + kCVImageBufferYCbCrMatrixKey, > + colormatrix, > + kCVAttachmentMode_ShouldPropagate); > + } > + if (colorpri) { > + CVBufferSetAttachment( > + pixbuf, > + kCVImageBufferColorPrimariesKey, > + colorpri, > + kCVAttachmentMode_ShouldPropagate); > + } > + if (colortrc) { > + CVBufferSetAttachment( > + pixbuf, > + kCVImageBufferTransferFunctionKey, > + colortrc, > + kCVAttachmentMode_ShouldPropagate); > + } > + if (gamma != 0) { > + CFNumberRef gamma_level = CFNumberCreate(NULL, > kCFNumberFloat32Type, &gamma); > + CVBufferSetAttachment( > + pixbuf, > + kCVImageBufferGammaLevelKey, > + gamma_level, > + kCVAttachmentMode_ShouldPropagate); > + CFRelease(gamma_level); > + } > + > + return 0; > +} > + > +static int vt_pixbuf_set_attachments(AVHWFramesContext *hwfc, > + CVPixelBufferRef pixbuf, const > AVFrame *src) > +{ > + int ret; > + ret = vt_pixbuf_set_par(hwfc, pixbuf, src); > + if (ret < 0) > + return ret; > + ret = vt_pixbuf_set_colorspace(hwfc, pixbuf, src); > + if (ret < 0) > + return ret; > + ret = vt_pixbuf_set_chromaloc(hwfc, pixbuf, src); > + if (ret < 0) > + return ret; > + return 0; > +} > + > static int vt_map_frame(AVHWFramesContext *ctx, AVFrame *dst, const > AVFrame *src, > int flags) > { > @@ -223,6 +626,10 @@ static int vt_transfer_data_to(AVHWFramesContext > *hwfc, > if (err) > goto fail; > > + err = vt_pixbuf_set_attachments(hwfc, (CVPixelBufferRef)dst->data[3], > src); > + if (err) > + goto fail; > + > err = 0; > fail: > av_frame_free(&map); > @@ -244,8 +651,13 @@ const HWContextType ff_hwcontext_type_videotoolbox = { > .type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX, > .name = "videotoolbox", > > + .frames_priv_size = sizeof(VTFramesContext), > + > .device_create = vt_device_create, > + .frames_init = vt_frames_init, > .frames_get_buffer = vt_get_buffer, > + .frames_get_constraints = vt_frames_get_constraints, > + .frames_uninit = vt_frames_uninit, > .transfer_get_formats = vt_transfer_get_formats, > .transfer_data_to = vt_transfer_data_to, > .transfer_data_from = vt_transfer_data_from, > diff --git a/libavutil/hwcontext_videotoolbox.h > b/libavutil/hwcontext_videotoolbox.h > index 5074d79e68..62cde07c51 100644 > --- a/libavutil/hwcontext_videotoolbox.h > +++ b/libavutil/hwcontext_videotoolbox.h > @@ -29,11 +29,14 @@ > * @file > * An API-specific header for AV_HWDEVICE_TYPE_VIDEOTOOLBOX. > * > - * This API currently does not support frame allocation, as the raw > VideoToolbox > - * API does allocation, and FFmpeg itself never has the need to allocate > frames. > + * This API supports frame allocation using a native CVPixelBufferPool > + * instead of an AVBufferPool. > * > * If the API user sets a custom pool, AVHWFramesContext.pool must return > * AVBufferRefs whose data pointer is a CVImageBufferRef or > CVPixelBufferRef. > + * Note that the underlying CVPixelBuffer could be retained by OS > frameworks > + * depending on application usage, so it is preferable to let CoreVideo > manage > + * the pool using the default implementation. > * > * Currently AVHWDeviceContext.hwctx and AVHWFramesContext.hwctx are > always > * NULL. > -- > 2.29.2 > >
diff --git a/libavutil/hwcontext_videotoolbox.c b/libavutil/hwcontext_videotoolbox.c index bded9873fe..58095a1fc9 100644 --- a/libavutil/hwcontext_videotoolbox.c +++ b/libavutil/hwcontext_videotoolbox.c @@ -24,6 +24,7 @@ #include <VideoToolbox/VideoToolbox.h> #include "buffer.h" +#include "buffer_internal.h" #include "common.h" #include "hwcontext.h" #include "hwcontext_internal.h" @@ -32,6 +33,10 @@ #include "pixfmt.h" #include "pixdesc.h" +typedef struct VTFramesContext { + CVPixelBufferPoolRef pool; +} VTFramesContext; + static const struct { uint32_t cv_fmt; bool full_range; @@ -50,6 +55,39 @@ static const struct { #endif }; +static const enum AVPixelFormat supported_formats[] = { + AV_PIX_FMT_NV12, + AV_PIX_FMT_YUV420P, + AV_PIX_FMT_UYVY422, + AV_PIX_FMT_P010, + AV_PIX_FMT_BGRA, +}; + +static int vt_frames_get_constraints(AVHWDeviceContext *ctx, + const void *hwconfig, + AVHWFramesConstraints *constraints) +{ + int i; + + constraints->valid_sw_formats = av_malloc_array(FF_ARRAY_ELEMS(supported_formats) + 1, + sizeof(*constraints->valid_sw_formats)); + if (!constraints->valid_sw_formats) + return AVERROR(ENOMEM); + + for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++) + constraints->valid_sw_formats[i] = supported_formats[i]; + constraints->valid_sw_formats[FF_ARRAY_ELEMS(supported_formats)] = AV_PIX_FMT_NONE; + + constraints->valid_hw_formats = av_malloc_array(2, sizeof(*constraints->valid_hw_formats)); + if (!constraints->valid_hw_formats) + return AVERROR(ENOMEM); + + constraints->valid_hw_formats[0] = AV_PIX_FMT_VIDEOTOOLBOX; + constraints->valid_hw_formats[1] = AV_PIX_FMT_NONE; + + return 0; +} + enum AVPixelFormat av_map_videotoolbox_format_to_pixfmt(uint32_t cv_fmt) { int i; @@ -75,11 +113,134 @@ uint32_t av_map_videotoolbox_format_from_pixfmt2(enum AVPixelFormat pix_fmt, boo return 0; } +static int vt_pool_alloc(AVHWFramesContext *ctx) +{ + VTFramesContext *fctx = ctx->internal->priv; + CVReturn err; + CFNumberRef w, h, pixfmt; + uint32_t cv_pixfmt; + CFMutableDictionaryRef attributes, iosurface_properties; + + attributes = CFDictionaryCreateMutable( + NULL, + 2, + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks); + + cv_pixfmt = av_map_videotoolbox_format_from_pixfmt(ctx->sw_format); + pixfmt = CFNumberCreate(NULL, kCFNumberSInt32Type, &cv_pixfmt); + CFDictionarySetValue( + attributes, + kCVPixelBufferPixelFormatTypeKey, + pixfmt); + CFRelease(pixfmt); + + iosurface_properties = CFDictionaryCreateMutable( + NULL, + 0, + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks); + CFDictionarySetValue(attributes, kCVPixelBufferIOSurfacePropertiesKey, iosurface_properties); + CFRelease(iosurface_properties); + + w = CFNumberCreate(NULL, kCFNumberSInt32Type, &ctx->width); + h = CFNumberCreate(NULL, kCFNumberSInt32Type, &ctx->height); + CFDictionarySetValue(attributes, kCVPixelBufferWidthKey, w); + CFDictionarySetValue(attributes, kCVPixelBufferHeightKey, h); + CFRelease(w); + CFRelease(h); + + err = CVPixelBufferPoolCreate( + NULL, + NULL, + attributes, + &fctx->pool); + CFRelease(attributes); + + if (err == kCVReturnSuccess) + return 0; + + av_log(ctx, AV_LOG_ERROR, "Error creating CVPixelBufferPool: %d\n", err); + return AVERROR_EXTERNAL; +} + +static AVBufferRef *vt_dummy_pool_alloc(void *opaque, size_t size) +{ + return NULL; +} + +static void vt_frames_uninit(AVHWFramesContext *ctx) +{ + VTFramesContext *fctx = ctx->internal->priv; + if (fctx->pool) { + CVPixelBufferPoolRelease(fctx->pool); + fctx->pool = NULL; + } +} + +static int vt_frames_init(AVHWFramesContext *ctx) +{ + int i, ret; + + for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++) { + if (ctx->sw_format == supported_formats[i]) + break; + } + if (i == FF_ARRAY_ELEMS(supported_formats)) { + av_log(ctx, AV_LOG_ERROR, "Pixel format '%s' is not supported\n", + av_get_pix_fmt_name(ctx->sw_format)); + return AVERROR(ENOSYS); + } + + // create a dummy pool so av_hwframe_get_buffer doesn't EINVAL + if (!ctx->pool) { + ctx->internal->pool_internal = av_buffer_pool_init2(0, ctx, vt_dummy_pool_alloc, NULL); + if (!ctx->internal->pool_internal) + return AVERROR(ENOMEM); + } + + ret = vt_pool_alloc(ctx); + if (ret < 0) + return ret; + + return 0; +} + +static void videotoolbox_buffer_release(void *opaque, uint8_t *data) +{ + CVPixelBufferRelease((CVPixelBufferRef)data); +} + static int vt_get_buffer(AVHWFramesContext *ctx, AVFrame *frame) { - frame->buf[0] = av_buffer_pool_get(ctx->pool); - if (!frame->buf[0]) - return AVERROR(ENOMEM); + VTFramesContext *fctx = ctx->internal->priv; + + if (ctx->pool && ctx->pool->size != 0) { + frame->buf[0] = av_buffer_pool_get(ctx->pool); + if (!frame->buf[0]) + return AVERROR(ENOMEM); + } else { + CVPixelBufferRef pixbuf; + AVBufferRef *buf = NULL; + CVReturn err; + + err = CVPixelBufferPoolCreatePixelBuffer( + NULL, + fctx->pool, + &pixbuf + ); + if (err != kCVReturnSuccess) { + av_log(ctx, AV_LOG_ERROR, "Failed to create pixel buffer from pool: %d\n", err); + return AVERROR_EXTERNAL; + } + + buf = av_buffer_create((uint8_t *)pixbuf, 1, videotoolbox_buffer_release, NULL, 0); + if (!buf) { + CVPixelBufferRelease(pixbuf); + return AVERROR(ENOMEM); + } + frame->buf[0] = buf; + } frame->data[3] = frame->buf[0]->data; frame->format = AV_PIX_FMT_VIDEOTOOLBOX; @@ -111,6 +272,248 @@ static void vt_unmap(AVHWFramesContext *ctx, HWMapDescriptor *hwmap) CVPixelBufferUnlockBaseAddress(pixbuf, (uintptr_t)hwmap->priv); } +static int vt_pixbuf_set_par(AVHWFramesContext *hwfc, + CVPixelBufferRef pixbuf, const AVFrame *src) +{ + CFMutableDictionaryRef par = NULL; + CFNumberRef num = NULL, den = NULL; + AVRational avpar = src->sample_aspect_ratio; + + if (avpar.num == 0) + return 0; + + av_reduce(&avpar.num, &avpar.den, + avpar.num, avpar.den, + 0xFFFFFFFF); + + num = CFNumberCreate(kCFAllocatorDefault, + kCFNumberIntType, + &avpar.num); + + den = CFNumberCreate(kCFAllocatorDefault, + kCFNumberIntType, + &avpar.den); + + par = CFDictionaryCreateMutable(kCFAllocatorDefault, + 2, + &kCFCopyStringDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks); + + if (!par || !num || !den) { + if (par) CFRelease(par); + if (num) CFRelease(num); + if (den) CFRelease(den); + return AVERROR(ENOMEM); + } + + CFDictionarySetValue( + par, + kCVImageBufferPixelAspectRatioHorizontalSpacingKey, + num); + CFDictionarySetValue( + par, + kCVImageBufferPixelAspectRatioVerticalSpacingKey, + den); + + CVBufferSetAttachment( + pixbuf, + kCVImageBufferPixelAspectRatioKey, + par, + kCVAttachmentMode_ShouldPropagate + ); + + CFRelease(par); + CFRelease(num); + CFRelease(den); + + return 0; +} + +static int vt_pixbuf_set_chromaloc(AVHWFramesContext *hwfc, + CVPixelBufferRef pixbuf, const AVFrame *src) +{ + CFStringRef loc = NULL; + + switch (src->chroma_location) { + case AVCHROMA_LOC_LEFT: + loc = kCVImageBufferChromaLocation_Left; + break; + case AVCHROMA_LOC_CENTER: + loc = kCVImageBufferChromaLocation_Center; + break; + case AVCHROMA_LOC_TOP: + loc = kCVImageBufferChromaLocation_Top; + break; + case AVCHROMA_LOC_BOTTOM: + loc = kCVImageBufferChromaLocation_Bottom; + break; + case AVCHROMA_LOC_TOPLEFT: + loc = kCVImageBufferChromaLocation_TopLeft; + break; + case AVCHROMA_LOC_BOTTOMLEFT: + loc = kCVImageBufferChromaLocation_BottomLeft; + break; + } + + if (loc) { + CVBufferSetAttachment( + pixbuf, + kCVImageBufferChromaLocationTopFieldKey, + loc, + kCVAttachmentMode_ShouldPropagate); + } + + return 0; +} + +static int vt_pixbuf_set_colorspace(AVHWFramesContext *hwfc, + CVPixelBufferRef pixbuf, const AVFrame *src) +{ + CFStringRef colormatrix = NULL, colorpri = NULL, colortrc = NULL; + Float32 gamma = 0; + + switch (src->colorspace) { + case AVCOL_SPC_BT2020_CL: + case AVCOL_SPC_BT2020_NCL: + if (__builtin_available(macOS 10.11, *)) + colormatrix = kCVImageBufferYCbCrMatrix_ITU_R_2020; + else + colormatrix = CFSTR("ITU_R_2020"); + break; + case AVCOL_SPC_BT470BG: + case AVCOL_SPC_SMPTE170M: + colormatrix = kCVImageBufferYCbCrMatrix_ITU_R_601_4; + break; + case AVCOL_SPC_BT709: + colormatrix = kCVImageBufferYCbCrMatrix_ITU_R_709_2; + break; + case AVCOL_SPC_SMPTE240M: + colormatrix = kCVImageBufferYCbCrMatrix_SMPTE_240M_1995; + break; + case AVCOL_SPC_UNSPECIFIED: + break; + default: + av_log(hwfc, AV_LOG_WARNING, "Color space %s is not supported.\n", av_color_space_name(src->colorspace)); + } + + switch (src->color_primaries) { + case AVCOL_PRI_BT2020: + if (__builtin_available(macOS 10.11, *)) + colorpri = kCVImageBufferColorPrimaries_ITU_R_2020; + else + colorpri = CFSTR("ITU_R_2020"); + break; + case AVCOL_PRI_BT709: + colorpri = kCVImageBufferColorPrimaries_ITU_R_709_2; + break; + case AVCOL_PRI_SMPTE170M: + colorpri = kCVImageBufferColorPrimaries_SMPTE_C; + break; + case AVCOL_PRI_BT470BG: + colorpri = kCVImageBufferColorPrimaries_EBU_3213; + break; + case AVCOL_PRI_UNSPECIFIED: + break; + default: + av_log(hwfc, AV_LOG_WARNING, "Color primaries %s is not supported.\n", av_color_primaries_name(src->color_primaries)); + } + + switch (src->color_trc) { + case AVCOL_TRC_SMPTE2084: + if (__builtin_available(macOS 10.13, *)) + colortrc = kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ; + else + colortrc = CFSTR("SMPTE_ST_2084_PQ"); + break; + case AVCOL_TRC_BT2020_10: + case AVCOL_TRC_BT2020_12: + if (__builtin_available(macOS 10.11, *)) + colortrc = kCVImageBufferTransferFunction_ITU_R_2020; + else + colortrc = CFSTR("ITU_R_2020"); + break; + case AVCOL_TRC_BT709: + colortrc = kCVImageBufferTransferFunction_ITU_R_709_2; + break; + case AVCOL_TRC_SMPTE240M: + colortrc = kCVImageBufferTransferFunction_SMPTE_240M_1995; + break; + case AVCOL_TRC_SMPTE428: + if (__builtin_available(macOS 10.12, *)) + colortrc = kCVImageBufferTransferFunction_SMPTE_ST_428_1; + else + colortrc = CFSTR("SMPTE_ST_428_1"); + break; + case AVCOL_TRC_ARIB_STD_B67: + if (__builtin_available(macOS 10.13, *)) + colortrc = kCVImageBufferTransferFunction_ITU_R_2100_HLG; + else + colortrc = CFSTR("ITU_R_2100_HLG"); + break; + case AVCOL_TRC_GAMMA22: + gamma = 2.2; + colortrc = kCVImageBufferTransferFunction_UseGamma; + break; + case AVCOL_TRC_GAMMA28: + gamma = 2.8; + colortrc = kCVImageBufferTransferFunction_UseGamma; + break; + case AVCOL_TRC_UNSPECIFIED: + break; + default: + av_log(hwfc, AV_LOG_WARNING, "Color transfer function %s is not supported.\n", av_color_transfer_name(src->color_trc)); + } + + if (colormatrix) { + CVBufferSetAttachment( + pixbuf, + kCVImageBufferYCbCrMatrixKey, + colormatrix, + kCVAttachmentMode_ShouldPropagate); + } + if (colorpri) { + CVBufferSetAttachment( + pixbuf, + kCVImageBufferColorPrimariesKey, + colorpri, + kCVAttachmentMode_ShouldPropagate); + } + if (colortrc) { + CVBufferSetAttachment( + pixbuf, + kCVImageBufferTransferFunctionKey, + colortrc, + kCVAttachmentMode_ShouldPropagate); + } + if (gamma != 0) { + CFNumberRef gamma_level = CFNumberCreate(NULL, kCFNumberFloat32Type, &gamma); + CVBufferSetAttachment( + pixbuf, + kCVImageBufferGammaLevelKey, + gamma_level, + kCVAttachmentMode_ShouldPropagate); + CFRelease(gamma_level); + } + + return 0; +} + +static int vt_pixbuf_set_attachments(AVHWFramesContext *hwfc, + CVPixelBufferRef pixbuf, const AVFrame *src) +{ + int ret; + ret = vt_pixbuf_set_par(hwfc, pixbuf, src); + if (ret < 0) + return ret; + ret = vt_pixbuf_set_colorspace(hwfc, pixbuf, src); + if (ret < 0) + return ret; + ret = vt_pixbuf_set_chromaloc(hwfc, pixbuf, src); + if (ret < 0) + return ret; + return 0; +} + static int vt_map_frame(AVHWFramesContext *ctx, AVFrame *dst, const AVFrame *src, int flags) { @@ -223,6 +626,10 @@ static int vt_transfer_data_to(AVHWFramesContext *hwfc, if (err) goto fail; + err = vt_pixbuf_set_attachments(hwfc, (CVPixelBufferRef)dst->data[3], src); + if (err) + goto fail; + err = 0; fail: av_frame_free(&map); @@ -244,8 +651,13 @@ const HWContextType ff_hwcontext_type_videotoolbox = { .type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX, .name = "videotoolbox", + .frames_priv_size = sizeof(VTFramesContext), + .device_create = vt_device_create, + .frames_init = vt_frames_init, .frames_get_buffer = vt_get_buffer, + .frames_get_constraints = vt_frames_get_constraints, + .frames_uninit = vt_frames_uninit, .transfer_get_formats = vt_transfer_get_formats, .transfer_data_to = vt_transfer_data_to, .transfer_data_from = vt_transfer_data_from, diff --git a/libavutil/hwcontext_videotoolbox.h b/libavutil/hwcontext_videotoolbox.h index 5074d79e68..62cde07c51 100644 --- a/libavutil/hwcontext_videotoolbox.h +++ b/libavutil/hwcontext_videotoolbox.h @@ -29,11 +29,14 @@ * @file * An API-specific header for AV_HWDEVICE_TYPE_VIDEOTOOLBOX. * - * This API currently does not support frame allocation, as the raw VideoToolbox - * API does allocation, and FFmpeg itself never has the need to allocate frames. + * This API supports frame allocation using a native CVPixelBufferPool + * instead of an AVBufferPool. * * If the API user sets a custom pool, AVHWFramesContext.pool must return * AVBufferRefs whose data pointer is a CVImageBufferRef or CVPixelBufferRef. + * Note that the underlying CVPixelBuffer could be retained by OS frameworks + * depending on application usage, so it is preferable to let CoreVideo manage + * the pool using the default implementation. * * Currently AVHWDeviceContext.hwctx and AVHWFramesContext.hwctx are always * NULL.