diff mbox series

[FFmpeg-devel,1/3] scale_vt frame crop support

Message ID 20240910181057.43453-1-koushd@gmail.com
State New
Headers show
Series [FFmpeg-devel,1/3] scale_vt frame crop support | expand

Checks

Context Check Description
andriy/commit_msg_x86 warning The first line of the commit message must start with a context terminated by a colon and a space, for example "lavu/opt: " or "doc: ".
andriy/make_x86 success Make finished
andriy/make_fate_x86 success Make fate finished

Commit Message

Koushik Dutta Sept. 10, 2024, 6:10 p.m. UTC 
The crop filter has no effect on scale_vt:

-vf crop=100:100,scale_vt=300x300

Hardware frames (AV_PIX_FMT_FLAG_HWACCEL) are expected to use the crop_* properties,
as seen in the implementation vf_crop.c.

The current workaround is to hwdownload the full frame
and perform the crop on CPU.
---
 libavfilter/vf_scale_vt.c | 50 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)
diff mbox series

Patch

diff --git a/libavfilter/vf_scale_vt.c b/libavfilter/vf_scale_vt.c
index 05f4e7b797..3da46a6cd5 100644
--- a/libavfilter/vf_scale_vt.c
+++ b/libavfilter/vf_scale_vt.c
@@ -109,6 +109,8 @@  static av_cold int scale_vt_init(AVFilterContext *avctx)
         VTSessionSetProperty(s->transfer, kVTPixelTransferPropertyKey_DestinationYCbCrMatrix, value);
     }
 
+    VTSessionSetProperty(s->transfer, kVTPixelTransferPropertyKey_ScalingMode, kVTScalingMode_CropSourceToCleanAperture);
+
     return 0;
 }
 
@@ -132,6 +134,18 @@  static int scale_vt_filter_frame(AVFilterLink *link, AVFrame *in)
     CVPixelBufferRef src;
     CVPixelBufferRef dst;
 
+    int left;
+    int top;
+    int width;
+    int height;
+    CFNumberRef crop_width_num;
+    CFNumberRef crop_height_num;
+    CFNumberRef crop_offset_left_num;
+    CFNumberRef crop_offset_top_num;
+    const void *clean_aperture_keys[4];
+    const void *source_clean_aperture_values[4];
+    CFDictionaryRef source_clean_aperture;
+
     AVFrame *out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
     if (!out) {
         ret = AVERROR(ENOMEM);
@@ -153,8 +167,43 @@  static int scale_vt_filter_frame(AVFilterLink *link, AVFrame *in)
     if (s->colour_matrix != AVCOL_SPC_UNSPECIFIED)
         out->colorspace = s->colour_matrix;
 
+    width = (in->width - in->crop_right) - in->crop_left;
+    height = (in->height - in->crop_bottom) - in->crop_top;
+    // The crop offsets are relative to the center of the frame.
+    // the crop width and crop height are relative to the center of the crop rect, not top left as normal.
+    left = in->crop_left - in->width / 2 + width / 2;
+    top = in->crop_top - in->height / 2 + height / 2;
+    crop_width_num = CFNumberCreate(kCFAllocatorDefault, kCFNumberIntType, &width);
+    crop_height_num = CFNumberCreate(kCFAllocatorDefault, kCFNumberIntType, &height);
+    crop_offset_left_num = CFNumberCreate(kCFAllocatorDefault, kCFNumberIntType, &left);
+    crop_offset_top_num = CFNumberCreate(kCFAllocatorDefault, kCFNumberIntType, &top);
+
+    clean_aperture_keys[0] = kCVImageBufferCleanApertureWidthKey;
+    clean_aperture_keys[1] = kCVImageBufferCleanApertureHeightKey;
+    clean_aperture_keys[2] = kCVImageBufferCleanApertureHorizontalOffsetKey;
+    clean_aperture_keys[3] = kCVImageBufferCleanApertureVerticalOffsetKey;
+
+    source_clean_aperture_values[0] = crop_width_num;
+    source_clean_aperture_values[1] = crop_height_num;
+    source_clean_aperture_values[2] = crop_offset_left_num;
+    source_clean_aperture_values[3] = crop_offset_top_num;
+
+    source_clean_aperture = CFDictionaryCreate(kCFAllocatorDefault,
+                                                    clean_aperture_keys,
+                                                    source_clean_aperture_values,
+                                                    4,
+                                                    &kCFTypeDictionaryKeyCallBacks,
+                                                    &kCFTypeDictionaryValueCallBacks);
+
+    CFRelease(crop_width_num);
+    CFRelease(crop_height_num);
+    CFRelease(crop_offset_left_num);
+    CFRelease(crop_offset_top_num);
+
     src = (CVPixelBufferRef)in->data[3];
     dst = (CVPixelBufferRef)out->data[3];
+    CVBufferSetAttachment(src, kCVImageBufferCleanApertureKey,
+                           source_clean_aperture, kCVAttachmentMode_ShouldPropagate);
     ret = VTPixelTransferSessionTransferImage(s->transfer, src, dst);
     if (ret != noErr) {
         av_log(ctx, AV_LOG_ERROR, "transfer image failed, %d\n", ret);
@@ -162,6 +211,7 @@  static int scale_vt_filter_frame(AVFilterLink *link, AVFrame *in)
         goto fail;
     }
 
+    CFRelease(source_clean_aperture);
     av_frame_free(&in);
 
     return ff_filter_frame(outlink, out);