From patchwork Sun Jun 30 12:15:25 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Thilo Borgmann X-Patchwork-Id: 13771 Return-Path: X-Original-To: patchwork@ffaux-bg.ffmpeg.org Delivered-To: patchwork@ffaux-bg.ffmpeg.org Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org [79.124.17.100]) by ffaux.localdomain (Postfix) with ESMTP id 25EC7447BBD for ; Sun, 30 Jun 2019 15:15:33 +0300 (EEST) Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 0642268AA39; Sun, 30 Jun 2019 15:15:33 +0300 (EEST) X-Original-To: ffmpeg-devel@ffmpeg.org Delivered-To: ffmpeg-devel@ffmpeg.org Received: from shout02.mail.de (shout02.mail.de [62.201.172.25]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id CB88768A8FF for ; Sun, 30 Jun 2019 15:15:26 +0300 (EEST) Received: from postfix01.mail.de (postfix01.bt.mail.de [10.0.121.125]) by shout02.mail.de (Postfix) with ESMTP id 6A38DC00C7 for ; Sun, 30 Jun 2019 14:15:26 +0200 (CEST) Received: from smtp01.mail.de (smtp01.bt.mail.de [10.0.121.211]) by postfix01.mail.de (Postfix) with ESMTP id 44BB68014F for ; Sun, 30 Jun 2019 14:15:26 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=mail.de; s=mailde201610; t=1561896926; bh=Wo0Mo9dHNBsLuJ+jQqSYlWpJ9Ra6U4DzbhRFX1u5jq0=; h=Subject:To:References:From:Date:In-Reply-To:From; b=DIrRG2vNHKIYEP4ZhWFGP6Pfke5W+MjvMFbtxICl8Fsc6iG/xtbUrDi0wkkNEox6t KMwWYKDjdZDWK6vSyQ+EBUiTP16Q9GCjWIVCfzej/gK+wowzJLjgZkitwmdY+ft9R6 SrQsqjlC1lhiRGP4R9I2+iiHk8sIC+QBZnP3kUrw= Received: from [127.0.0.1] (localhost [127.0.0.1]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp01.mail.de (Postfix) with ESMTPSA id EEA4DE5006 for ; Sun, 30 Jun 2019 14:15:25 +0200 (CEST) To: ffmpeg-devel@ffmpeg.org References: <853b2e0f-0b46-4de5-e6d1-c738e5d3142e@mail.de> From: Thilo Borgmann Message-ID: <8bc5f556-f7b7-2518-7ee3-eb1d6a636a23@mail.de> Date: Sun, 30 Jun 2019 14:15:25 +0200 MIME-Version: 1.0 In-Reply-To: <853b2e0f-0b46-4de5-e6d1-c738e5d3142e@mail.de> Content-Language: en-US X-purgate: clean X-purgate: This mail is considered clean (visit http://www.eleven.de for further information) X-purgate-type: clean X-purgate-Ad: Categorized by eleven eXpurgate (R) http://www.eleven.de X-purgate: This mail is considered clean (visit http://www.eleven.de for further information) X-purgate: clean X-purgate-size: 14627 X-purgate-ID: 154282::1561896926-00001DC7-067E8A26/0/0 Subject: [FFmpeg-devel] [PATCH 4/4] lavd/avfoundation: Support muxed type of devices X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" $SUBJECT -Thilo From 3287fbbcbf625a8eebf3aac17871836a6f28c824 Mon Sep 17 00:00:00 2001 From: Thilo Borgmann Date: Sun, 30 Jun 2019 14:01:56 +0200 Subject: [PATCH 4/4] lavd/avfoundation: Support muxed type of devices including raw muxed data capture. --- libavdevice/avfoundation.m | 91 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 82 insertions(+), 9 deletions(-) diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m index 5dd6ea5..715b7b3 100644 --- a/libavdevice/avfoundation.m +++ b/libavdevice/avfoundation.m @@ -97,6 +97,8 @@ typedef struct int capture_cursor; int capture_mouse_clicks; + int capture_raw_data; + int video_is_muxed; int list_devices; int video_device_index; @@ -291,6 +293,10 @@ static int configure_video_device(AVFormatContext *s, AVCaptureDevice *video_dev NSObject *selected_range = nil; NSObject *selected_format = nil; + // try to configure format by formats list + // might raise an exception if no format list is given + // (then fallback to default, no configuration) + @try { for (format in [video_device valueForKey:@"formats"]) { CMFormatDescriptionRef formatDescription; CMVideoDimensions dimensions; @@ -324,19 +330,29 @@ static int configure_video_device(AVFormatContext *s, AVCaptureDevice *video_dev if (!selected_range) { av_log(s, AV_LOG_ERROR, "Selected framerate (%f) is not supported by the device.\n", framerate); + if (ctx->video_is_muxed) { + av_log(s, AV_LOG_ERROR, "Falling back to default.\n"); + } else { goto unsupported_format; + } } if ([video_device lockForConfiguration:NULL] == YES) { + if (selected_format) { + [video_device setValue:selected_format forKey:@"activeFormat"]; + } + if (selected_range) { NSValue *min_frame_duration = [selected_range valueForKey:@"minFrameDuration"]; - - [video_device setValue:selected_format forKey:@"activeFormat"]; [video_device setValue:min_frame_duration forKey:@"activeVideoMinFrameDuration"]; [video_device setValue:min_frame_duration forKey:@"activeVideoMaxFrameDuration"]; + } } else { av_log(s, AV_LOG_ERROR, "Could not lock device for configuration.\n"); return AVERROR(EINVAL); } + } @catch(NSException *e) { + av_log(ctx, AV_LOG_WARNING, "Configuration of video device failed, falling back to default.\n"); + } return 0; @@ -468,12 +484,18 @@ static int add_video_device(AVFormatContext *s, AVCaptureDevice *video_device) } } + // set videoSettings to an empty dict for receiving raw data of muxed devices + if (ctx->capture_raw_data) { + ctx->pixel_format = pxl_fmt_spec.ff_id; + ctx->video_output.videoSettings = @{ }; + } else { ctx->pixel_format = pxl_fmt_spec.ff_id; pixel_format = [NSNumber numberWithUnsignedInt:pxl_fmt_spec.avf_id]; capture_dict = [NSDictionary dictionaryWithObject:pixel_format forKey:(id)kCVPixelBufferPixelFormatTypeKey]; [ctx->video_output setVideoSettings:capture_dict]; + } [ctx->video_output setAlwaysDiscardsLateVideoFrames:YES]; ctx->avf_delegate = [[AVFFrameReceiver alloc] initWithContext:ctx]; @@ -540,6 +562,7 @@ static int get_video_config(AVFormatContext *s) { AVFContext *ctx = (AVFContext*)s->priv_data; CVImageBufferRef image_buffer; + CMBlockBufferRef block_buffer; CGSize image_buffer_size; AVStream* stream = avformat_new_stream(s, NULL); @@ -558,14 +581,22 @@ static int get_video_config(AVFormatContext *s) avpriv_set_pts_info(stream, 64, 1, avf_time_base); - image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame); - image_buffer_size = CVImageBufferGetEncodedSize(image_buffer); + image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame); + block_buffer = CMSampleBufferGetDataBuffer(ctx->current_frame); + + if (image_buffer) { + image_buffer_size = CVImageBufferGetEncodedSize(image_buffer); stream->codecpar->codec_id = AV_CODEC_ID_RAWVIDEO; stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO; stream->codecpar->width = (int)image_buffer_size.width; stream->codecpar->height = (int)image_buffer_size.height; stream->codecpar->format = ctx->pixel_format; + } else { + stream->codecpar->codec_id = AV_CODEC_ID_DVVIDEO; + stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO; + stream->codecpar->format = ctx->pixel_format; + } CFRelease(ctx->current_frame); ctx->current_frame = nil; @@ -670,8 +701,9 @@ static int avf_read_header(AVFormatContext *s) AVCaptureDevice *audio_device = nil; // Find capture device NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]; - ctx->num_video_devices = [devices count]; + NSArray *devices_muxed = [AVCaptureDevice devicesWithMediaType:AVMediaTypeMuxed]; + ctx->num_video_devices = [devices count] + [devices_muxed count]; ctx->first_pts = av_gettime(); ctx->first_audio_pts = av_gettime(); @@ -691,12 +723,17 @@ static int avf_read_header(AVFormatContext *s) index = [devices indexOfObject:device]; av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name); } + for (AVCaptureDevice *device in devices_muxed) { + const char *name = [[device localizedName] UTF8String]; + index = [devices count] + [devices_muxed indexOfObject:device]; + av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name); + } #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070 if (num_screens > 0) { CGDirectDisplayID screens[num_screens]; CGGetActiveDisplayList(num_screens, screens, &num_screens); for (int i = 0; i < num_screens; i++) { - av_log(ctx, AV_LOG_INFO, "[%d] Capture screen %d\n", index + i, i); + av_log(ctx, AV_LOG_INFO, "[%d] Capture screen %d\n", ctx->num_video_devices + i, i); } } #endif @@ -724,7 +761,12 @@ static int avf_read_header(AVFormatContext *s) if (ctx->video_device_index >= 0) { if (ctx->video_device_index < ctx->num_video_devices) { + if (ctx->video_device_index < [devices count]) { video_device = [devices objectAtIndex:ctx->video_device_index]; + } else { + video_device = [devices_muxed objectAtIndex:(ctx->video_device_index - [devices count])]; + ctx->video_is_muxed = 1; + } } else if (ctx->video_device_index < ctx->num_video_devices + num_screens) { #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070 CGDirectDisplayID screens[num_screens]; @@ -768,6 +810,14 @@ static int avf_read_header(AVFormatContext *s) break; } } + // looking for muxed inputs + for (AVCaptureDevice *device in devices_muxed) { + if (!strncmp(ctx->video_filename, [[device localizedName] UTF8String], strlen(ctx->video_filename))) { + video_device = device; + ctx->video_is_muxed = 1; + break; + } + } #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070 // looking for screen inputs @@ -941,13 +991,25 @@ static int avf_read_packet(AVFormatContext *s, AVPacket *pkt) do { CVImageBufferRef image_buffer; + CMBlockBufferRef block_buffer; lock_frames(ctx); - image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame); - if (ctx->current_frame != nil) { int status; - if (av_new_packet(pkt, (int)CVPixelBufferGetDataSize(image_buffer)) < 0) { + int length = 0; + + image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame); + block_buffer = CMSampleBufferGetDataBuffer(ctx->current_frame); + + if (image_buffer != nil) { + length = (int)CVPixelBufferGetDataSize(image_buffer); + } else if (block_buffer != nil) { + length = (int)CMBlockBufferGetDataLength(block_buffer); + } else { + return AVERROR(EINVAL); + } + + if (av_new_packet(pkt, length) < 0) { return AVERROR(EIO); } @@ -962,7 +1024,15 @@ static int avf_read_packet(AVFormatContext *s, AVPacket *pkt) pkt->stream_index = ctx->video_stream_index; pkt->flags |= AV_PKT_FLAG_KEY; + if (image_buffer) { status = copy_cvpixelbuffer(s, image_buffer, pkt); + } else { + status = 0; + OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, pkt->data); + if (ret != kCMBlockBufferNoErr) { + status = AVERROR(EIO); + } + } CFRelease(ctx->current_frame); ctx->current_frame = nil; @@ -1070,6 +1140,9 @@ static const AVOption options[] = { { "capture_mouse_clicks", "capture the screen mouse clicks", offsetof(AVFContext, capture_mouse_clicks), AV_OPT_TYPE_INT, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM, "capture_mouse_clicks" }, { "true", "", 0, AV_OPT_TYPE_CONST, {.i64=1}, 0, 0, AV_OPT_FLAG_DECODING_PARAM, "capture_mouse_clicks" }, { "false", "", 0, AV_OPT_TYPE_CONST, {.i64=0}, 0, 0, AV_OPT_FLAG_DECODING_PARAM, "capture_mouse_clicks" }, + { "capture_raw_data", "capture the raw data from device connection", offsetof(AVFContext, capture_raw_data), AV_OPT_TYPE_INT, {.i64=1}, 0, 1, AV_OPT_FLAG_DECODING_PARAM, "capture_raw_data" }, + { "true", "", 0, AV_OPT_TYPE_CONST, {.i64=1}, 0, 0, AV_OPT_FLAG_DECODING_PARAM, "capture_raw_data" }, + { "false", "", 0, AV_OPT_TYPE_CONST, {.i64=0}, 0, 0, AV_OPT_FLAG_DECODING_PARAM, "capture_raw_data" }, { NULL }, };