From patchwork Mon Apr 12 02:31:41 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mark Reid X-Patchwork-Id: 26872 Return-Path: X-Original-To: patchwork@ffaux-bg.ffmpeg.org Delivered-To: patchwork@ffaux-bg.ffmpeg.org Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org [79.124.17.100]) by ffaux.localdomain (Postfix) with ESMTP id 4192A449793 for ; Mon, 12 Apr 2021 05:31:54 +0300 (EEST) Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 11200687F1A; Mon, 12 Apr 2021 05:31:54 +0300 (EEST) X-Original-To: ffmpeg-devel@ffmpeg.org Delivered-To: ffmpeg-devel@ffmpeg.org Received: from mail-pj1-f46.google.com (mail-pj1-f46.google.com [209.85.216.46]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id B73D9680869 for ; Mon, 12 Apr 2021 05:31:47 +0300 (EEST) Received: by mail-pj1-f46.google.com with SMTP id b8-20020a17090a5508b029014d0fbe9b64so7961813pji.5 for ; Sun, 11 Apr 2021 19:31:47 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20161025; h=from:to:cc:subject:date:message-id:mime-version :content-transfer-encoding; bh=8J0LiGrkKohFYEyPir47PF4pBJKe4z0i8N45EygAdSQ=; b=CamO5cueV+9Om5PMxTZHKWiDGJthrl5GZKKgTo9cnOPeCInfbCDvJL/TPRIKfDrkEV FgKfusF+mqeWmB+KWlk1SCmTPwvYXNiHas0qtO+uLmZ58xajHHm/p6OIpLvslON/tqRA i3IyJ7bjhsNfCSZn3aKETxYP3ryZJ/WZWo3SUIAmGJxXr3FoIE+rJCPO4dbrQob3gz+U fFRQfr70IbA0kLsxe69I171axaN8ZFhNVQGuJd/X5rYrhmOSnvjEplsUWO3Rz4op3ncy IcPSTWNPyg2xlhdfqF7iIExXXNx7ZExqde92+PxPYiEslCyt4xpDkMq0NhXCajlA6iTV SXcw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:mime-version :content-transfer-encoding; bh=8J0LiGrkKohFYEyPir47PF4pBJKe4z0i8N45EygAdSQ=; b=fShfcZPUDcDpiJQkrG+mlPBAdX8Hnfbo219y598vCtIZYxua2w24yDSo3AGWI8jvsr oVLPQffO5Itt/8LwRCEk00+CImqk1WvAXe58nxp2wP0iWnZodnkoGW5fqQr/6TMHbnv5 lOP4lz9Z0IeNm+d3hz0KkYhnSnPx68xAu97YHgBptB6lgU1UkGanMoAeyROk2qRZgIt7 o9WZDODXtEfaJTg3bHWjk2TjpC3c0JLuHPObMIvICntre62FL0TKACisdkEGLh7M5DQe o2mTY5NSzEIZgCqwiq6zTWG8fr/J+w4rTJnFdQBLxvwMVzb6xPR3YDbOpIKpEbQH1gBK qUfA== X-Gm-Message-State: AOAM532BWcnZDJBLqqBDjOXP/p5I2c4PGloO3QuFh9De+Sv/CWcnEZXh mZjrZpunhhe8xPqkUpuvWDiY9PxizuQ= X-Google-Smtp-Source: ABdhPJwcpUEvj7/V4VduDseWrGLgzd3R3o7Fp8P4PptUkxwQEHxUjEu3wSrJGf3mjlxvidG0+CF6RQ== X-Received: by 2002:a17:902:24:b029:e9:3f8f:9af9 with SMTP id 33-20020a1709020024b02900e93f8f9af9mr24696982pla.34.1618194705829; Sun, 11 Apr 2021 19:31:45 -0700 (PDT) Received: from localhost.localdomain (S0106bc4dfba470f3.vc.shawcable.net. [174.7.244.175]) by smtp.gmail.com with ESMTPSA id q95sm9258811pjq.20.2021.04.11.19.31.45 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Sun, 11 Apr 2021 19:31:45 -0700 (PDT) From: mindmark@gmail.com To: ffmpeg-devel@ffmpeg.org Date: Sun, 11 Apr 2021 19:31:41 -0700 Message-Id: <20210412023142.8470-1-mindmark@gmail.com> X-Mailer: git-send-email 2.29.2 MIME-Version: 1.0 Subject: [FFmpeg-devel] [PATCH v2 1/2] libavdevice/avfoundation: add buffer fifo and output packets in order they arrive X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Cc: Mark Reid Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" From: Mark Reid This fixes audio issues I've had with some capture devices. The audio gets really choppy and stops working. This seems to be because avf_read_packet stops outputting the audio frames because a video frame happens to be available first. It base on the approach used in a patch from #4437 https://trac.ffmpeg.org/ticket/4437 My approach uses an AVFifoBuffer instead of NSMutableArray and also outputs the packets in the same order they arrive from AVFFoundation. should fix ticket #4437 and #4513 --- libavdevice/avfoundation.m | 160 ++++++++++++++++++++++++++++--------- 1 file changed, 124 insertions(+), 36 deletions(-) diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m index 59d5b0af4f..5ac6ec4183 100644 --- a/libavdevice/avfoundation.m +++ b/libavdevice/avfoundation.m @@ -31,13 +31,17 @@ #include "libavutil/pixdesc.h" #include "libavutil/opt.h" #include "libavutil/avstring.h" +#include "libavutil/avassert.h" #include "libavformat/internal.h" #include "libavutil/internal.h" #include "libavutil/parseutils.h" #include "libavutil/time.h" #include "libavutil/imgutils.h" +#include "libavutil/fifo.h" #include "avdevice.h" +#define FIFO_SIZE 4 + static const int avf_time_base = 1000000; static const AVRational avf_time_base_q = { @@ -128,8 +132,8 @@ typedef struct AVCaptureSession *capture_session; AVCaptureVideoDataOutput *video_output; AVCaptureAudioDataOutput *audio_output; - CMSampleBufferRef current_frame; - CMSampleBufferRef current_audio_frame; + AVFifoBuffer *video_fifo; + AVFifoBuffer *audio_fifo; AVCaptureDevice *observed_device; #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070 @@ -138,6 +142,11 @@ typedef struct int observed_quit; } AVFContext; +typedef struct { + int64_t ts; + CMSampleBufferRef frame; +} BufferRef; + static void lock_frames(AVFContext* ctx) { pthread_mutex_lock(&ctx->frame_lock); @@ -148,6 +157,48 @@ static void unlock_frames(AVFContext* ctx) pthread_mutex_unlock(&ctx->frame_lock); } +static inline void fifo_write(AVFifoBuffer* f, int64_t ts, CMSampleBufferRef frame) +{ + BufferRef buf = { + .ts = ts, + .frame = frame, + }; + + CFRetain(frame); + av_fifo_generic_write(f, &buf, sizeof(BufferRef), NULL); +} + +static inline void fifo_peek(AVFifoBuffer* f, BufferRef *buf) +{ + if (av_fifo_size(f)) { + av_fifo_generic_peek(f, buf, sizeof(BufferRef), NULL); + return; + } + buf->frame = nil; + return; +} + +static inline void fifo_drain(AVFifoBuffer* f, int release) +{ + av_assert2(av_fifo_size(f) >= sizeof(BufferRef)); + if (release) { + BufferRef buf; + fifo_peek(f, &buf); + CFRelease(buf.frame); + } + av_fifo_drain(f, sizeof(BufferRef)); +} + +static inline void fifo_freep(AVFifoBuffer **f) +{ + if (f) { + while (av_fifo_size(*f)) { + fifo_drain(*f, 1); + } + av_fifo_freep(f); + } +} + /** FrameReciever class - delegate for AVCaptureSession */ @interface AVFFrameReceiver : NSObject @@ -225,13 +276,16 @@ static void unlock_frames(AVFContext* ctx) didOutputSampleBuffer:(CMSampleBufferRef)videoFrame fromConnection:(AVCaptureConnection *)connection { + AVFifoBuffer *fifo = _context->video_fifo; + int64_t ts = av_gettime_relative(); lock_frames(_context); - if (_context->current_frame != nil) { - CFRelease(_context->current_frame); + if (av_fifo_space(fifo) == 0) { + av_log(_context, AV_LOG_DEBUG, "video fifo is full, the oldest frame has been dropped\n"); + fifo_drain(fifo, 1); } - _context->current_frame = (CMSampleBufferRef)CFRetain(videoFrame); + fifo_write(fifo, ts, videoFrame); unlock_frames(_context); @@ -269,13 +323,16 @@ static void unlock_frames(AVFContext* ctx) didOutputSampleBuffer:(CMSampleBufferRef)audioFrame fromConnection:(AVCaptureConnection *)connection { + AVFifoBuffer *fifo = _context->audio_fifo; + int64_t ts = av_gettime_relative(); lock_frames(_context); - if (_context->current_audio_frame != nil) { - CFRelease(_context->current_audio_frame); + if (!av_fifo_space(fifo)) { + av_log(_context, AV_LOG_DEBUG, "audio fifo is full, the oldest frame has been dropped\n"); + fifo_drain(fifo, 1); } - _context->current_audio_frame = (CMSampleBufferRef)CFRetain(audioFrame); + fifo_write(fifo, ts, audioFrame); unlock_frames(_context); @@ -301,12 +358,10 @@ static void destroy_context(AVFContext* ctx) ctx->avf_audio_delegate = NULL; av_freep(&ctx->audio_buffer); + fifo_freep(&ctx->video_fifo); + fifo_freep(&ctx->audio_fifo); pthread_mutex_destroy(&ctx->frame_lock); - - if (ctx->current_frame) { - CFRelease(ctx->current_frame); - } } static void parse_device_name(AVFormatContext *s) @@ -624,6 +679,7 @@ static int add_audio_device(AVFormatContext *s, AVCaptureDevice *audio_device) static int get_video_config(AVFormatContext *s) { AVFContext *ctx = (AVFContext*)s->priv_data; + BufferRef buf; CVImageBufferRef image_buffer; CMBlockBufferRef block_buffer; CGSize image_buffer_size; @@ -644,8 +700,13 @@ static int get_video_config(AVFormatContext *s) avpriv_set_pts_info(stream, 64, 1, avf_time_base); - image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame); - block_buffer = CMSampleBufferGetDataBuffer(ctx->current_frame); + fifo_peek(ctx->video_fifo, &buf); + if (buf.frame == nil) { + return 1; + } + + image_buffer = CMSampleBufferGetImageBuffer(buf.frame); + block_buffer = CMSampleBufferGetDataBuffer(buf.frame); if (image_buffer) { image_buffer_size = CVImageBufferGetEncodedSize(image_buffer); @@ -661,9 +722,6 @@ static int get_video_config(AVFormatContext *s) stream->codecpar->format = ctx->pixel_format; } - CFRelease(ctx->current_frame); - ctx->current_frame = nil; - unlock_frames(ctx); return 0; @@ -672,6 +730,7 @@ static int get_video_config(AVFormatContext *s) static int get_audio_config(AVFormatContext *s) { AVFContext *ctx = (AVFContext*)s->priv_data; + BufferRef buf; CMFormatDescriptionRef format_desc; AVStream* stream = avformat_new_stream(s, NULL); @@ -690,7 +749,12 @@ static int get_audio_config(AVFormatContext *s) avpriv_set_pts_info(stream, 64, 1, avf_time_base); - format_desc = CMSampleBufferGetFormatDescription(ctx->current_audio_frame); + fifo_peek(ctx->audio_fifo, &buf); + if (buf.frame == nil) { + return 1; + } + + format_desc = CMSampleBufferGetFormatDescription(buf.frame); const AudioStreamBasicDescription *basic_desc = CMAudioFormatDescriptionGetStreamBasicDescription(format_desc); if (!basic_desc) { @@ -737,7 +801,7 @@ static int get_audio_config(AVFormatContext *s) } if (ctx->audio_non_interleaved) { - CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame); + CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(buf.frame); ctx->audio_buffer_size = CMBlockBufferGetDataLength(block_buffer); ctx->audio_buffer = av_malloc(ctx->audio_buffer_size); if (!ctx->audio_buffer) { @@ -746,9 +810,6 @@ static int get_audio_config(AVFormatContext *s) } } - CFRelease(ctx->current_audio_frame); - ctx->current_audio_frame = nil; - unlock_frames(ctx); return 0; @@ -771,6 +832,9 @@ static int avf_read_header(AVFormatContext *s) pthread_mutex_init(&ctx->frame_lock, NULL); + ctx->video_fifo = av_fifo_alloc_array(FIFO_SIZE, sizeof(BufferRef)); + ctx->audio_fifo = av_fifo_alloc_array(FIFO_SIZE, sizeof(BufferRef)); + #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070 CGGetActiveDisplayList(0, NULL, &num_screens); #endif @@ -1051,33 +1115,52 @@ static int avf_read_packet(AVFormatContext *s, AVPacket *pkt) AVFContext* ctx = (AVFContext*)s->priv_data; do { + BufferRef video; + BufferRef audio; CVImageBufferRef image_buffer; CMBlockBufferRef block_buffer; lock_frames(ctx); - if (ctx->current_frame != nil) { + fifo_peek(ctx->video_fifo, &video); + fifo_peek(ctx->audio_fifo, &audio); + + if (video.frame != nil && audio.frame != nil) { + // process oldest CMSampleBufferRef first + if (audio.ts <= video.ts) { + video.frame = nil; + } else { + audio.frame = nil; + } + } + + if (video.frame != nil) { int status; int length = 0; - image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame); - block_buffer = CMSampleBufferGetDataBuffer(ctx->current_frame); + fifo_drain(ctx->video_fifo, 0); + unlock_frames(ctx); + + image_buffer = CMSampleBufferGetImageBuffer(video.frame); + block_buffer = CMSampleBufferGetDataBuffer(video.frame); if (image_buffer != nil) { length = (int)CVPixelBufferGetDataSize(image_buffer); } else if (block_buffer != nil) { length = (int)CMBlockBufferGetDataLength(block_buffer); } else { + CFRelease(video.frame); return AVERROR(EINVAL); } if (av_new_packet(pkt, length) < 0) { + CFRelease(video.frame); return AVERROR(EIO); } CMItemCount count; CMSampleTimingInfo timing_info; - if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_frame, 1, &timing_info, &count) == noErr) { + if (CMSampleBufferGetOutputSampleTimingInfoArray(video.frame, 1, &timing_info, &count) == noErr) { AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale); pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q); } @@ -1094,31 +1177,37 @@ static int avf_read_packet(AVFormatContext *s, AVPacket *pkt) status = AVERROR(EIO); } } - CFRelease(ctx->current_frame); - ctx->current_frame = nil; + CFRelease(video.frame); - if (status < 0) + if (status < 0) { return status; - } else if (ctx->current_audio_frame != nil) { - CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame); + } + } else if (audio.frame != nil) { + CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(audio.frame); int block_buffer_size = CMBlockBufferGetDataLength(block_buffer); + fifo_drain(ctx->audio_fifo, 0); + unlock_frames(ctx); + if (!block_buffer || !block_buffer_size) { + CFRelease(audio.frame); return AVERROR(EIO); } if (ctx->audio_non_interleaved && block_buffer_size > ctx->audio_buffer_size) { + CFRelease(audio.frame); return AVERROR_BUFFER_TOO_SMALL; } if (av_new_packet(pkt, block_buffer_size) < 0) { + CFRelease(audio.frame); return AVERROR(EIO); } CMItemCount count; CMSampleTimingInfo timing_info; - if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_audio_frame, 1, &timing_info, &count) == noErr) { + if (CMSampleBufferGetOutputSampleTimingInfoArray(audio.frame, 1, &timing_info, &count) == noErr) { AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale); pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q); } @@ -1131,6 +1220,7 @@ static int avf_read_packet(AVFormatContext *s, AVPacket *pkt) OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, ctx->audio_buffer); if (ret != kCMBlockBufferNoErr) { + CFRelease(audio.frame); return AVERROR(EIO); } @@ -1162,12 +1252,12 @@ static int avf_read_packet(AVFormatContext *s, AVPacket *pkt) } else { OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, pkt->data); if (ret != kCMBlockBufferNoErr) { + CFRelease(audio.frame); return AVERROR(EIO); } } - CFRelease(ctx->current_audio_frame); - ctx->current_audio_frame = nil; + CFRelease(audio.frame); } else { pkt->data = NULL; unlock_frames(ctx); @@ -1177,8 +1267,6 @@ static int avf_read_packet(AVFormatContext *s, AVPacket *pkt) return AVERROR(EAGAIN); } } - - unlock_frames(ctx); } while (!pkt->data); return 0; From patchwork Mon Apr 12 02:31:42 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Mark Reid X-Patchwork-Id: 26873 Return-Path: X-Original-To: patchwork@ffaux-bg.ffmpeg.org Delivered-To: patchwork@ffaux-bg.ffmpeg.org Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org [79.124.17.100]) by ffaux.localdomain (Postfix) with ESMTP id 56E43449793 for ; Mon, 12 Apr 2021 05:31:55 +0300 (EEST) Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 3EB80688132; Mon, 12 Apr 2021 05:31:55 +0300 (EEST) X-Original-To: ffmpeg-devel@ffmpeg.org Delivered-To: ffmpeg-devel@ffmpeg.org Received: from mail-pj1-f48.google.com (mail-pj1-f48.google.com [209.85.216.48]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id 7B5AC680869 for ; Mon, 12 Apr 2021 05:31:48 +0300 (EEST) Received: by mail-pj1-f48.google.com with SMTP id nm3-20020a17090b19c3b029014e1bbf6c60so2058520pjb.4 for ; Sun, 11 Apr 2021 19:31:48 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20161025; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=DqOS3S5Q8bs6xsDVzGZ0eAGq1zWITCddzDMyyLDcxak=; b=A1PH3MoZjY5FFjq+GQGFpR9VsiI/kq09CHf+lAkVUjslAbIlfLHESyPdoPn6ys/W+4 libGTRYOd8962RUmikZuscXtgFfSdrGPb0Cw9g+lEgmW1Yhu/90ocv9n9p4i/qCXLGgH yVyYMfAWEjAnWA2fD0zS35nLjgxwi+2l3TUaB6tY5PMOMWf+ra/Na8cDna25tMBb5Stj XzmgvgSSw/w+jftm/A+tWwiR3pE6awrwFChO8L2rdQJdVFjZdg98nnpEumlhRkatJHFp LsDESGD28im3wT8s9d1zTsXktlbzCr9TNlHB6TU8iRIhO8AExCXGpuo+iXji2PQU5Tic +SwA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=DqOS3S5Q8bs6xsDVzGZ0eAGq1zWITCddzDMyyLDcxak=; b=qylgfA1FYYyF6UgpkscNXWUDo0xrvtNV9ka/AsOgYPBrt9TxJDqPsziRTkU9mYbb55 bvLbohCqf9cSeTUqTWTQaAJ04wNc4pYrP1rGcNosGZKDBDP1IgPaUDjn4wnVwBRF/EJO p2Sd/1n78HE2NXB246Zzw8G8bFS57g5L2n8qexdDSWfxJq1/xz3jsD8mVYdAcn80ZAF9 H3GwMtQx9Xi1aB18pNkb1cu1ujyMof6TAduXaAGe2oBaj3g1pAFNpBUiJJCOiTkHbtII FRF/TmCLGit4QnPzH8PJuKyksTz3NaQND5iKv0KCWHcI7uL6KQC7bClvIOCod1t8CBJN 07zQ== X-Gm-Message-State: AOAM5327jadHfowIW2ns6ron17zPcqfOjEtLAWhgGPjI12/fyx9DJbTO LHK70nQmtg4tT9WMb/trY5UvtdFXa0k= X-Google-Smtp-Source: ABdhPJxc4wXYlvKcSGzOGK88d6dK2VK/hA38Qy59uYLwGX0yDG70snaEJqj2AAPP15DWEI1FLraapA== X-Received: by 2002:a17:90b:1b42:: with SMTP id nv2mr26477520pjb.190.1618194706726; Sun, 11 Apr 2021 19:31:46 -0700 (PDT) Received: from localhost.localdomain (S0106bc4dfba470f3.vc.shawcable.net. [174.7.244.175]) by smtp.gmail.com with ESMTPSA id q95sm9258811pjq.20.2021.04.11.19.31.46 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Sun, 11 Apr 2021 19:31:46 -0700 (PDT) From: mindmark@gmail.com To: ffmpeg-devel@ffmpeg.org Date: Sun, 11 Apr 2021 19:31:42 -0700 Message-Id: <20210412023142.8470-2-mindmark@gmail.com> X-Mailer: git-send-email 2.29.2 In-Reply-To: <20210412023142.8470-1-mindmark@gmail.com> References: <20210412023142.8470-1-mindmark@gmail.com> MIME-Version: 1.0 Subject: [FFmpeg-devel] [PATCH v2 2/2] libavdevice/avfoundation: add option to set audio sample rate and use native device formats X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Cc: Mark Reid Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" From: Mark Reid This also seems to prevent the audio format changing after format has been identified. This can happen in ffplay and might have something to do with sdl configuring the audio devices. --- libavdevice/avfoundation.m | 123 ++++++++++++++++++++++++++++--------- 1 file changed, 94 insertions(+), 29 deletions(-) diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m index 5ac6ec4183..70226cfdc8 100644 --- a/libavdevice/avfoundation.m +++ b/libavdevice/avfoundation.m @@ -118,11 +118,8 @@ typedef struct int audio_channels; int audio_bits_per_sample; - int audio_float; - int audio_be; - int audio_signed_integer; - int audio_packed; int audio_non_interleaved; + int audio_sample_rate; int32_t *audio_buffer; int audio_buffer_size; @@ -632,12 +629,47 @@ static int add_video_device(AVFormatContext *s, AVCaptureDevice *video_device) return 0; } +static enum AVCodecID find_audio_codec_id(const AudioStreamBasicDescription *basic_desc) +{ + int audio_float = basic_desc->mFormatFlags & kAudioFormatFlagIsFloat; + int audio_signed_integer = basic_desc->mFormatFlags & kAudioFormatFlagIsSignedInteger; + int audio_be = basic_desc->mFormatFlags & kAudioFormatFlagIsBigEndian; + int audio_packed = basic_desc->mFormatFlags & kAudioFormatFlagIsPacked; + int audio_bits_per_sample = basic_desc->mBitsPerChannel; + + if (basic_desc->mFormatID == kAudioFormatLinearPCM && + audio_float && + audio_bits_per_sample == 32 && + audio_packed) { + return audio_be ? AV_CODEC_ID_PCM_F32BE : AV_CODEC_ID_PCM_F32LE; + } else if (basic_desc->mFormatID == kAudioFormatLinearPCM && + audio_signed_integer && + audio_bits_per_sample == 16 && + audio_packed) { + return audio_be ? AV_CODEC_ID_PCM_S16BE : AV_CODEC_ID_PCM_S16LE; + } else if (basic_desc->mFormatID == kAudioFormatLinearPCM && + audio_signed_integer && + audio_bits_per_sample == 24 && + audio_packed) { + return audio_be ? AV_CODEC_ID_PCM_S24BE : AV_CODEC_ID_PCM_S24LE; + } else if (basic_desc->mFormatID == kAudioFormatLinearPCM && + audio_signed_integer && + audio_bits_per_sample == 32 && + audio_packed) { + return audio_be ? AV_CODEC_ID_PCM_S32BE : AV_CODEC_ID_PCM_S32LE; + } else { + return AV_CODEC_ID_NONE; + } +} + static int add_audio_device(AVFormatContext *s, AVCaptureDevice *audio_device) { AVFContext *ctx = (AVFContext*)s->priv_data; NSError *error = nil; AVCaptureDeviceInput* audio_dev_input = [[[AVCaptureDeviceInput alloc] initWithDevice:audio_device error:&error] autorelease]; dispatch_queue_t queue; + NSObject *format = nil; + const AudioStreamBasicDescription *format_desc = NULL; if (!audio_dev_input) { av_log(s, AV_LOG_ERROR, "Failed to create AV capture input device: %s\n", @@ -660,6 +692,61 @@ static int add_audio_device(AVFormatContext *s, AVCaptureDevice *audio_device) return 1; } +#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070 + + for (format in [audio_device valueForKey:@"formats"]) { + CMFormatDescriptionRef formatDescription; + formatDescription = (CMFormatDescriptionRef) [format performSelector:@selector(formatDescription)]; + const AudioStreamBasicDescription *desc = CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription); + + if (desc->mSampleRate == ctx->audio_sample_rate) { + format_desc = desc; + break; + } + } + + if(!format_desc) { + av_log(s, AV_LOG_ERROR, "Selected audio sample rate (%d Hz) is not supported\n", ctx->audio_sample_rate); + av_log(s, AV_LOG_ERROR, "Supported audio formats:\n"); + for (format in [audio_device valueForKey:@"formats"]) { + const char *codec_name; + CMFormatDescriptionRef formatDescription; + formatDescription = (CMFormatDescriptionRef) [format performSelector:@selector(formatDescription)]; + const AudioStreamBasicDescription *desc = CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription); + + enum AVCodecID codec_id = find_audio_codec_id(desc); + if (codec_id == AV_CODEC_ID_NONE) { + continue; + } + + codec_name = avcodec_get_name(codec_id); + av_log(s, AV_LOG_ERROR, " %s, %d ch, %0.0f Hz \n", codec_name, desc->mChannelsPerFrame, desc->mSampleRate); + } + + format_desc = CMAudioFormatDescriptionGetStreamBasicDescription(audio_device.activeFormat.formatDescription); + if (format_desc) + av_log(s, AV_LOG_WARNING, "Overriding selected sample rate with active sample rate: %0.0f Hz instead\n", format_desc->mSampleRate); + } + + if (format_desc) { + int is_float = format_desc->mFormatFlags & kAudioFormatFlagIsFloat; + int audio_non_interleaved = format_desc->mFormatFlags & kAudioFormatFlagIsNonInterleaved; + int audio_be = format_desc->mFormatFlags & kAudioFormatFlagIsBigEndian; + + NSDictionary *settings = [NSDictionary dictionaryWithObjectsAndKeys: + [NSNumber numberWithUnsignedInt:kAudioFormatLinearPCM], AVFormatIDKey, + [NSNumber numberWithFloat:format_desc->mSampleRate], AVSampleRateKey, + [NSNumber numberWithUnsignedInteger:format_desc->mChannelsPerFrame], AVNumberOfChannelsKey, + [NSNumber numberWithInt:format_desc->mBitsPerChannel], AVLinearPCMBitDepthKey, + [NSNumber numberWithBool:is_float], AVLinearPCMIsFloatKey, + [NSNumber numberWithBool:audio_non_interleaved], AVLinearPCMIsNonInterleaved, + [NSNumber numberWithBool:audio_be], AVLinearPCMIsBigEndianKey, + nil]; + + ctx->audio_output.audioSettings = settings; + } +#endif + ctx->avf_audio_delegate = [[AVFAudioReceiver alloc] initWithContext:ctx]; queue = dispatch_queue_create("avf_audio_queue", NULL); @@ -769,33 +856,10 @@ static int get_audio_config(AVFormatContext *s) ctx->audio_channels = basic_desc->mChannelsPerFrame; ctx->audio_bits_per_sample = basic_desc->mBitsPerChannel; - ctx->audio_float = basic_desc->mFormatFlags & kAudioFormatFlagIsFloat; - ctx->audio_be = basic_desc->mFormatFlags & kAudioFormatFlagIsBigEndian; - ctx->audio_signed_integer = basic_desc->mFormatFlags & kAudioFormatFlagIsSignedInteger; - ctx->audio_packed = basic_desc->mFormatFlags & kAudioFormatFlagIsPacked; ctx->audio_non_interleaved = basic_desc->mFormatFlags & kAudioFormatFlagIsNonInterleaved; - if (basic_desc->mFormatID == kAudioFormatLinearPCM && - ctx->audio_float && - ctx->audio_bits_per_sample == 32 && - ctx->audio_packed) { - stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_F32BE : AV_CODEC_ID_PCM_F32LE; - } else if (basic_desc->mFormatID == kAudioFormatLinearPCM && - ctx->audio_signed_integer && - ctx->audio_bits_per_sample == 16 && - ctx->audio_packed) { - stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S16BE : AV_CODEC_ID_PCM_S16LE; - } else if (basic_desc->mFormatID == kAudioFormatLinearPCM && - ctx->audio_signed_integer && - ctx->audio_bits_per_sample == 24 && - ctx->audio_packed) { - stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S24BE : AV_CODEC_ID_PCM_S24LE; - } else if (basic_desc->mFormatID == kAudioFormatLinearPCM && - ctx->audio_signed_integer && - ctx->audio_bits_per_sample == 32 && - ctx->audio_packed) { - stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S32BE : AV_CODEC_ID_PCM_S32LE; - } else { + stream->codecpar->codec_id = find_audio_codec_id(basic_desc); + if (stream->codecpar->codec_id == AV_CODEC_ID_NONE) { av_log(s, AV_LOG_ERROR, "audio format is not supported\n"); return 1; } @@ -1286,6 +1350,7 @@ static const AVOption options[] = { { "pixel_format", "set pixel format", offsetof(AVFContext, pixel_format), AV_OPT_TYPE_PIXEL_FMT, {.i64 = AV_PIX_FMT_YUV420P}, 0, INT_MAX, AV_OPT_FLAG_DECODING_PARAM}, { "framerate", "set frame rate", offsetof(AVFContext, framerate), AV_OPT_TYPE_VIDEO_RATE, {.str = "ntsc"}, 0, INT_MAX, AV_OPT_FLAG_DECODING_PARAM }, { "video_size", "set video size", offsetof(AVFContext, width), AV_OPT_TYPE_IMAGE_SIZE, {.str = NULL}, 0, 0, AV_OPT_FLAG_DECODING_PARAM }, + { "audio_sample_rate", "set audio sample rate in Hz", offsetof(AVFContext, audio_sample_rate), AV_OPT_TYPE_INT, {.i64 = 48000}, 1, INT_MAX, AV_OPT_FLAG_DECODING_PARAM }, { "capture_cursor", "capture the screen cursor", offsetof(AVFContext, capture_cursor), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM }, { "capture_mouse_clicks", "capture the screen mouse clicks", offsetof(AVFContext, capture_mouse_clicks), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM }, { "capture_raw_data", "capture the raw data from device connection", offsetof(AVFContext, capture_raw_data), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },