From patchwork Mon Jan 9 18:22:22 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavel Koshevoy X-Patchwork-Id: 2145 Delivered-To: ffmpegpatchwork@gmail.com Received: by 10.103.89.21 with SMTP id n21csp7171447vsb; Mon, 9 Jan 2017 10:22:49 -0800 (PST) X-Received: by 10.28.229.73 with SMTP id c70mr1640471wmh.82.1483986169795; Mon, 09 Jan 2017 10:22:49 -0800 (PST) Return-Path: Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org. [79.124.17.100]) by mx.google.com with ESMTP id t2si822257wma.109.2017.01.09.10.22.48; Mon, 09 Jan 2017 10:22:49 -0800 (PST) Received-SPF: pass (google.com: domain of ffmpeg-devel-bounces@ffmpeg.org designates 79.124.17.100 as permitted sender) client-ip=79.124.17.100; Authentication-Results: mx.google.com; dkim=neutral (body hash did not verify) header.i=@gmail.com; spf=pass (google.com: domain of ffmpeg-devel-bounces@ffmpeg.org designates 79.124.17.100 as permitted sender) smtp.mailfrom=ffmpeg-devel-bounces@ffmpeg.org; dmarc=fail (p=NONE sp=NONE dis=NONE) header.from=gmail.com Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 62BD4680A7F; Mon, 9 Jan 2017 20:22:38 +0200 (EET) X-Original-To: ffmpeg-devel@ffmpeg.org Delivered-To: ffmpeg-devel@ffmpeg.org Received: from mail-pf0-f193.google.com (mail-pf0-f193.google.com [209.85.192.193]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id A0B8068028E for ; Mon, 9 Jan 2017 20:22:31 +0200 (EET) Received: by mail-pf0-f193.google.com with SMTP id y143so3912625pfb.1 for ; Mon, 09 Jan 2017 10:22:38 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20161025; h=from:to:cc:subject:date:message-id:in-reply-to:references; bh=s3KeqmatDtlGne/nbK2Cf1WL9EtPsKXdA8ICMkjfUgc=; b=F1j1jImpXAZvw1E4UJoXNL/ilWbLDyy4nAc4W0nkEZi/n1ikVvnznrQvxgAYQxA7b0 r2RbQewaMkYAYJflDKFnFdGocU4BZpnjz+8TUShLh/5e49IuyZUr4X39C6zUHqF1dsKK wnTwMwfGrAnFxxx97MN69dQcZxLVDu77jOqYsnB/eCMTfd26mIF6pqhtt+qeqlLF9fsr H4MUecw5N9YEq5wodlr+O0WvPz8v1rIbQMjLlVpYYuGKyE3w/5tZN4w228okUHDmPLQ2 mCmX9pRPAPObawQhR1iJYmUp4sem/qMmdMovyZUbJXc3LzQH4Ql6OFiw0zKuqarOAGvk gUXw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references; bh=s3KeqmatDtlGne/nbK2Cf1WL9EtPsKXdA8ICMkjfUgc=; b=Z/YDf1ZB3BOfe93xO+hD/em6FlKoFuVrxGfSJGi2x5vldlLioAzIyXP+6TAWKTqRv9 TjV0XiPl1AINbHm28CyHqZmO2Io+T3kJNgdmbrJLYTW+M4jgXU4LUrPqjDgO+daAZGSr T9mf1IddbENqzomYUklxU8su+NgTPLfhJ0k7OYg1O/rSe3dlCso1JRajw41fnJroH1ox y7a2OA3EsR+M9HWzN4XU5baCTCKldmn2ziiWmfkahrsllIbVM3h2+cCwKUOEZgbEzV/v FHuuW2/J11SGg4LfvwVL3go2BiIk8F0bco+4MV72X8i88c7AxZk/wt7TAHp+FrxpYUmC Sf5A== X-Gm-Message-State: AIkVDXIPKkPXIDxtgRjry3Ya56XK1A6V2On0CByJA03X7nSHeKlWSxjneg4Fl31g8ZT+Dw== X-Received: by 10.84.248.70 with SMTP id e6mr24192449pln.179.1483986157400; Mon, 09 Jan 2017 10:22:37 -0800 (PST) Received: from homestead.aragog.com ([136.60.170.51]) by smtp.gmail.com with ESMTPSA id y23sm728896pfi.66.2017.01.09.10.22.36 (version=TLS1_2 cipher=ECDHE-RSA-AES128-SHA bits=128/128); Mon, 09 Jan 2017 10:22:36 -0800 (PST) From: pkoshevoy@gmail.com To: ffmpeg-devel@ffmpeg.org Date: Mon, 9 Jan 2017 11:22:22 -0700 Message-Id: <1483986142-27328-1-git-send-email-pkoshevoy@gmail.com> X-Mailer: git-send-email 2.6.6 In-Reply-To: References: Subject: [FFmpeg-devel] [PATCH] lavc/cuvid: fail early if GPU can't handle video parameters X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Cc: Pavel Koshevoy MIME-Version: 1.0 Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" From: Pavel Koshevoy Evidently CUVID doesn't support decoding 422 or 444 chroma formats, and only a limited set of resolutions per codec are supported. Given that stream resolution and pixel format are typically known as a result of probing it is better to use this info during avcodec_open2 call and fail early in case the video parameters are not supported, rather than failing later during avcodec_send_packet calls. This problem surfaced when trying to decode 5120x2700 h246 video on GeForce GT 730, or when decoding 422 mpeg2/h264 streams on same GPU - avcodec_open2 succeeds but decoding fails. --- libavcodec/cuvid.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 5 deletions(-) diff --git a/libavcodec/cuvid.c b/libavcodec/cuvid.c index 8fc713d..ed21f94 100644 --- a/libavcodec/cuvid.c +++ b/libavcodec/cuvid.c @@ -612,7 +612,11 @@ static av_cold int cuvid_decode_end(AVCodecContext *avctx) return 0; } -static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cuparseinfo) +static int cuvid_test_dummy_decoder(AVCodecContext *avctx, + const CUVIDPARSERPARAMS *cuparseinfo, + cudaVideoChromaFormat probed_chroma_format, + int probed_width, + int probed_height) { CuvidContext *ctx = avctx->priv_data; CUVIDDECODECREATEINFO cuinfo; @@ -622,11 +626,11 @@ static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cu memset(&cuinfo, 0, sizeof(cuinfo)); cuinfo.CodecType = cuparseinfo->CodecType; - cuinfo.ChromaFormat = cudaVideoChromaFormat_420; + cuinfo.ChromaFormat = probed_chroma_format; cuinfo.OutputFormat = cudaVideoSurfaceFormat_NV12; - cuinfo.ulWidth = 1280; - cuinfo.ulHeight = 720; + cuinfo.ulWidth = probed_width; + cuinfo.ulHeight = probed_height; cuinfo.ulTargetWidth = cuinfo.ulWidth; cuinfo.ulTargetHeight = cuinfo.ulHeight; @@ -653,6 +657,36 @@ static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cu return 0; } +static int convert_to_cuda_video_chroma_format(enum AVPixelFormat pix_fmt, + cudaVideoChromaFormat *out) +{ + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt); + if (!(out && desc && + (desc->nb_components == 1 || desc->nb_components == 3) && + (desc->log2_chroma_w < 2 && desc->log2_chroma_h < 2))) + { + return AVERROR(EINVAL); + } + + if (desc->nb_components == 1) + { + *out = cudaVideoChromaFormat_Monochrome; + } + else if (desc->flags == AV_PIX_FMT_FLAG_PLANAR) + { + *out = ((desc->log2_chroma_w == 0) ? cudaVideoChromaFormat_444 : + (desc->log2_chroma_h == 0) ? cudaVideoChromaFormat_422 : + cudaVideoChromaFormat_420); + } + else + { + return AVERROR(EINVAL); + } + + // unfortunately, 420 is the only one that works: + return (*out == cudaVideoChromaFormat_420) ? 0 : AVERROR_EXTERNAL; +} + static av_cold int cuvid_decode_init(AVCodecContext *avctx) { CuvidContext *ctx = avctx->priv_data; @@ -663,12 +697,27 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx) CUcontext cuda_ctx = NULL; CUcontext dummy; const AVBitStreamFilter *bsf; + cudaVideoChromaFormat probed_chroma_format; + int probed_width; + int probed_height; int ret = 0; enum AVPixelFormat pix_fmts[3] = { AV_PIX_FMT_CUDA, AV_PIX_FMT_NV12, AV_PIX_FMT_NONE }; + enum AVPixelFormat probed_pix_fmt = (avctx->pix_fmt < 0 ? + AV_PIX_FMT_YUV420P : + avctx->pix_fmt); + + ret = convert_to_cuda_video_chroma_format(probed_pix_fmt, &probed_chroma_format); + if (ret < 0) { + // pixel format is not supported: + return ret; + } + probed_width = avctx->coded_width ? avctx->coded_width : 1280; + probed_height = avctx->coded_height ? avctx->coded_height : 720; + // Accelerated transcoding scenarios with 'ffmpeg' require that the // pix_fmt be set to AV_PIX_FMT_CUDA early. The sw_pix_fmt, and the // pix_fmt for non-accelerated transcoding, do not need to be correct @@ -824,7 +873,10 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx) if (ret < 0) goto error; - ret = cuvid_test_dummy_decoder(avctx, &ctx->cuparseinfo); + ret = cuvid_test_dummy_decoder(avctx, &ctx->cuparseinfo, + probed_chroma_format, + probed_width, + probed_height); if (ret < 0) goto error;