| Message ID | 1483986142-27328-1-git-send-email-pkoshevoy@gmail.com |
|---|---|
| State | Superseded |
| Headers | show |
On 1/9/2017 7:22 PM, pkoshevoy@gmail.com wrote: > From: Pavel Koshevoy <pkoshevoy@gmail.com> > > Evidently CUVID doesn't support decoding 422 or 444 chroma formats, > and only a limited set of resolutions per codec are supported. > > Given that stream resolution and pixel format are typically known as a > result of probing it is better to use this info during avcodec_open2 > call and fail early in case the video parameters are not supported, > rather than failing later during avcodec_send_packet calls. > > This problem surfaced when trying to decode 5120x2700 h246 video on > GeForce GT 730, or when decoding 422 mpeg2/h264 streams on same GPU - > avcodec_open2 succeeds but decoding fails. > --- > libavcodec/cuvid.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++++----- > 1 file changed, 57 insertions(+), 5 deletions(-) > > diff --git a/libavcodec/cuvid.c b/libavcodec/cuvid.c > index 8fc713d..ed21f94 100644 > --- a/libavcodec/cuvid.c > +++ b/libavcodec/cuvid.c > @@ -612,7 +612,11 @@ static av_cold int cuvid_decode_end(AVCodecContext *avctx) > return 0; > } > > -static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cuparseinfo) > +static int cuvid_test_dummy_decoder(AVCodecContext *avctx, > + const CUVIDPARSERPARAMS *cuparseinfo, > + cudaVideoChromaFormat probed_chroma_format, > + int probed_width, > + int probed_height) > { > CuvidContext *ctx = avctx->priv_data; > CUVIDDECODECREATEINFO cuinfo; > @@ -622,11 +626,11 @@ static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cu > memset(&cuinfo, 0, sizeof(cuinfo)); > > cuinfo.CodecType = cuparseinfo->CodecType; > - cuinfo.ChromaFormat = cudaVideoChromaFormat_420; > + cuinfo.ChromaFormat = probed_chroma_format; > cuinfo.OutputFormat = cudaVideoSurfaceFormat_NV12; > > - cuinfo.ulWidth = 1280; > - cuinfo.ulHeight = 720; > + cuinfo.ulWidth = probed_width; > + cuinfo.ulHeight = probed_height; > cuinfo.ulTargetWidth = cuinfo.ulWidth; > cuinfo.ulTargetHeight = cuinfo.ulHeight; > > @@ -653,6 +657,36 @@ static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cu > return 0; > } > > +static int convert_to_cuda_video_chroma_format(enum AVPixelFormat pix_fmt, > + cudaVideoChromaFormat *out) > +{ > + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt); > + if (!(out && desc && > + (desc->nb_components == 1 || desc->nb_components == 3) && > + (desc->log2_chroma_w < 2 && desc->log2_chroma_h < 2))) > + { > + return AVERROR(EINVAL); > + } > + > + if (desc->nb_components == 1) > + { > + *out = cudaVideoChromaFormat_Monochrome; > + } > + else if (desc->flags == AV_PIX_FMT_FLAG_PLANAR) > + { > + *out = ((desc->log2_chroma_w == 0) ? cudaVideoChromaFormat_444 : > + (desc->log2_chroma_h == 0) ? cudaVideoChromaFormat_422 : > + cudaVideoChromaFormat_420); > + } > + else > + { > + return AVERROR(EINVAL); > + } > + > + // unfortunately, 420 is the only one that works: > + return (*out == cudaVideoChromaFormat_420) ? 0 : AVERROR_EXTERNAL; > +} > + > static av_cold int cuvid_decode_init(AVCodecContext *avctx) > { > CuvidContext *ctx = avctx->priv_data; > @@ -663,12 +697,27 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx) > CUcontext cuda_ctx = NULL; > CUcontext dummy; > const AVBitStreamFilter *bsf; > + cudaVideoChromaFormat probed_chroma_format; > + int probed_width; > + int probed_height; > int ret = 0; > > enum AVPixelFormat pix_fmts[3] = { AV_PIX_FMT_CUDA, > AV_PIX_FMT_NV12, > AV_PIX_FMT_NONE }; > > + enum AVPixelFormat probed_pix_fmt = (avctx->pix_fmt < 0 ? Shouldn't this be <= 0? If unset it will most likely be 0. Also, the braces seem unneccesary or misplaced. > + AV_PIX_FMT_YUV420P : > + avctx->pix_fmt); > + > + ret = convert_to_cuda_video_chroma_format(probed_pix_fmt, &probed_chroma_format); > + if (ret < 0) { > + // pixel format is not supported: Printing an error here would be better, instead of failing init without any indication what went wrong. > + return ret; > + } > + probed_width = avctx->coded_width ? avctx->coded_width : 1280; > + probed_height = avctx->coded_height ? avctx->coded_height : 720; > + > // Accelerated transcoding scenarios with 'ffmpeg' require that the > // pix_fmt be set to AV_PIX_FMT_CUDA early. The sw_pix_fmt, and the > // pix_fmt for non-accelerated transcoding, do not need to be correct > @@ -824,7 +873,10 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx) > if (ret < 0) > goto error; > > - ret = cuvid_test_dummy_decoder(avctx, &ctx->cuparseinfo); > + ret = cuvid_test_dummy_decoder(avctx, &ctx->cuparseinfo, > + probed_chroma_format, > + probed_width, > + probed_height); > if (ret < 0) > goto error; > > LGTM otherwise
On 01/12/2017 10:32 AM, Timo Rothenpieler wrote: > On 1/9/2017 7:22 PM, pkoshevoy@gmail.com wrote: >> From: Pavel Koshevoy <pkoshevoy@gmail.com> >> >> Evidently CUVID doesn't support decoding 422 or 444 chroma formats, >> and only a limited set of resolutions per codec are supported. >> >> Given that stream resolution and pixel format are typically known as a >> result of probing it is better to use this info during avcodec_open2 >> call and fail early in case the video parameters are not supported, >> rather than failing later during avcodec_send_packet calls. >> >> This problem surfaced when trying to decode 5120x2700 h246 video on >> GeForce GT 730, or when decoding 422 mpeg2/h264 streams on same GPU - >> avcodec_open2 succeeds but decoding fails. >> --- >> libavcodec/cuvid.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++++----- >> 1 file changed, 57 insertions(+), 5 deletions(-) >> >> diff --git a/libavcodec/cuvid.c b/libavcodec/cuvid.c >> index 8fc713d..ed21f94 100644 >> --- a/libavcodec/cuvid.c >> +++ b/libavcodec/cuvid.c >> @@ -612,7 +612,11 @@ static av_cold int cuvid_decode_end(AVCodecContext *avctx) >> return 0; >> } >> >> -static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cuparseinfo) >> +static int cuvid_test_dummy_decoder(AVCodecContext *avctx, >> + const CUVIDPARSERPARAMS *cuparseinfo, >> + cudaVideoChromaFormat probed_chroma_format, >> + int probed_width, >> + int probed_height) >> { >> CuvidContext *ctx = avctx->priv_data; >> CUVIDDECODECREATEINFO cuinfo; >> @@ -622,11 +626,11 @@ static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cu >> memset(&cuinfo, 0, sizeof(cuinfo)); >> >> cuinfo.CodecType = cuparseinfo->CodecType; >> - cuinfo.ChromaFormat = cudaVideoChromaFormat_420; >> + cuinfo.ChromaFormat = probed_chroma_format; >> cuinfo.OutputFormat = cudaVideoSurfaceFormat_NV12; >> >> - cuinfo.ulWidth = 1280; >> - cuinfo.ulHeight = 720; >> + cuinfo.ulWidth = probed_width; >> + cuinfo.ulHeight = probed_height; >> cuinfo.ulTargetWidth = cuinfo.ulWidth; >> cuinfo.ulTargetHeight = cuinfo.ulHeight; >> >> @@ -653,6 +657,36 @@ static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cu >> return 0; >> } >> >> +static int convert_to_cuda_video_chroma_format(enum AVPixelFormat pix_fmt, >> + cudaVideoChromaFormat *out) >> +{ >> + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt); >> + if (!(out && desc && >> + (desc->nb_components == 1 || desc->nb_components == 3) && >> + (desc->log2_chroma_w < 2 && desc->log2_chroma_h < 2))) >> + { >> + return AVERROR(EINVAL); >> + } >> + >> + if (desc->nb_components == 1) >> + { >> + *out = cudaVideoChromaFormat_Monochrome; >> + } >> + else if (desc->flags == AV_PIX_FMT_FLAG_PLANAR) >> + { >> + *out = ((desc->log2_chroma_w == 0) ? cudaVideoChromaFormat_444 : >> + (desc->log2_chroma_h == 0) ? cudaVideoChromaFormat_422 : >> + cudaVideoChromaFormat_420); >> + } >> + else >> + { >> + return AVERROR(EINVAL); >> + } >> + >> + // unfortunately, 420 is the only one that works: >> + return (*out == cudaVideoChromaFormat_420) ? 0 : AVERROR_EXTERNAL; >> +} >> + >> static av_cold int cuvid_decode_init(AVCodecContext *avctx) >> { >> CuvidContext *ctx = avctx->priv_data; >> @@ -663,12 +697,27 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx) >> CUcontext cuda_ctx = NULL; >> CUcontext dummy; >> const AVBitStreamFilter *bsf; >> + cudaVideoChromaFormat probed_chroma_format; >> + int probed_width; >> + int probed_height; >> int ret = 0; >> >> enum AVPixelFormat pix_fmts[3] = { AV_PIX_FMT_CUDA, >> AV_PIX_FMT_NV12, >> AV_PIX_FMT_NONE }; >> >> + enum AVPixelFormat probed_pix_fmt = (avctx->pix_fmt < 0 ? > Shouldn't this be <= 0? If unset it will most likely be 0. > Also, the braces seem unneccesary or misplaced. > >> + AV_PIX_FMT_YUV420P : >> + avctx->pix_fmt); >> + >> + ret = convert_to_cuda_video_chroma_format(probed_pix_fmt, &probed_chroma_format); >> + if (ret < 0) { >> + // pixel format is not supported: > Printing an error here would be better, instead of failing init without > any indication what went wrong. > >> + return ret; >> + } >> + probed_width = avctx->coded_width ? avctx->coded_width : 1280; >> + probed_height = avctx->coded_height ? avctx->coded_height : 720; >> + >> // Accelerated transcoding scenarios with 'ffmpeg' require that the >> // pix_fmt be set to AV_PIX_FMT_CUDA early. The sw_pix_fmt, and the >> // pix_fmt for non-accelerated transcoding, do not need to be correct >> @@ -824,7 +873,10 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx) >> if (ret < 0) >> goto error; >> >> - ret = cuvid_test_dummy_decoder(avctx, &ctx->cuparseinfo); >> + ret = cuvid_test_dummy_decoder(avctx, &ctx->cuparseinfo, >> + probed_chroma_format, >> + probed_width, >> + probed_height); >> if (ret < 0) >> goto error; >> >> > > LGTM otherwise This patch was rejected by wm4, so consider it dropped. Pavel.
On Thu, Jan 12, 2017 at 10:32 AM, Timo Rothenpieler <timo@rothenpieler.org> wrote: > On 1/9/2017 7:22 PM, pkoshevoy@gmail.com wrote: >> From: Pavel Koshevoy <pkoshevoy@gmail.com> <snip> >> enum AVPixelFormat pix_fmts[3] = { AV_PIX_FMT_CUDA, >> AV_PIX_FMT_NV12, >> AV_PIX_FMT_NONE }; >> >> + enum AVPixelFormat probed_pix_fmt = (avctx->pix_fmt < 0 ? > > Shouldn't this be <= 0? If unset it will most likely be 0. fixed and resubmitted in patch v4. >> + AV_PIX_FMT_YUV420P : >> + avctx->pix_fmt); >> + >> + ret = convert_to_cuda_video_chroma_format(probed_pix_fmt, &probed_chroma_format); >> + if (ret < 0) { >> + // pixel format is not supported: > > Printing an error here would be better, instead of failing init without > any indication what went wrong. added an av_log call and resubmitted in patch v4. >> + return ret; >> + } >> + probed_width = avctx->coded_width ? avctx->coded_width : 1280; >> + probed_height = avctx->coded_height ? avctx->coded_height : 720; >> + Thank you, Pavel.
diff --git a/libavcodec/cuvid.c b/libavcodec/cuvid.c index 8fc713d..ed21f94 100644 --- a/libavcodec/cuvid.c +++ b/libavcodec/cuvid.c @@ -612,7 +612,11 @@ static av_cold int cuvid_decode_end(AVCodecContext *avctx) return 0; } -static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cuparseinfo) +static int cuvid_test_dummy_decoder(AVCodecContext *avctx, + const CUVIDPARSERPARAMS *cuparseinfo, + cudaVideoChromaFormat probed_chroma_format, + int probed_width, + int probed_height) { CuvidContext *ctx = avctx->priv_data; CUVIDDECODECREATEINFO cuinfo; @@ -622,11 +626,11 @@ static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cu memset(&cuinfo, 0, sizeof(cuinfo)); cuinfo.CodecType = cuparseinfo->CodecType; - cuinfo.ChromaFormat = cudaVideoChromaFormat_420; + cuinfo.ChromaFormat = probed_chroma_format; cuinfo.OutputFormat = cudaVideoSurfaceFormat_NV12; - cuinfo.ulWidth = 1280; - cuinfo.ulHeight = 720; + cuinfo.ulWidth = probed_width; + cuinfo.ulHeight = probed_height; cuinfo.ulTargetWidth = cuinfo.ulWidth; cuinfo.ulTargetHeight = cuinfo.ulHeight; @@ -653,6 +657,36 @@ static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cu return 0; } +static int convert_to_cuda_video_chroma_format(enum AVPixelFormat pix_fmt, + cudaVideoChromaFormat *out) +{ + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt); + if (!(out && desc && + (desc->nb_components == 1 || desc->nb_components == 3) && + (desc->log2_chroma_w < 2 && desc->log2_chroma_h < 2))) + { + return AVERROR(EINVAL); + } + + if (desc->nb_components == 1) + { + *out = cudaVideoChromaFormat_Monochrome; + } + else if (desc->flags == AV_PIX_FMT_FLAG_PLANAR) + { + *out = ((desc->log2_chroma_w == 0) ? cudaVideoChromaFormat_444 : + (desc->log2_chroma_h == 0) ? cudaVideoChromaFormat_422 : + cudaVideoChromaFormat_420); + } + else + { + return AVERROR(EINVAL); + } + + // unfortunately, 420 is the only one that works: + return (*out == cudaVideoChromaFormat_420) ? 0 : AVERROR_EXTERNAL; +} + static av_cold int cuvid_decode_init(AVCodecContext *avctx) { CuvidContext *ctx = avctx->priv_data; @@ -663,12 +697,27 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx) CUcontext cuda_ctx = NULL; CUcontext dummy; const AVBitStreamFilter *bsf; + cudaVideoChromaFormat probed_chroma_format; + int probed_width; + int probed_height; int ret = 0; enum AVPixelFormat pix_fmts[3] = { AV_PIX_FMT_CUDA, AV_PIX_FMT_NV12, AV_PIX_FMT_NONE }; + enum AVPixelFormat probed_pix_fmt = (avctx->pix_fmt < 0 ? + AV_PIX_FMT_YUV420P : + avctx->pix_fmt); + + ret = convert_to_cuda_video_chroma_format(probed_pix_fmt, &probed_chroma_format); + if (ret < 0) { + // pixel format is not supported: + return ret; + } + probed_width = avctx->coded_width ? avctx->coded_width : 1280; + probed_height = avctx->coded_height ? avctx->coded_height : 720; + // Accelerated transcoding scenarios with 'ffmpeg' require that the // pix_fmt be set to AV_PIX_FMT_CUDA early. The sw_pix_fmt, and the // pix_fmt for non-accelerated transcoding, do not need to be correct @@ -824,7 +873,10 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx) if (ret < 0) goto error; - ret = cuvid_test_dummy_decoder(avctx, &ctx->cuparseinfo); + ret = cuvid_test_dummy_decoder(avctx, &ctx->cuparseinfo, + probed_chroma_format, + probed_width, + probed_height); if (ret < 0) goto error;
From: Pavel Koshevoy <pkoshevoy@gmail.com> Evidently CUVID doesn't support decoding 422 or 444 chroma formats, and only a limited set of resolutions per codec are supported. Given that stream resolution and pixel format are typically known as a result of probing it is better to use this info during avcodec_open2 call and fail early in case the video parameters are not supported, rather than failing later during avcodec_send_packet calls. This problem surfaced when trying to decode 5120x2700 h246 video on GeForce GT 730, or when decoding 422 mpeg2/h264 streams on same GPU - avcodec_open2 succeeds but decoding fails. --- libavcodec/cuvid.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 5 deletions(-)