From patchwork Mon Apr 12 20:21:25 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Dhanish Vijayan X-Patchwork-Id: 26886 Return-Path: X-Original-To: patchwork@ffaux-bg.ffmpeg.org Delivered-To: patchwork@ffaux-bg.ffmpeg.org Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org [79.124.17.100]) by ffaux.localdomain (Postfix) with ESMTP id DE04644A885 for ; Mon, 12 Apr 2021 23:29:19 +0300 (EEST) Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id B2B99688178; Mon, 12 Apr 2021 23:29:19 +0300 (EEST) X-Original-To: ffmpeg-devel@ffmpeg.org Delivered-To: ffmpeg-devel@ffmpeg.org Received: from mail-pl1-f173.google.com (mail-pl1-f173.google.com [209.85.214.173]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id 1E7896804B3 for ; Mon, 12 Apr 2021 23:29:13 +0300 (EEST) Received: by mail-pl1-f173.google.com with SMTP id t22so6675851ply.1 for ; Mon, 12 Apr 2021 13:29:13 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20161025; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=op7fvNLxtx6aZkPEZ67cBlYhvUon/QhWFQOxQyCYfZQ=; b=mRPwjNhhzZ8Dt0lO9QWsSarIPTbKyUFb0GuGEe/Du8CJeZIhll+yi1o15RH9tSa3sS TJE+Zke9TSc1fEkBcUSKu99QIS7nuogl0gXMQS4soFBgN3MASa47BDz3bDN33ZPlKEsf hTyvXGm0KK9FEb5ejqp0UCXvhwpKBXq/bcthaGtXfGuK3wlSlekRtnPtS+lFvDcoaAy9 XMR8LQHhouGMSUnzb+fQ0t7syHpw3pPZ1dLgxQ9DO+dNELJIf6ipbSAtJYTwGn0p2g2X u7pDL48wta3fXuWhuQ30OHPwSBBAQPsrY0vmcLgbxxnsOBSiMk+8vKFF0fK+1L/erNc4 Jxog== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=op7fvNLxtx6aZkPEZ67cBlYhvUon/QhWFQOxQyCYfZQ=; b=hT95mf28YwI5ehQteaXXIa1sgh9egM9rXPw78FDdcpIjKgZidDjpLXkatg8/eLV6Zu QAmWgH5jRqCUMwCsgtorFZqjGQjPRZXvj1H7Akm0Dmtn/ZtnWUdfUKFerwsC0WvVTjtm zD5ENx4nVczAKatok3Vr3cNLr2TtRtux8SevHWyff7SYXB4jwlaOmn0vxJ5pJw/lvJHo p5MJ9IMljin5ptHm/X8X8lI8qyPdG7G3RZg7dxK/IpFo0BU4XgSqWZaJmm0KK8BGjllz Tjnc+iArFMwVnmsQ12LNaIMbs8/FpDika0Da+Av4uDyACM2K2LpS7uemamDRZ7azUrFD NG1A== X-Gm-Message-State: AOAM5325MiLEraICDiX4xY/meOPZTOV233NShCjZMMkfwccbSIRJA6c/ mAa+OKkVC48ckQwjEI1zhI3MS+Dwz8NEkEi9 X-Google-Smtp-Source: ABdhPJxPTKA3OjTGQaMYTkA5oK9uXpgt8XKD3joHhIux3oFGJa2i5rDjljW1ggW/5sZrmg/fw0DQJQ== X-Received: by 2002:aa7:9190:0:b029:22d:6789:cc83 with SMTP id x16-20020aa791900000b029022d6789cc83mr26837360pfa.9.1618258895579; Mon, 12 Apr 2021 13:21:35 -0700 (PDT) Received: from localhost.localdomain ([116.68.72.41]) by smtp.googlemail.com with ESMTPSA id e13sm3148450pfd.64.2021.04.12.13.21.34 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Mon, 12 Apr 2021 13:21:35 -0700 (PDT) From: Dhanish Vijayan To: ffmpeg-devel@ffmpeg.org Date: Tue, 13 Apr 2021 01:51:25 +0530 Message-Id: <20210412202125.149031-1-dhanishvijayan@gmail.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: References: MIME-Version: 1.0 Subject: [FFmpeg-devel] [PATCH] Added Closed caption support for cuviddec for preserving a53 data n GPU decoding X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" Signed-off-by: Dhanish Vijayan --- libavcodec/cuviddec.c | 199 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) diff --git a/libavcodec/cuviddec.c b/libavcodec/cuviddec.c index ec57afdefe..3b07d0a874 100644 --- a/libavcodec/cuviddec.c +++ b/libavcodec/cuviddec.c @@ -46,6 +46,9 @@ #define CUVID_HAS_AV1_SUPPORT #endif +#define MAX_FRAME_COUNT 25 +#define A53_QUEUE_SIZE (MAX_FRAME_COUNT + 8) + typedef struct CuvidContext { AVClass *avclass; @@ -89,6 +92,11 @@ typedef struct CuvidContext cudaVideoCodec codec_type; cudaVideoChromaFormat chroma_format; + uint8_t* a53_caption; + int a53_caption_size; + uint8_t* a53_caption_queue[A53_QUEUE_SIZE]; + int a53_caption_size_queue[A53_QUEUE_SIZE]; + CUVIDDECODECAPS caps8, caps10, caps12; CUVIDPARSERPARAMS cuparseinfo; @@ -103,6 +111,8 @@ typedef struct CuvidParsedFrame CUVIDPARSERDISPINFO dispinfo; int second_field; int is_deinterlacing; + uint8_t* a53_caption; + int a53_caption_size; } CuvidParsedFrame; #define CHECK_CU(x) FF_CUDA_CHECK_DL(avctx, ctx->cudl, x) @@ -338,6 +348,24 @@ static int CUDAAPI cuvid_handle_picture_decode(void *opaque, CUVIDPICPARAMS* pic ctx->key_frame[picparams->CurrPicIdx] = picparams->intra_pic_flag; + if (ctx->a53_caption) + { + + if (picparams->CurrPicIdx >= A53_QUEUE_SIZE) + { + av_log(avctx, AV_LOG_WARNING, "CurrPicIdx too big: %d\n", picparams->CurrPicIdx); + av_freep(&ctx->a53_caption); + } + else + { + int pos = picparams->CurrPicIdx; + av_freep(&ctx->a53_caption_queue[pos]); + ctx->a53_caption_queue[pos] = ctx->a53_caption; + ctx->a53_caption_size_queue[pos] = ctx->a53_caption_size; + ctx->a53_caption = NULL; + } + } + ctx->internal_error = CHECK_CU(ctx->cvdl->cuvidDecodePicture(ctx->cudecoder, picparams)); if (ctx->internal_error < 0) return 0; @@ -350,6 +378,20 @@ static int CUDAAPI cuvid_handle_picture_display(void *opaque, CUVIDPARSERDISPINF AVCodecContext *avctx = opaque; CuvidContext *ctx = avctx->priv_data; CuvidParsedFrame parsed_frame = { { 0 } }; + uint8_t* a53_caption = NULL; + int a53_caption_size = 0; + + if (dispinfo->picture_index >= A53_QUEUE_SIZE) + { + av_log(avctx, AV_LOG_WARNING, "picture_index too big: %d\n", dispinfo->picture_index); + } + else + { + int pos = dispinfo->picture_index; + a53_caption = ctx->a53_caption_queue[pos]; + a53_caption_size = ctx->a53_caption_size_queue[pos]; + ctx->a53_caption_queue[pos] = NULL; + } parsed_frame.dispinfo = *dispinfo; ctx->internal_error = 0; @@ -358,11 +400,17 @@ static int CUDAAPI cuvid_handle_picture_display(void *opaque, CUVIDPARSERDISPINF parsed_frame.dispinfo.progressive_frame = ctx->progressive_sequence; if (ctx->deint_mode_current == cudaVideoDeinterlaceMode_Weave) { + parsed_frame.a53_caption = a53_caption; + parsed_frame.a53_caption_size = a53_caption_size; av_fifo_generic_write(ctx->frame_queue, &parsed_frame, sizeof(CuvidParsedFrame), NULL); } else { parsed_frame.is_deinterlacing = 1; + parsed_frame.a53_caption = a53_caption; + parsed_frame.a53_caption_size = a53_caption_size; av_fifo_generic_write(ctx->frame_queue, &parsed_frame, sizeof(CuvidParsedFrame), NULL); if (!ctx->drop_second_field) { + parsed_frame.a53_caption = NULL; + parsed_frame.a53_caption_size = 0; parsed_frame.second_field = 1; av_fifo_generic_write(ctx->frame_queue, &parsed_frame, sizeof(CuvidParsedFrame), NULL); } @@ -382,6 +430,139 @@ static int cuvid_is_buffer_full(AVCodecContext *avctx) return (av_fifo_size(ctx->frame_queue) / sizeof(CuvidParsedFrame)) + delay >= ctx->nb_surfaces; } + +static void cuvid_mpeg_parse_a53(CuvidContext *ctx, const uint8_t* p, int buf_size) +{ + const uint8_t* buf_end = p + buf_size; + for(;;) + { + uint32_t start_code = -1; + p = avpriv_find_start_code(p, buf_end, &start_code); + if (start_code > 0x1ff) + break; + if (start_code != 0x1b2) + continue; + buf_size = buf_end - p; + if (buf_size >= 6 && + p[0] == 'G' && p[1] == 'A' && p[2] == '9' && p[3] == '4' && p[4] == 3 && (p[5] & 0x40)) + { + /* extract A53 Part 4 CC data */ + int cc_count = p[5] & 0x1f; + if (cc_count > 0 && buf_size >= 7 + cc_count * 3) + { + av_freep(&ctx->a53_caption); + ctx->a53_caption_size = cc_count * 3; + ctx->a53_caption = av_malloc(ctx->a53_caption_size); + if (ctx->a53_caption) + memcpy(ctx->a53_caption, p + 7, ctx->a53_caption_size); + } + } + else if (buf_size >= 11 && p[0] == 'C' && p[1] == 'C' && p[2] == 0x01 && p[3] == 0xf8) + { + int cc_count = 0; + int i; + // There is a caption count field in the data, but it is often + // incorrect. So count the number of captions present. + for (i = 5; i + 6 <= buf_size && ((p[i] & 0xfe) == 0xfe); i += 6) + cc_count++; + // Transform the DVD format into A53 Part 4 format + if (cc_count > 0) { + av_freep(&ctx->a53_caption); + ctx->a53_caption_size = cc_count * 6; + ctx->a53_caption = av_malloc(ctx->a53_caption_size); + if (ctx->a53_caption) { + uint8_t field1 = !!(p[4] & 0x80); + uint8_t *cap = ctx->a53_caption; + p += 5; + for (i = 0; i < cc_count; i++) + { + cap[0] = (p[0] == 0xff && field1) ? 0xfc : 0xfd; + cap[1] = p[1]; + cap[2] = p[2]; + cap[3] = (p[3] == 0xff && !field1) ? 0xfc : 0xfd; + cap[4] = p[4]; + cap[5] = p[5]; + cap += 6; + p += 6; + } + } + } + } + } +} + + +static void cuvid_h264_parse_a53(CuvidContext *ctx, const uint8_t* p, int buf_size) +{ + const uint8_t* buf_end = p + buf_size; + while(p < buf_end) + { + int i, size, cc_count; + uint32_t start_code = -1; + uint64_t new_size; + p = avpriv_find_start_code(p, buf_end, &start_code); + if (start_code > 0x1ff) + break; + if (start_code != 0x106) + continue; + buf_size = buf_end - p; + if (buf_size < 1 || p[0] != 4) + continue; + p += 1; buf_size -= 1; + size = 0; + while (buf_size > 0) + { + size += p[0]; + buf_size -= 1; + if (*(p++) != 0xFF) + break; + } + if (buf_size <= 0 || buf_size < size) + continue; + if (size < 7) + continue; + if (p[0] == 0xFF) + { + p+=4; + size-=4; + } + else + { + p+=3; + size-=3; + } + if (p[0] != 'G' || p[1] != 'A' || p[2] != '9' || p[3] != '4') + continue; + p += 4; + size -= 4; + + if (size < 3) + continue; + if (p[0] != 3) + continue; + if (!(p[1] & 0x40)) + continue; + cc_count = p[1] & 0x1F; + p+=3; + size -= 3; + + if (!cc_count || size < cc_count * 3) + continue; + + if (!ctx->a53_caption) + ctx->a53_caption_size = 0; + new_size = (ctx->a53_caption_size + cc_count * 3); + if (av_reallocp(&ctx->a53_caption, new_size) < 0) + continue; + for(i = 0; i < cc_count; ++i, p += 3) + { + ctx->a53_caption[ctx->a53_caption_size++] = p[0]; + ctx->a53_caption[ctx->a53_caption_size++] = p[1]; + ctx->a53_caption[ctx->a53_caption_size++] = p[2]; + } + } +} + static int cuvid_decode_packet(AVCodecContext *avctx, const AVPacket *avpkt) { CuvidContext *ctx = avctx->priv_data; @@ -424,6 +605,15 @@ static int cuvid_decode_packet(AVCodecContext *avctx, const AVPacket *avpkt) ret = CHECK_CU(ctx->cvdl->cuvidParseVideoData(ctx->cuparser, &cupkt)); + // assume there is one frame delay (the parser outputs previous picture once it sees new frame data) + av_freep(&ctx->a53_caption); + if (avpkt && avpkt->size) { + if (ctx->cuparseinfo.CodecType == cudaVideoCodec_MPEG2) + cuvid_mpeg_parse_a53(ctx, avpkt->data, avpkt->size); + else if (ctx->cuparseinfo.CodecType == cudaVideoCodec_H264) + cuvid_h264_parse_a53(ctx, avpkt->data, avpkt->size); + } + if (ret < 0) goto error; @@ -627,6 +817,15 @@ FF_ENABLE_DEPRECATION_WARNINGS if (frame->interlaced_frame) frame->top_field_first = parsed_frame.dispinfo.top_field_first; + + if (parsed_frame.a53_caption) + { + AVFrameSideData *sd = av_frame_new_side_data(frame, AV_FRAME_DATA_A53_CC, parsed_frame.a53_caption_size); + if (sd) + memcpy(sd->data, parsed_frame.a53_caption, parsed_frame.a53_caption_size); + av_freep(&parsed_frame.a53_caption); + avctx->properties |= FF_CODEC_PROPERTY_CLOSED_CAPTIONS; + } } else if (ctx->decoder_flushing) { ret = AVERROR_EOF; } else {