From patchwork Fri Jul 26 12:32:42 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Gyan Doshi X-Patchwork-Id: 14081 Return-Path: X-Original-To: patchwork@ffaux-bg.ffmpeg.org Delivered-To: patchwork@ffaux-bg.ffmpeg.org Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org [79.124.17.100]) by ffaux.localdomain (Postfix) with ESMTP id 833BD447D76 for ; Fri, 26 Jul 2019 15:32:57 +0300 (EEST) Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 57E4368AC11; Fri, 26 Jul 2019 15:32:57 +0300 (EEST) X-Original-To: ffmpeg-devel@ffmpeg.org Delivered-To: ffmpeg-devel@ffmpeg.org Received: from mx2.mailbox.org (mx2.mailbox.org [80.241.60.215]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTPS id A272168ABA6 for ; Fri, 26 Jul 2019 15:32:50 +0300 (EEST) Received: from smtp2.mailbox.org (smtp2.mailbox.org [80.241.60.241]) (using TLSv1.2 with cipher ECDHE-RSA-CHACHA20-POLY1305 (256/256 bits)) (No client certificate requested) by mx2.mailbox.org (Postfix) with ESMTPS id 74E2FA1712 for ; Fri, 26 Jul 2019 14:32:49 +0200 (CEST) X-Virus-Scanned: amavisd-new at heinlein-support.de Received: from smtp2.mailbox.org ([80.241.60.241]) by spamfilter01.heinlein-hosting.de (spamfilter01.heinlein-hosting.de [80.241.56.115]) (amavisd-new, port 10030) with ESMTP id 3K7BIugSOSfu for ; Fri, 26 Jul 2019 14:32:47 +0200 (CEST) To: FFmpeg development discussions and patches From: Gyan Message-ID: <16eb1661-7d7e-f28b-ea3c-38a7b7dd8d04@gyani.pro> Date: Fri, 26 Jul 2019 18:02:42 +0530 MIME-Version: 1.0 Content-Language: en-US Subject: [FFmpeg-devel] [PATCH] avdevice/decklink: adjust for timecode lag X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.20 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" Patch supported by and tested at Google. Gyan From e51bd8201ddf618dce33ada70a9bc6ce2f33b07b Mon Sep 17 00:00:00 2001 From: Gyan Doshi Date: Mon, 1 Jul 2019 23:43:44 +0530 Subject: [PATCH] avdevice/decklink: adjust for timecode lag The decklink demuxer may not start receiving timecode with the first frame, but only some frames later. The demuxer, at present, naively stores the first received timecode. This patch monitors the lag in video frames, and adjusts the timecode before saving it in stream metadata. --- libavdevice/decklink_common.h | 4 +++ libavdevice/decklink_dec.cpp | 53 +++++++++++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/libavdevice/decklink_common.h b/libavdevice/decklink_common.h index 921818ba41..f149e7ff66 100644 --- a/libavdevice/decklink_common.h +++ b/libavdevice/decklink_common.h @@ -149,6 +149,10 @@ struct decklink_ctx { int channels; int audio_depth; + + /* Fields for timecode correction */ + unsigned long vidframeCount; // frameCount tracks audio-only packets as well + unsigned long firstframeIndex; // value of frameCount for first successfully demuxed video frame }; typedef enum { DIRECTION_IN, DIRECTION_OUT} decklink_direction_t; diff --git a/libavdevice/decklink_dec.cpp b/libavdevice/decklink_dec.cpp index 4da9122bff..d0ebd0ac0a 100644 --- a/libavdevice/decklink_dec.cpp +++ b/libavdevice/decklink_dec.cpp @@ -41,6 +41,7 @@ extern "C" { #include "libavutil/imgutils.h" #include "libavutil/intreadwrite.h" #include "libavutil/time.h" +#include "libavutil/timecode.h" #include "libavutil/mathematics.h" #include "libavutil/reverse.h" #include "avdevice.h" @@ -736,6 +737,8 @@ HRESULT decklink_input_callback::VideoInputFrameArrived( videoFrame->GetStreamTime(&frameTime, &frameDuration, ctx->video_st->time_base.den); + ctx->vidframeCount++; + if (videoFrame->GetFlags() & bmdFrameHasNoInputSource) { if (ctx->draw_bars && videoFrame->GetPixelFormat() == bmdFormat8BitYUV) { unsigned bars[8] = { @@ -765,6 +768,8 @@ HRESULT decklink_input_callback::VideoInputFrameArrived( // Handle Timecode (if requested) if (ctx->tc_format) { + if (!ctx->firstframeIndex) + ctx->firstframeIndex = ctx->vidframeCount; IDeckLinkTimecode *timecode; if (videoFrame->GetTimecode(ctx->tc_format, &timecode) == S_OK) { const char *tc = NULL; @@ -778,7 +783,9 @@ HRESULT decklink_input_callback::VideoInputFrameArrived( AVDictionary* metadata_dict = NULL; int metadata_len; uint8_t* packed_metadata; + unsigned long offset = ctx->firstframeIndex - ctx->vidframeCount; // represents offset of first returned frame relative to first frame with timecode if (av_dict_set(&metadata_dict, "timecode", tc, AV_DICT_DONT_STRDUP_VAL) >= 0) { + av_dict_set_int(&metadata_dict, "tc_offset", offset, 0); packed_metadata = av_packet_pack_dictionary(metadata_dict, &metadata_len); av_dict_free(&metadata_dict); if (packed_metadata) { @@ -880,6 +887,8 @@ HRESULT decklink_input_callback::VideoInputFrameArrived( videoFrame->AddRef(); if (avpacket_queue_put(&ctx->queue, &pkt) < 0) { + if (ctx->firstframeIndex == ctx->vidframeCount) + ctx->firstframeIndex = 0; ++ctx->dropped; } } @@ -1266,10 +1275,48 @@ int ff_decklink_read_packet(AVFormatContext *avctx, AVPacket *pkt) if (ctx->tc_format && !(av_dict_get(ctx->video_st->metadata, "timecode", NULL, 0))) { int size; const uint8_t *side_metadata = av_packet_get_side_data(pkt, AV_PKT_DATA_STRINGS_METADATA, &size); + if (side_metadata) { - if (av_packet_unpack_dictionary(side_metadata, size, &ctx->video_st->metadata) < 0) - av_log(avctx, AV_LOG_ERROR, "Unable to set timecode\n"); - } + char offset_tc[AV_TIMECODE_STR_SIZE]; + int ret; + AVTimecode tc; + AVRational vid_rate = ctx->video_st->r_frame_rate; + int64_t offset = 0; + AVDictionary *first_tc = NULL; + AVDictionaryEntry *tcstr = NULL; + AVDictionaryEntry *offsetstr = NULL; + + if (av_packet_unpack_dictionary(side_metadata, size, &first_tc) < 0 || + !(tcstr = av_dict_get(first_tc, "timecode", NULL, 0)) ) { + av_log(avctx, AV_LOG_ERROR, "Unable to find timecode\n"); + return 0; + } + + if (tcstr) + av_log(avctx, AV_LOG_DEBUG, "Serial frame timecode is %s.\n", tcstr->value); + + if (av_timecode_init_from_string(&tc, vid_rate, tcstr->value, avctx) < 0) { + av_log(avctx, AV_LOG_ERROR, "Unable to set timecode\n"); + return 0; + } + + offsetstr = av_dict_get(first_tc, "tc_offset", NULL, 0); + if (offsetstr) { + offset = strtol(offsetstr->value, NULL, 10); + av_log(avctx, AV_LOG_INFO, "Timecode offset is %" PRId64 ".\n", offset); + } else + av_log(avctx, AV_LOG_WARNING, "Unable to get correction offset for timecode.\n"); + + ret = av_dict_set(&ctx->video_st->metadata, "timecode", av_timecode_make_string(&tc, offset_tc, offset), 0); + if (ret >= 0) + av_log(avctx, AV_LOG_INFO, "Timecode %s after offset of %" PRId64 " is stored.\n", offset_tc, offset); + else + av_log(avctx, AV_LOG_ERROR, "Unable to store timecode.\n"); + + return 0; + + } else + av_log(avctx, AV_LOG_TRACE, "No timecode side data found as of pkt pts %" PRId64 "\n", pkt->pts); } return 0;