| Message ID | 20171116160958.92931-2-dheitmueller@ltnglobal.com |
|---|---|
| State | Superseded |
| Headers | show |
On Thu, 16 Nov 2017, Devin Heitmueller wrote: > Hook in libklvanc and use it for output of EIA-708 captions over > SDI. The bulk of this patch is just general support for ancillary > data for the Decklink SDI module - the real work for construction > of the EIA-708 CDP and VANC line construction is done by libklvanc. > > Libklvanc can be found at: https://github.com/stoth68000/libklvanc > > Updated to reflect feedback from Marton Balint <cus@passwd.hu> Sorry for the delayed response, comments are below: > > Signed-off-by: Devin Heitmueller <dheitmueller@ltnglobal.com> > --- > configure | 4 ++ > libavcodec/v210enc.c | 12 ++++ > libavdevice/decklink_common.cpp | 17 +++-- > libavdevice/decklink_common.h | 10 +++ > libavdevice/decklink_enc.cpp | 150 ++++++++++++++++++++++++++++++++++++++-- > 5 files changed, 183 insertions(+), 10 deletions(-) > > diff --git a/configure b/configure > index 934ac3a..d5e3dcc 100755 > --- a/configure > +++ b/configure > @@ -238,6 +238,7 @@ External library support: > --enable-libiec61883 enable iec61883 via libiec61883 [no] > --enable-libilbc enable iLBC de/encoding via libilbc [no] > --enable-libjack enable JACK audio sound server [no] > + --enable-libklvanc enable Kernel Labs VANC processing [no] > --enable-libkvazaar enable HEVC encoding via libkvazaar [no] > --enable-libmodplug enable ModPlug via libmodplug [no] > --enable-libmp3lame enable MP3 encoding via libmp3lame [no] > @@ -1602,6 +1603,7 @@ EXTERNAL_LIBRARY_LIST=" > libiec61883 > libilbc > libjack > + libklvanc > libkvazaar > libmodplug > libmp3lame > @@ -3076,6 +3078,7 @@ decklink_deps_any="libdl LoadLibrary" > decklink_indev_deps="decklink threads" > decklink_indev_extralibs="-lstdc++" > decklink_outdev_deps="decklink threads" > +decklink_outdev_suggest="libklvanc" > decklink_outdev_extralibs="-lstdc++" > libndi_newtek_indev_deps="libndi_newtek" > libndi_newtek_indev_extralibs="-lndi" > @@ -5847,6 +5850,7 @@ enabled libgsm && { for gsm_hdr in "gsm.h" "gsm/gsm.h"; do > check_lib libgsm "${gsm_hdr}" gsm_create -lgsm && break; > done || die "ERROR: libgsm not found"; } > enabled libilbc && require libilbc ilbc.h WebRtcIlbcfix_InitDecode -lilbc $pthreads_extralibs > +enabled libklvanc && require libklvanc libklvanc/vanc.h klvanc_context_create -lklvanc > enabled libkvazaar && require_pkg_config libkvazaar "kvazaar >= 0.8.1" kvazaar.h kvz_api_get > # While it may appear that require is being used as a pkg-config > # fallback for libmfx, it is actually being used to detect a different > diff --git a/libavcodec/v210enc.c b/libavcodec/v210enc.c > index a6afbbf..a825c03 100644 > --- a/libavcodec/v210enc.c > +++ b/libavcodec/v210enc.c > @@ -123,6 +123,7 @@ static int encode_frame(AVCodecContext *avctx, AVPacket *pkt, > int aligned_width = ((avctx->width + 47) / 48) * 48; > int stride = aligned_width * 8 / 3; > int line_padding = stride - ((avctx->width * 8 + 11) / 12) * 4; > + AVFrameSideData *side_data; > int h, w, ret; > uint8_t *dst; > > @@ -233,6 +234,17 @@ static int encode_frame(AVCodecContext *avctx, AVPacket *pkt, > } > } > > + side_data = av_frame_get_side_data(pic, AV_FRAME_DATA_A53_CC); > + if (side_data && side_data->size) { > + uint8_t* buf = av_packet_new_side_data(pkt, AV_PKT_DATA_A53_CC, side_data->size); > + if (buf) { > + memcpy(buf, side_data->data, side_data->size); > + } else { > + av_log(avctx, AV_LOG_ERROR, "Unable to allocate side data\n"); For ENOMEM, I think you can skip the error message. > + return AVERROR(ENOMEM); > + } > + } > + > pkt->flags |= AV_PKT_FLAG_KEY; > *got_packet = 1; > return 0; > diff --git a/libavdevice/decklink_common.cpp b/libavdevice/decklink_common.cpp > index 2bd63ac..c425f4a 100644 > --- a/libavdevice/decklink_common.cpp > +++ b/libavdevice/decklink_common.cpp > @@ -247,10 +247,19 @@ int ff_decklink_set_format(AVFormatContext *avctx, > &support, NULL) != S_OK) > return -1; > } else { > - if (ctx->dlo->DoesSupportVideoMode(ctx->bmd_mode, bmdFormat8BitYUV, > - bmdVideoOutputFlagDefault, > - &support, NULL) != S_OK) > - return -1; > + ctx->supports_vanc = 1; > + if (ctx->dlo->DoesSupportVideoMode(ctx->bmd_mode, (BMDPixelFormat) ctx->raw_format, > + bmdVideoOutputVANC, > + &support, NULL) != S_OK) { > + /* Try again, but without VANC enabled */ > + if (ctx->dlo->DoesSupportVideoMode(ctx->bmd_mode, (BMDPixelFormat) ctx->raw_format, > + bmdVideoOutputFlagDefault, > + &support, NULL) != S_OK) { > + return -1; > + } > + ctx->supports_vanc = 0; > + } > + > } > if (support == bmdDisplayModeSupported) > return 0; > diff --git a/libavdevice/decklink_common.h b/libavdevice/decklink_common.h > index b6acb01..e951b17 100644 > --- a/libavdevice/decklink_common.h > +++ b/libavdevice/decklink_common.h > @@ -27,6 +27,9 @@ > > #include "libavutil/thread.h" > #include "decklink_common_c.h" > +#if CONFIG_LIBKLVANC > +#include "libklvanc/vanc.h" > +#endif > > class decklink_output_callback; > class decklink_input_callback; > @@ -61,6 +64,7 @@ struct decklink_ctx { > int bmd_width; > int bmd_height; > int bmd_field_dominance; > + int supports_vanc; > > /* Capture buffer queue */ > AVPacketQueue queue; > @@ -78,6 +82,7 @@ struct decklink_ctx { > AVStream *audio_st; > AVStream *video_st; > AVStream *teletext_st; > + uint16_t cdp_sequence_num; > > /* Options */ > int list_devices; > @@ -88,6 +93,7 @@ struct decklink_ctx { > DecklinkPtsSource audio_pts_source; > DecklinkPtsSource video_pts_source; > int draw_bars; > + int raw_format; Since this header includes decklink headers, this can be BMDPixelFormat instead of int, and you can use the decklink constants directly instead of MKBETAG. > > int frames_preroll; > int frames_buffer; > @@ -96,6 +102,10 @@ struct decklink_ctx { > pthread_cond_t cond; > int frames_buffer_available_spots; > > +#if CONFIG_LIBKLVANC > + struct klvanc_context_s *vanc_ctx; > +#endif > + > int channels; > int audio_depth; > }; > diff --git a/libavdevice/decklink_enc.cpp b/libavdevice/decklink_enc.cpp > index 81df563..0dcbe79 100644 > --- a/libavdevice/decklink_enc.cpp > +++ b/libavdevice/decklink_enc.cpp > @@ -38,16 +38,20 @@ extern "C" { > > #include "decklink_common.h" > #include "decklink_enc.h" > - > +#if CONFIG_LIBKLVANC > +#include "libklvanc/vanc.h" > +#include "libklvanc/vanc-lines.h" > +#include "libklvanc/pixels.h" > +#endif > > /* DeckLink callback class declaration */ > class decklink_frame : public IDeckLinkVideoFrame > { > public: > decklink_frame(struct decklink_ctx *ctx, AVFrame *avframe, AVCodecID codec_id, int height, int width) : > - _ctx(ctx), _avframe(avframe), _avpacket(NULL), _codec_id(codec_id), _height(height), _width(width), _refs(1) { } > + _ctx(ctx), _avframe(avframe), _avpacket(NULL), _codec_id(codec_id), _ancillary(NULL), _height(height), _width(width), _refs(1) { } > decklink_frame(struct decklink_ctx *ctx, AVPacket *avpacket, AVCodecID codec_id, int height, int width) : > - _ctx(ctx), _avframe(NULL), _avpacket(avpacket), _codec_id(codec_id), _height(height), _width(width), _refs(1) { } > + _ctx(ctx), _avframe(NULL), _avpacket(avpacket), _codec_id(codec_id), _ancillary(NULL), _height(height), _width(width), _refs(1) { } > > virtual long STDMETHODCALLTYPE GetWidth (void) { return _width; } > virtual long STDMETHODCALLTYPE GetHeight (void) { return _height; } > @@ -87,8 +91,13 @@ public: > } > > virtual HRESULT STDMETHODCALLTYPE GetTimecode (BMDTimecodeFormat format, IDeckLinkTimecode **timecode) { return S_FALSE; } > - virtual HRESULT STDMETHODCALLTYPE GetAncillaryData(IDeckLinkVideoFrameAncillary **ancillary) { return S_FALSE; } > - > + virtual HRESULT STDMETHODCALLTYPE GetAncillaryData(IDeckLinkVideoFrameAncillary **ancillary) > + { > + *ancillary = _ancillary; > + return _ancillary ? S_OK : S_FALSE; > + } > + virtual HRESULT STDMETHODCALLTYPE SetAncillaryData(IDeckLinkVideoFrameAncillary > + *ancillary) { _ancillary = ancillary; return S_OK; } > virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID iid, LPVOID *ppv) { return E_NOINTERFACE; } > virtual ULONG STDMETHODCALLTYPE AddRef(void) { return ++_refs; } > virtual ULONG STDMETHODCALLTYPE Release(void) > @@ -106,6 +115,7 @@ public: > AVFrame *_avframe; > AVPacket *_avpacket; > AVCodecID _codec_id; > + IDeckLinkVideoFrameAncillary *_ancillary; > int _height; > int _width; > > @@ -156,10 +166,13 @@ static int decklink_setup_video(AVFormatContext *avctx, AVStream *st) > " Only AV_PIX_FMT_UYVY422 is supported.\n"); > return -1; > } > + ctx->raw_format = MKBETAG('2','v','u','y'); > } else if (c->codec_id != AV_CODEC_ID_V210) { > av_log(avctx, AV_LOG_ERROR, "Unsupported codec type!" > " Only V210 and wrapped frame with AV_PIX_FMT_UYVY422 are supported.\n"); > return -1; > + } else { > + ctx->raw_format = MKBETAG('v','2','1','0'); > } > > if (ff_decklink_set_format(avctx, c->width, c->height, > @@ -169,7 +182,7 @@ static int decklink_setup_video(AVFormatContext *avctx, AVStream *st) > return -1; > } > if (ctx->dlo->EnableVideoOutput(ctx->bmd_mode, > - bmdVideoOutputFlagDefault) != S_OK) { > + ctx->supports_vanc ? bmdVideoOutputVANC : bmdVideoOutputFlagDefault) != S_OK) { > av_log(avctx, AV_LOG_ERROR, "Could not enable video output!\n"); > return -1; > } > @@ -265,6 +278,118 @@ av_cold int ff_decklink_write_trailer(AVFormatContext *avctx) > return 0; > } > > +#if CONFIG_LIBKLVANC > +static int decklink_construct_vanc(AVFormatContext *avctx, struct decklink_ctx *ctx, > + AVPacket *pkt, decklink_frame *frame) > +{ > + struct klvanc_line_set_s vanc_lines = { 0 }; > + int ret, size; > + > + if (ctx->supports_vanc == 0) > + return 0; > + > + const uint8_t *data = av_packet_get_side_data(pkt, AV_PKT_DATA_A53_CC, &size); > + if (data) { > + struct klvanc_packet_eia_708b_s *pkt; > + uint16_t *cdp; > + uint16_t len; > + uint8_t cc_count = size / 3; > + > + ret = klvanc_create_eia708_cdp(&pkt); > + if (ret != 0) > + return AVERROR(ENOMEM); > + > + klvanc_set_framerate_EIA_708B(pkt, ctx->bmd_tb_num, ctx->bmd_tb_den); You can squash your later patch to this one to fix the missing ret = ... > + if (ret != 0) { > + av_log(avctx, AV_LOG_ERROR, "Invalid framerate specified: %lld/%lld\n", > + ctx->bmd_tb_num, ctx->bmd_tb_den); > + klvanc_destroy_eia708_cdp(pkt); > + return AVERROR(EINVAL); > + } > + > + if (cc_count > KLVANC_MAX_CC_COUNT) { > + av_log(avctx, AV_LOG_ERROR, "Illegal cc_count received: %d\n", cc_count); > + cc_count = KLVANC_MAX_CC_COUNT; > + } > + > + /* CC data */ > + pkt->header.ccdata_present = 1; > + pkt->ccdata.cc_count = cc_count; > + for (size_t i = 0; i < cc_count; i++) { > + if (data [3*i] & 0x40) > + pkt->ccdata.cc[i].cc_valid = 1; > + pkt->ccdata.cc[i].cc_type = data[3*i] & 0x03; > + pkt->ccdata.cc[i].cc_data[0] = data[3*i+1]; > + pkt->ccdata.cc[i].cc_data[1] = data[3*i+2]; > + } > + > + klvanc_finalize_EIA_708B(pkt, ctx->cdp_sequence_num++); > + klvanc_convert_EIA_708B_to_words(pkt, &cdp, &len); Missing error check. > + klvanc_destroy_eia708_cdp(pkt); > + > + ret = klvanc_line_insert(ctx->vanc_ctx, &vanc_lines, cdp, len, 11, 0); > + if (ret != 0) { > + av_log(avctx, AV_LOG_ERROR, "VANC line insertion failed\n"); > + return AVERROR(ENOMEM); > + } > + } > + > + IDeckLinkVideoFrameAncillary *vanc; > + int result = ctx->dlo->CreateAncillaryData(bmdFormat10BitYUV, &vanc); > + if (result != S_OK) { > + av_log(avctx, AV_LOG_ERROR, "Failed to create vanc\n"); For older decklink models (E.g. Decklink SDI, Decklink Duo 1), when you capture in 8 bit mode, you can only query 8bit VANC. For output, can you always use 10-bit VANC? Even if you use 8bit mode for video? Because if you can't, then it might make sense to return silently here, or only warn to user once, not for every frame (and maybe disable vanc_support?). > + return -1; > + } > + > + /* Now that we've got all the VANC lines in a nice orderly manner, generate the > + final VANC sections for the Decklink output */ > + for (int i = 0; i < vanc_lines.num_lines; i++) { > + struct klvanc_line_s *line = vanc_lines.lines[i]; > + uint16_t *out_line; > + int real_line; > + int out_len; > + void *buf; > + > + if (line == NULL) > + break; > + > + real_line = line->line_number; > +#if 0 > + /* FIXME: include hack for certain Decklink cards which mis-represent > + line numbers for pSF frames */ > + if (decklink_sys->b_psf_interlaced) > + real_line = Calculate1080psfVancLine(line->line_number); > +#endif > + result = vanc->GetBufferForVerticalBlankingLine(real_line, &buf); > + if (result != S_OK) { > + av_log(avctx, AV_LOG_ERROR, "Failed to get VANC line %d: %d", real_line, result); > + klvanc_line_free(line); > + continue; > + } > + > + /* Generate the full line taking into account all VANC packets on that line */ > + klvanc_generate_vanc_line(ctx->vanc_ctx, line, &out_line, &out_len, ctx->bmd_width); yet another missing result= I guess. > + if (result != 0) { > + av_log(avctx, AV_LOG_ERROR, "Failed to generate VANC line\n"); > + klvanc_line_free(line); > + continue; > + } > + > + /* Repack the 16-bit ints into 10-bit, and push into final buffer */ > + klvanc_y10_to_v210(out_line, (uint8_t *) buf, out_len); > + free(out_line); > + klvanc_line_free(line); > + } > + > + result = frame->SetAncillaryData(vanc); > + if (result != S_OK) { > + av_log(avctx, AV_LOG_ERROR, "Failed to set vanc: %d", result); > + return AVERROR(EIO); > + } > + return 0; > +} > +#endif > + > static int decklink_write_video_packet(AVFormatContext *avctx, AVPacket *pkt) > { > struct decklink_cctx *cctx = (struct decklink_cctx *)avctx->priv_data; > @@ -275,6 +400,9 @@ static int decklink_write_video_packet(AVFormatContext *avctx, AVPacket *pkt) > decklink_frame *frame; > buffercount_type buffered; > HRESULT hr; > +#if CONFIG_LIBKLVANC > + int ret; > +#endif > > if (st->codecpar->codec_id == AV_CODEC_ID_WRAPPED_AVFRAME) { > if (tmp->format != AV_PIX_FMT_UYVY422 || > @@ -299,6 +427,13 @@ static int decklink_write_video_packet(AVFormatContext *avctx, AVPacket *pkt) > } > > frame = new decklink_frame(ctx, avpacket, st->codecpar->codec_id, ctx->bmd_height, ctx->bmd_width); > + > +#if CONFIG_LIBKLVANC > + ret = decklink_construct_vanc(avctx, ctx, pkt, frame); > + if (ret != 0) { > + av_log(avctx, AV_LOG_ERROR, "Failed to construct VANC\n"); > + } > +#endif > } > > if (!frame) { > @@ -389,6 +524,9 @@ av_cold int ff_decklink_write_header(AVFormatContext *avctx) > ctx->list_formats = cctx->list_formats; > ctx->preroll = cctx->preroll; > cctx->ctx = ctx; > +#if CONFIG_LIBKLVANC > + klvanc_context_create(&ctx->vanc_ctx); > +#endif > > /* List available devices and exit. */ > if (ctx->list_devices) { > -- > 1.8.3.1 > Regards, Marton
Hello Marton, Thanks for taking the time to review. Most of the comments you’ve raised will be fixed and I’ll resubmit an updated patch. Comments on other issues inline below. >> >> /* Options */ >> int list_devices; >> @@ -88,6 +93,7 @@ struct decklink_ctx { >> DecklinkPtsSource audio_pts_source; >> DecklinkPtsSource video_pts_source; >> int draw_bars; >> + int raw_format; > > Since this header includes decklink headers, this can be BMDPixelFormat instead of int, and you can use the decklink constants directly instead of MKBETAG. I used MKBETAG because that was what was being used in decklink_dec.cpp (and I wanted to be consistent). That said, I have no objection to changing it. > > For older decklink models (E.g. Decklink SDI, Decklink Duo 1), when you capture in 8 bit mode, you can only query 8bit VANC. For output, can you always use 10-bit VANC? Even if you use 8bit mode for video? Because if you can't, then it might make sense to return silently here, or only warn to user once, not for every frame (and maybe disable vanc_support?). All decklink models require that VANC be in the same bit depth as video capture (i.e. with both older and newer models you cannot do 8-bit video with 10-bit VANC or vice-versa). The only exception is the RGB formats which do VANC in 10-bit YUV. The decklink_construct_vanc() function is only ever called if the device is putting out 10-bit video, and thus your question about putting out 10-bit VANC when doing 8-bit video isn’t an issue since we never hit that code path. In summary, 8-bit VANC isn’t supported in the module, and I don’t have any immediate plans to do such given how rare it is nowadays. If somebody really cares about that use case, we can discuss further. Devin
On Tue, 28 Nov 2017, Devin Heitmueller wrote: > Hello Marton, > > Thanks for taking the time to review. Most of the comments you’ve raised will be fixed and I’ll resubmit an updated patch. Comments on other issues inline below. >>> >>> /* Options */ >>> int list_devices; >>> @@ -88,6 +93,7 @@ struct decklink_ctx { >>> DecklinkPtsSource audio_pts_source; >>> DecklinkPtsSource video_pts_source; >>> int draw_bars; >>> + int raw_format; >> >> Since this header includes decklink headers, this can be BMDPixelFormat instead of int, and you can use the decklink constants directly instead of MKBETAG. > > I used MKBETAG because that was what was being used in decklink_dec.cpp (and I wanted to be consistent). That said, I have no objection to changing it. Ok, maybe better to change it. > >> >> For older decklink models (E.g. Decklink SDI, Decklink Duo 1), when you >> capture in 8 bit mode, you can only query 8bit VANC. For output, can >> you always use 10-bit VANC? Even if you use 8bit mode for video? >> Because if you can't, then it might make sense to return silently here, >> or only warn to user once, not for every frame (and maybe disable >> vanc_support?). > > All decklink models require that VANC be in the same bit depth as video > capture (i.e. with both older and newer models you cannot do 8-bit video > with 10-bit VANC or vice-versa). The only exception is the RGB formats > which do VANC in 10-bit YUV. SDK says: When capturing ancillary data with a 4K DeckLink device, the ancillary data will always be in the 10-bit YUV pixel format. This also applies to 8 bit YUV captures according to my experience. > The decklink_construct_vanc() function is > only ever called if the device is putting out 10-bit video, and thus > your question about putting out 10-bit VANC when doing 8-bit video isn’t > an issue since we never hit that code path. Ah, OK. I missed that. You can keep the code as is then, somebody else interested can figure out if outputting 10 bit VANC works with 8 bit video or not. Regards, Marton
Hello Marton, > SDK says: > > When capturing ancillary data with a 4K DeckLink device, the ancillary > data will always be in the 10-bit YUV pixel format. > > This also applies to 8 bit YUV captures according to my experience. > Thanks for refreshing my memory. I remember reading that text, but for some reason thought it was RGB formats, not 4K capture products. That’s actually a really nice feature - if you’re not feeding a 10-bit encoder then it would avoid having to do 10-to-8bit colorspace conversion in software in order for VANC to be properly preserved. In earlier products you either had to choose between 8-bit video but VANC wouldn’t be properly preserved, or you could do 10-bit video but then have to colorspace convert from V210 to an 8-bit format. In the future I’ll have to look and see if that’s exposed through a decklink attribute (or do we have to hard-code the model info into the application to know which cards work this way). Regards, Devin
diff --git a/configure b/configure index 934ac3a..d5e3dcc 100755 --- a/configure +++ b/configure @@ -238,6 +238,7 @@ External library support: --enable-libiec61883 enable iec61883 via libiec61883 [no] --enable-libilbc enable iLBC de/encoding via libilbc [no] --enable-libjack enable JACK audio sound server [no] + --enable-libklvanc enable Kernel Labs VANC processing [no] --enable-libkvazaar enable HEVC encoding via libkvazaar [no] --enable-libmodplug enable ModPlug via libmodplug [no] --enable-libmp3lame enable MP3 encoding via libmp3lame [no] @@ -1602,6 +1603,7 @@ EXTERNAL_LIBRARY_LIST=" libiec61883 libilbc libjack + libklvanc libkvazaar libmodplug libmp3lame @@ -3076,6 +3078,7 @@ decklink_deps_any="libdl LoadLibrary" decklink_indev_deps="decklink threads" decklink_indev_extralibs="-lstdc++" decklink_outdev_deps="decklink threads" +decklink_outdev_suggest="libklvanc" decklink_outdev_extralibs="-lstdc++" libndi_newtek_indev_deps="libndi_newtek" libndi_newtek_indev_extralibs="-lndi" @@ -5847,6 +5850,7 @@ enabled libgsm && { for gsm_hdr in "gsm.h" "gsm/gsm.h"; do check_lib libgsm "${gsm_hdr}" gsm_create -lgsm && break; done || die "ERROR: libgsm not found"; } enabled libilbc && require libilbc ilbc.h WebRtcIlbcfix_InitDecode -lilbc $pthreads_extralibs +enabled libklvanc && require libklvanc libklvanc/vanc.h klvanc_context_create -lklvanc enabled libkvazaar && require_pkg_config libkvazaar "kvazaar >= 0.8.1" kvazaar.h kvz_api_get # While it may appear that require is being used as a pkg-config # fallback for libmfx, it is actually being used to detect a different diff --git a/libavcodec/v210enc.c b/libavcodec/v210enc.c index a6afbbf..a825c03 100644 --- a/libavcodec/v210enc.c +++ b/libavcodec/v210enc.c @@ -123,6 +123,7 @@ static int encode_frame(AVCodecContext *avctx, AVPacket *pkt, int aligned_width = ((avctx->width + 47) / 48) * 48; int stride = aligned_width * 8 / 3; int line_padding = stride - ((avctx->width * 8 + 11) / 12) * 4; + AVFrameSideData *side_data; int h, w, ret; uint8_t *dst; @@ -233,6 +234,17 @@ static int encode_frame(AVCodecContext *avctx, AVPacket *pkt, } } + side_data = av_frame_get_side_data(pic, AV_FRAME_DATA_A53_CC); + if (side_data && side_data->size) { + uint8_t* buf = av_packet_new_side_data(pkt, AV_PKT_DATA_A53_CC, side_data->size); + if (buf) { + memcpy(buf, side_data->data, side_data->size); + } else { + av_log(avctx, AV_LOG_ERROR, "Unable to allocate side data\n"); + return AVERROR(ENOMEM); + } + } + pkt->flags |= AV_PKT_FLAG_KEY; *got_packet = 1; return 0; diff --git a/libavdevice/decklink_common.cpp b/libavdevice/decklink_common.cpp index 2bd63ac..c425f4a 100644 --- a/libavdevice/decklink_common.cpp +++ b/libavdevice/decklink_common.cpp @@ -247,10 +247,19 @@ int ff_decklink_set_format(AVFormatContext *avctx, &support, NULL) != S_OK) return -1; } else { - if (ctx->dlo->DoesSupportVideoMode(ctx->bmd_mode, bmdFormat8BitYUV, - bmdVideoOutputFlagDefault, - &support, NULL) != S_OK) - return -1; + ctx->supports_vanc = 1; + if (ctx->dlo->DoesSupportVideoMode(ctx->bmd_mode, (BMDPixelFormat) ctx->raw_format, + bmdVideoOutputVANC, + &support, NULL) != S_OK) { + /* Try again, but without VANC enabled */ + if (ctx->dlo->DoesSupportVideoMode(ctx->bmd_mode, (BMDPixelFormat) ctx->raw_format, + bmdVideoOutputFlagDefault, + &support, NULL) != S_OK) { + return -1; + } + ctx->supports_vanc = 0; + } + } if (support == bmdDisplayModeSupported) return 0; diff --git a/libavdevice/decklink_common.h b/libavdevice/decklink_common.h index b6acb01..e951b17 100644 --- a/libavdevice/decklink_common.h +++ b/libavdevice/decklink_common.h @@ -27,6 +27,9 @@ #include "libavutil/thread.h" #include "decklink_common_c.h" +#if CONFIG_LIBKLVANC +#include "libklvanc/vanc.h" +#endif class decklink_output_callback; class decklink_input_callback; @@ -61,6 +64,7 @@ struct decklink_ctx { int bmd_width; int bmd_height; int bmd_field_dominance; + int supports_vanc; /* Capture buffer queue */ AVPacketQueue queue; @@ -78,6 +82,7 @@ struct decklink_ctx { AVStream *audio_st; AVStream *video_st; AVStream *teletext_st; + uint16_t cdp_sequence_num; /* Options */ int list_devices; @@ -88,6 +93,7 @@ struct decklink_ctx { DecklinkPtsSource audio_pts_source; DecklinkPtsSource video_pts_source; int draw_bars; + int raw_format; int frames_preroll; int frames_buffer; @@ -96,6 +102,10 @@ struct decklink_ctx { pthread_cond_t cond; int frames_buffer_available_spots; +#if CONFIG_LIBKLVANC + struct klvanc_context_s *vanc_ctx; +#endif + int channels; int audio_depth; }; diff --git a/libavdevice/decklink_enc.cpp b/libavdevice/decklink_enc.cpp index 81df563..0dcbe79 100644 --- a/libavdevice/decklink_enc.cpp +++ b/libavdevice/decklink_enc.cpp @@ -38,16 +38,20 @@ extern "C" { #include "decklink_common.h" #include "decklink_enc.h" - +#if CONFIG_LIBKLVANC +#include "libklvanc/vanc.h" +#include "libklvanc/vanc-lines.h" +#include "libklvanc/pixels.h" +#endif /* DeckLink callback class declaration */ class decklink_frame : public IDeckLinkVideoFrame { public: decklink_frame(struct decklink_ctx *ctx, AVFrame *avframe, AVCodecID codec_id, int height, int width) : - _ctx(ctx), _avframe(avframe), _avpacket(NULL), _codec_id(codec_id), _height(height), _width(width), _refs(1) { } + _ctx(ctx), _avframe(avframe), _avpacket(NULL), _codec_id(codec_id), _ancillary(NULL), _height(height), _width(width), _refs(1) { } decklink_frame(struct decklink_ctx *ctx, AVPacket *avpacket, AVCodecID codec_id, int height, int width) : - _ctx(ctx), _avframe(NULL), _avpacket(avpacket), _codec_id(codec_id), _height(height), _width(width), _refs(1) { } + _ctx(ctx), _avframe(NULL), _avpacket(avpacket), _codec_id(codec_id), _ancillary(NULL), _height(height), _width(width), _refs(1) { } virtual long STDMETHODCALLTYPE GetWidth (void) { return _width; } virtual long STDMETHODCALLTYPE GetHeight (void) { return _height; } @@ -87,8 +91,13 @@ public: } virtual HRESULT STDMETHODCALLTYPE GetTimecode (BMDTimecodeFormat format, IDeckLinkTimecode **timecode) { return S_FALSE; } - virtual HRESULT STDMETHODCALLTYPE GetAncillaryData(IDeckLinkVideoFrameAncillary **ancillary) { return S_FALSE; } - + virtual HRESULT STDMETHODCALLTYPE GetAncillaryData(IDeckLinkVideoFrameAncillary **ancillary) + { + *ancillary = _ancillary; + return _ancillary ? S_OK : S_FALSE; + } + virtual HRESULT STDMETHODCALLTYPE SetAncillaryData(IDeckLinkVideoFrameAncillary + *ancillary) { _ancillary = ancillary; return S_OK; } virtual HRESULT STDMETHODCALLTYPE QueryInterface(REFIID iid, LPVOID *ppv) { return E_NOINTERFACE; } virtual ULONG STDMETHODCALLTYPE AddRef(void) { return ++_refs; } virtual ULONG STDMETHODCALLTYPE Release(void) @@ -106,6 +115,7 @@ public: AVFrame *_avframe; AVPacket *_avpacket; AVCodecID _codec_id; + IDeckLinkVideoFrameAncillary *_ancillary; int _height; int _width; @@ -156,10 +166,13 @@ static int decklink_setup_video(AVFormatContext *avctx, AVStream *st) " Only AV_PIX_FMT_UYVY422 is supported.\n"); return -1; } + ctx->raw_format = MKBETAG('2','v','u','y'); } else if (c->codec_id != AV_CODEC_ID_V210) { av_log(avctx, AV_LOG_ERROR, "Unsupported codec type!" " Only V210 and wrapped frame with AV_PIX_FMT_UYVY422 are supported.\n"); return -1; + } else { + ctx->raw_format = MKBETAG('v','2','1','0'); } if (ff_decklink_set_format(avctx, c->width, c->height, @@ -169,7 +182,7 @@ static int decklink_setup_video(AVFormatContext *avctx, AVStream *st) return -1; } if (ctx->dlo->EnableVideoOutput(ctx->bmd_mode, - bmdVideoOutputFlagDefault) != S_OK) { + ctx->supports_vanc ? bmdVideoOutputVANC : bmdVideoOutputFlagDefault) != S_OK) { av_log(avctx, AV_LOG_ERROR, "Could not enable video output!\n"); return -1; } @@ -265,6 +278,118 @@ av_cold int ff_decklink_write_trailer(AVFormatContext *avctx) return 0; } +#if CONFIG_LIBKLVANC +static int decklink_construct_vanc(AVFormatContext *avctx, struct decklink_ctx *ctx, + AVPacket *pkt, decklink_frame *frame) +{ + struct klvanc_line_set_s vanc_lines = { 0 }; + int ret, size; + + if (ctx->supports_vanc == 0) + return 0; + + const uint8_t *data = av_packet_get_side_data(pkt, AV_PKT_DATA_A53_CC, &size); + if (data) { + struct klvanc_packet_eia_708b_s *pkt; + uint16_t *cdp; + uint16_t len; + uint8_t cc_count = size / 3; + + ret = klvanc_create_eia708_cdp(&pkt); + if (ret != 0) + return AVERROR(ENOMEM); + + klvanc_set_framerate_EIA_708B(pkt, ctx->bmd_tb_num, ctx->bmd_tb_den); + if (ret != 0) { + av_log(avctx, AV_LOG_ERROR, "Invalid framerate specified: %lld/%lld\n", + ctx->bmd_tb_num, ctx->bmd_tb_den); + klvanc_destroy_eia708_cdp(pkt); + return AVERROR(EINVAL); + } + + if (cc_count > KLVANC_MAX_CC_COUNT) { + av_log(avctx, AV_LOG_ERROR, "Illegal cc_count received: %d\n", cc_count); + cc_count = KLVANC_MAX_CC_COUNT; + } + + /* CC data */ + pkt->header.ccdata_present = 1; + pkt->ccdata.cc_count = cc_count; + for (size_t i = 0; i < cc_count; i++) { + if (data [3*i] & 0x40) + pkt->ccdata.cc[i].cc_valid = 1; + pkt->ccdata.cc[i].cc_type = data[3*i] & 0x03; + pkt->ccdata.cc[i].cc_data[0] = data[3*i+1]; + pkt->ccdata.cc[i].cc_data[1] = data[3*i+2]; + } + + klvanc_finalize_EIA_708B(pkt, ctx->cdp_sequence_num++); + klvanc_convert_EIA_708B_to_words(pkt, &cdp, &len); + klvanc_destroy_eia708_cdp(pkt); + + ret = klvanc_line_insert(ctx->vanc_ctx, &vanc_lines, cdp, len, 11, 0); + if (ret != 0) { + av_log(avctx, AV_LOG_ERROR, "VANC line insertion failed\n"); + return AVERROR(ENOMEM); + } + } + + IDeckLinkVideoFrameAncillary *vanc; + int result = ctx->dlo->CreateAncillaryData(bmdFormat10BitYUV, &vanc); + if (result != S_OK) { + av_log(avctx, AV_LOG_ERROR, "Failed to create vanc\n"); + return -1; + } + + /* Now that we've got all the VANC lines in a nice orderly manner, generate the + final VANC sections for the Decklink output */ + for (int i = 0; i < vanc_lines.num_lines; i++) { + struct klvanc_line_s *line = vanc_lines.lines[i]; + uint16_t *out_line; + int real_line; + int out_len; + void *buf; + + if (line == NULL) + break; + + real_line = line->line_number; +#if 0 + /* FIXME: include hack for certain Decklink cards which mis-represent + line numbers for pSF frames */ + if (decklink_sys->b_psf_interlaced) + real_line = Calculate1080psfVancLine(line->line_number); +#endif + result = vanc->GetBufferForVerticalBlankingLine(real_line, &buf); + if (result != S_OK) { + av_log(avctx, AV_LOG_ERROR, "Failed to get VANC line %d: %d", real_line, result); + klvanc_line_free(line); + continue; + } + + /* Generate the full line taking into account all VANC packets on that line */ + klvanc_generate_vanc_line(ctx->vanc_ctx, line, &out_line, &out_len, ctx->bmd_width); + if (result != 0) { + av_log(avctx, AV_LOG_ERROR, "Failed to generate VANC line\n"); + klvanc_line_free(line); + continue; + } + + /* Repack the 16-bit ints into 10-bit, and push into final buffer */ + klvanc_y10_to_v210(out_line, (uint8_t *) buf, out_len); + free(out_line); + klvanc_line_free(line); + } + + result = frame->SetAncillaryData(vanc); + if (result != S_OK) { + av_log(avctx, AV_LOG_ERROR, "Failed to set vanc: %d", result); + return AVERROR(EIO); + } + return 0; +} +#endif + static int decklink_write_video_packet(AVFormatContext *avctx, AVPacket *pkt) { struct decklink_cctx *cctx = (struct decklink_cctx *)avctx->priv_data; @@ -275,6 +400,9 @@ static int decklink_write_video_packet(AVFormatContext *avctx, AVPacket *pkt) decklink_frame *frame; buffercount_type buffered; HRESULT hr; +#if CONFIG_LIBKLVANC + int ret; +#endif if (st->codecpar->codec_id == AV_CODEC_ID_WRAPPED_AVFRAME) { if (tmp->format != AV_PIX_FMT_UYVY422 || @@ -299,6 +427,13 @@ static int decklink_write_video_packet(AVFormatContext *avctx, AVPacket *pkt) } frame = new decklink_frame(ctx, avpacket, st->codecpar->codec_id, ctx->bmd_height, ctx->bmd_width); + +#if CONFIG_LIBKLVANC + ret = decklink_construct_vanc(avctx, ctx, pkt, frame); + if (ret != 0) { + av_log(avctx, AV_LOG_ERROR, "Failed to construct VANC\n"); + } +#endif } if (!frame) { @@ -389,6 +524,9 @@ av_cold int ff_decklink_write_header(AVFormatContext *avctx) ctx->list_formats = cctx->list_formats; ctx->preroll = cctx->preroll; cctx->ctx = ctx; +#if CONFIG_LIBKLVANC + klvanc_context_create(&ctx->vanc_ctx); +#endif /* List available devices and exit. */ if (ctx->list_devices) {
Hook in libklvanc and use it for output of EIA-708 captions over SDI. The bulk of this patch is just general support for ancillary data for the Decklink SDI module - the real work for construction of the EIA-708 CDP and VANC line construction is done by libklvanc. Libklvanc can be found at: https://github.com/stoth68000/libklvanc Updated to reflect feedback from Marton Balint <cus@passwd.hu> Signed-off-by: Devin Heitmueller <dheitmueller@ltnglobal.com> --- configure | 4 ++ libavcodec/v210enc.c | 12 ++++ libavdevice/decklink_common.cpp | 17 +++-- libavdevice/decklink_common.h | 10 +++ libavdevice/decklink_enc.cpp | 150 ++++++++++++++++++++++++++++++++++++++-- 5 files changed, 183 insertions(+), 10 deletions(-)