| Message ID | 20230317200941.3936-4-dheitmueller@ltnglobal.com |
|---|---|
| State | New |
| Headers | show |
| Series | Properly handle CEA-708 caption data when transcoding | expand |
On Fri, 17 Mar 2023, 23:11 Devin Heitmueller, < devin.heitmueller@ltnglobal.com> wrote: > Various deinterlacing modes have the effect of doubling the > framerate, and we need to ensure that the caption data isn't > duplicated (or else you get double captions on-screen). > > Use the new ccfifo mechanism for yadif (and yadif_cuda and bwdif > since they use the same yadif core) so that CEA-708 data is > properly preserved through this filter. > > Signed-off-by: Devin Heitmueller <dheitmueller@ltnglobal.com> > --- > libavfilter/vf_bwdif.c | 7 +++++++ > libavfilter/vf_yadif.c | 6 ++++++ > libavfilter/vf_yadif_cuda.c | 8 ++++++++ > libavfilter/yadif.h | 2 ++ > libavfilter/yadif_common.c | 5 +++++ > 5 files changed, 28 insertions(+) > > diff --git a/libavfilter/vf_bwdif.c b/libavfilter/vf_bwdif.c > index 65c617ebb3..de937d83cf 100644 > --- a/libavfilter/vf_bwdif.c > +++ b/libavfilter/vf_bwdif.c > @@ -297,6 +297,7 @@ static av_cold void uninit(AVFilterContext *ctx) > av_frame_free(&yadif->prev); > av_frame_free(&yadif->cur ); > av_frame_free(&yadif->next); > + av_ccfifo_freep(&yadif->cc_fifo); > } > > static const enum AVPixelFormat pix_fmts[] = { > @@ -332,6 +333,12 @@ static int config_props(AVFilterLink *link) > > if(yadif->mode&1) > link->frame_rate = av_mul_q(link->src->inputs[0]->frame_rate, > (AVRational){2,1}); > + else > + link->frame_rate = ctx->inputs[0]->frame_rate; > + > + if (!(yadif->cc_fifo = av_ccfifo_alloc(&link->frame_rate, ctx))) > + av_log(ctx, AV_LOG_VERBOSE, "Failure to setup CC FIFO queue. > Captions will be passed through\n"); > + > > if (link->w < 3 || link->h < 4) { > av_log(ctx, AV_LOG_ERROR, "Video of less than 3 columns or 4 > lines is not supported\n"); > diff --git a/libavfilter/vf_yadif.c b/libavfilter/vf_yadif.c > index 1be02de1a9..b51d21f6ff 100644 > --- a/libavfilter/vf_yadif.c > +++ b/libavfilter/vf_yadif.c > @@ -261,6 +261,7 @@ static av_cold void uninit(AVFilterContext *ctx) > av_frame_free(&yadif->prev); > av_frame_free(&yadif->cur ); > av_frame_free(&yadif->next); > + av_ccfifo_freep(&yadif->cc_fifo); > } > > static const enum AVPixelFormat pix_fmts[] = { > @@ -293,6 +294,11 @@ static int config_output(AVFilterLink *outlink) > if(s->mode & 1) > outlink->frame_rate = av_mul_q(ctx->inputs[0]->frame_rate, > (AVRational){2, 1}); > + else > + outlink->frame_rate = ctx->inputs[0]->frame_rate; > + > + if (!(s->cc_fifo = av_ccfifo_alloc(&outlink->frame_rate, ctx))) > + av_log(ctx, AV_LOG_VERBOSE, "Failure to setup CC FIFO queue. > Captions will be passed through\n"); > > if (outlink->w < 3 || outlink->h < 3) { > av_log(ctx, AV_LOG_ERROR, "Video of less than 3 columns or lines > is not supported\n"); > diff --git a/libavfilter/vf_yadif_cuda.c b/libavfilter/vf_yadif_cuda.c > index 685b8a2035..d96ec391a0 100644 > --- a/libavfilter/vf_yadif_cuda.c > +++ b/libavfilter/vf_yadif_cuda.c > @@ -206,6 +206,9 @@ static av_cold void deint_cuda_uninit(AVFilterContext > *ctx) > av_frame_free(&y->cur); > av_frame_free(&y->next); > > + if (yadif->cc_fifo) > + av_cc_fifo_free(yadif->cc_fifo); > + > av_buffer_unref(&s->device_ref); > s->hwctx = NULL; > av_buffer_unref(&s->input_frames_ref); > @@ -291,6 +294,11 @@ static int config_output(AVFilterLink *link) > if(y->mode & 1) > link->frame_rate = av_mul_q(ctx->inputs[0]->frame_rate, > (AVRational){2, 1}); > + else > + outlink->frame_rate = ctx->inputs[0]->frame_rate; > + > + if (!(s->cc_fifo = av_cc_fifo_alloc(&outlink->frame_rate, ctx))) > + av_log(ctx, AV_LOG_VERBOSE, "Failure to setup CC FIFO queue. > Captions will be passed through\n"); > > if (link->w < 3 || link->h < 3) { > av_log(ctx, AV_LOG_ERROR, "Video of less than 3 columns or lines > is not supported\n"); > diff --git a/libavfilter/yadif.h b/libavfilter/yadif.h > index c928911b35..ccd4304860 100644 > --- a/libavfilter/yadif.h > +++ b/libavfilter/yadif.h > @@ -21,6 +21,7 @@ > > #include "libavutil/opt.h" > #include "libavutil/pixdesc.h" > +#include "libavutil/ccfifo.h" > #include "avfilter.h" > > enum YADIFMode { > @@ -76,6 +77,7 @@ typedef struct YADIFContext { > int eof; > uint8_t *temp_line; > int temp_line_size; > + AVCCFifo *cc_fifo; > > /* > * An algorithm that treats first and/or last fields in a sequence > diff --git a/libavfilter/yadif_common.c b/libavfilter/yadif_common.c > index a10cf7a17f..f95b8c10d3 100644 > --- a/libavfilter/yadif_common.c > +++ b/libavfilter/yadif_common.c > @@ -43,6 +43,7 @@ static int return_frame(AVFilterContext *ctx, int > is_second) > return AVERROR(ENOMEM); > > av_frame_copy_props(yadif->out, yadif->cur); > + av_ccfifo_inject(yadif->cc_fifo, yadif->out); > yadif->out->interlaced_frame = 0; > if (yadif->current_field == YADIF_FIELD_BACK_END) > yadif->current_field = YADIF_FIELD_END; > @@ -96,6 +97,8 @@ int ff_yadif_filter_frame(AVFilterLink *link, AVFrame > *frame) > > av_assert0(frame); > > + av_ccfifo_extract(yadif->cc_fifo, frame); > + > if (yadif->frame_pending) > return_frame(ctx, 1); > > @@ -137,6 +140,7 @@ int ff_yadif_filter_frame(AVFilterLink *link, AVFrame > *frame) > if (!yadif->out) > return AVERROR(ENOMEM); > > + av_ccfifo_inject(yadif->cc_fifo, yadif->out); > av_frame_free(&yadif->prev); > if (yadif->out->pts != AV_NOPTS_VALUE) > yadif->out->pts *= 2; > @@ -148,6 +152,7 @@ int ff_yadif_filter_frame(AVFilterLink *link, AVFrame > *frame) > return AVERROR(ENOMEM); > > av_frame_copy_props(yadif->out, yadif->cur); > + av_ccfifo_inject(yadif->cc_fifo, yadif->out); > yadif->out->interlaced_frame = 0; > > if (yadif->out->pts != AV_NOPTS_VALUE) > -- > 2.35.1.655.ga68dfadae5 > Hello Devin, How would this be propagated (or replicated) in other deinterlacers, namely Intel's OneVPL based deinterlace_qsv, vpp_qsv and VAAPI's deinterlace_vaapi? >
Hello Dennis, On Sat, Mar 18, 2023 at 11:48 AM Dennis Mungai <dmngaie@gmail.com> wrote: > Hello Devin, > > How would this be propagated (or replicated) in other deinterlacers, namely Intel's OneVPL based deinterlace_qsv, vpp_qsv and VAAPI's deinterlace_vaapi? So I was focused mainly on the video filters that I use regularly, which didn't include the hardware accelerated ones. That said, if you look at the patches themselves, they're actually only a few lines of code that need to be added to the individual filters, so the maintainers of those filters should have little difficulty reusing the framework. I would be quite happy to see other filters reuse the same mechanism, rather than reinventing the wheel, and I intentionally tried to minimize the amount of code required by modules to make use of the framework. Devin
diff --git a/libavfilter/vf_bwdif.c b/libavfilter/vf_bwdif.c index 65c617ebb3..de937d83cf 100644 --- a/libavfilter/vf_bwdif.c +++ b/libavfilter/vf_bwdif.c @@ -297,6 +297,7 @@ static av_cold void uninit(AVFilterContext *ctx) av_frame_free(&yadif->prev); av_frame_free(&yadif->cur ); av_frame_free(&yadif->next); + av_ccfifo_freep(&yadif->cc_fifo); } static const enum AVPixelFormat pix_fmts[] = { @@ -332,6 +333,12 @@ static int config_props(AVFilterLink *link) if(yadif->mode&1) link->frame_rate = av_mul_q(link->src->inputs[0]->frame_rate, (AVRational){2,1}); + else + link->frame_rate = ctx->inputs[0]->frame_rate; + + if (!(yadif->cc_fifo = av_ccfifo_alloc(&link->frame_rate, ctx))) + av_log(ctx, AV_LOG_VERBOSE, "Failure to setup CC FIFO queue. Captions will be passed through\n"); + if (link->w < 3 || link->h < 4) { av_log(ctx, AV_LOG_ERROR, "Video of less than 3 columns or 4 lines is not supported\n"); diff --git a/libavfilter/vf_yadif.c b/libavfilter/vf_yadif.c index 1be02de1a9..b51d21f6ff 100644 --- a/libavfilter/vf_yadif.c +++ b/libavfilter/vf_yadif.c @@ -261,6 +261,7 @@ static av_cold void uninit(AVFilterContext *ctx) av_frame_free(&yadif->prev); av_frame_free(&yadif->cur ); av_frame_free(&yadif->next); + av_ccfifo_freep(&yadif->cc_fifo); } static const enum AVPixelFormat pix_fmts[] = { @@ -293,6 +294,11 @@ static int config_output(AVFilterLink *outlink) if(s->mode & 1) outlink->frame_rate = av_mul_q(ctx->inputs[0]->frame_rate, (AVRational){2, 1}); + else + outlink->frame_rate = ctx->inputs[0]->frame_rate; + + if (!(s->cc_fifo = av_ccfifo_alloc(&outlink->frame_rate, ctx))) + av_log(ctx, AV_LOG_VERBOSE, "Failure to setup CC FIFO queue. Captions will be passed through\n"); if (outlink->w < 3 || outlink->h < 3) { av_log(ctx, AV_LOG_ERROR, "Video of less than 3 columns or lines is not supported\n"); diff --git a/libavfilter/vf_yadif_cuda.c b/libavfilter/vf_yadif_cuda.c index 685b8a2035..d96ec391a0 100644 --- a/libavfilter/vf_yadif_cuda.c +++ b/libavfilter/vf_yadif_cuda.c @@ -206,6 +206,9 @@ static av_cold void deint_cuda_uninit(AVFilterContext *ctx) av_frame_free(&y->cur); av_frame_free(&y->next); + if (yadif->cc_fifo) + av_cc_fifo_free(yadif->cc_fifo); + av_buffer_unref(&s->device_ref); s->hwctx = NULL; av_buffer_unref(&s->input_frames_ref); @@ -291,6 +294,11 @@ static int config_output(AVFilterLink *link) if(y->mode & 1) link->frame_rate = av_mul_q(ctx->inputs[0]->frame_rate, (AVRational){2, 1}); + else + outlink->frame_rate = ctx->inputs[0]->frame_rate; + + if (!(s->cc_fifo = av_cc_fifo_alloc(&outlink->frame_rate, ctx))) + av_log(ctx, AV_LOG_VERBOSE, "Failure to setup CC FIFO queue. Captions will be passed through\n"); if (link->w < 3 || link->h < 3) { av_log(ctx, AV_LOG_ERROR, "Video of less than 3 columns or lines is not supported\n"); diff --git a/libavfilter/yadif.h b/libavfilter/yadif.h index c928911b35..ccd4304860 100644 --- a/libavfilter/yadif.h +++ b/libavfilter/yadif.h @@ -21,6 +21,7 @@ #include "libavutil/opt.h" #include "libavutil/pixdesc.h" +#include "libavutil/ccfifo.h" #include "avfilter.h" enum YADIFMode { @@ -76,6 +77,7 @@ typedef struct YADIFContext { int eof; uint8_t *temp_line; int temp_line_size; + AVCCFifo *cc_fifo; /* * An algorithm that treats first and/or last fields in a sequence diff --git a/libavfilter/yadif_common.c b/libavfilter/yadif_common.c index a10cf7a17f..f95b8c10d3 100644 --- a/libavfilter/yadif_common.c +++ b/libavfilter/yadif_common.c @@ -43,6 +43,7 @@ static int return_frame(AVFilterContext *ctx, int is_second) return AVERROR(ENOMEM); av_frame_copy_props(yadif->out, yadif->cur); + av_ccfifo_inject(yadif->cc_fifo, yadif->out); yadif->out->interlaced_frame = 0; if (yadif->current_field == YADIF_FIELD_BACK_END) yadif->current_field = YADIF_FIELD_END; @@ -96,6 +97,8 @@ int ff_yadif_filter_frame(AVFilterLink *link, AVFrame *frame) av_assert0(frame); + av_ccfifo_extract(yadif->cc_fifo, frame); + if (yadif->frame_pending) return_frame(ctx, 1); @@ -137,6 +140,7 @@ int ff_yadif_filter_frame(AVFilterLink *link, AVFrame *frame) if (!yadif->out) return AVERROR(ENOMEM); + av_ccfifo_inject(yadif->cc_fifo, yadif->out); av_frame_free(&yadif->prev); if (yadif->out->pts != AV_NOPTS_VALUE) yadif->out->pts *= 2; @@ -148,6 +152,7 @@ int ff_yadif_filter_frame(AVFilterLink *link, AVFrame *frame) return AVERROR(ENOMEM); av_frame_copy_props(yadif->out, yadif->cur); + av_ccfifo_inject(yadif->cc_fifo, yadif->out); yadif->out->interlaced_frame = 0; if (yadif->out->pts != AV_NOPTS_VALUE)
Various deinterlacing modes have the effect of doubling the framerate, and we need to ensure that the caption data isn't duplicated (or else you get double captions on-screen). Use the new ccfifo mechanism for yadif (and yadif_cuda and bwdif since they use the same yadif core) so that CEA-708 data is properly preserved through this filter. Signed-off-by: Devin Heitmueller <dheitmueller@ltnglobal.com> --- libavfilter/vf_bwdif.c | 7 +++++++ libavfilter/vf_yadif.c | 6 ++++++ libavfilter/vf_yadif_cuda.c | 8 ++++++++ libavfilter/yadif.h | 2 ++ libavfilter/yadif_common.c | 5 +++++ 5 files changed, 28 insertions(+)