[FFmpeg-devel] [RFC PATCH 3/5] yadif: Properly preserve CEA-708 closed captions

[Dennis Mungai]

On Fri, 17 Mar 2023, 23:11 Devin Heitmueller, <
devin.heitmueller at ltnglobal.com> wrote:

> Various deinterlacing modes have the effect of doubling the
> framerate, and we need to ensure that the caption data isn't
> duplicated (or else you get double captions on-screen).
>
> Use the new ccfifo mechanism for yadif (and yadif_cuda and bwdif
> since they use the same yadif core) so that CEA-708 data is
> properly preserved through this filter.
>
> Signed-off-by: Devin Heitmueller <dheitmueller at ltnglobal.com>
> ---
>  libavfilter/vf_bwdif.c      | 7 +++++++
>  libavfilter/vf_yadif.c      | 6 ++++++
>  libavfilter/vf_yadif_cuda.c | 8 ++++++++
>  libavfilter/yadif.h         | 2 ++
>  libavfilter/yadif_common.c  | 5 +++++
>  5 files changed, 28 insertions(+)
>
> diff --git a/libavfilter/vf_bwdif.c b/libavfilter/vf_bwdif.c
> index 65c617ebb3..de937d83cf 100644
> --- a/libavfilter/vf_bwdif.c
> +++ b/libavfilter/vf_bwdif.c
> @@ -297,6 +297,7 @@ static av_cold void uninit(AVFilterContext *ctx)
>      av_frame_free(&yadif->prev);
>      av_frame_free(&yadif->cur );
>      av_frame_free(&yadif->next);
> +    av_ccfifo_freep(&yadif->cc_fifo);
>  }
>
>  static const enum AVPixelFormat pix_fmts[] = {
> @@ -332,6 +333,12 @@ static int config_props(AVFilterLink *link)
>
>      if(yadif->mode&1)
>          link->frame_rate = av_mul_q(link->src->inputs[0]->frame_rate,
> (AVRational){2,1});
> +    else
> +        link->frame_rate = ctx->inputs[0]->frame_rate;
> +
> +    if (!(yadif->cc_fifo = av_ccfifo_alloc(&link->frame_rate, ctx)))
> +        av_log(ctx, AV_LOG_VERBOSE, "Failure to setup CC FIFO queue.
> Captions will be passed through\n");
> +
>
>      if (link->w < 3 || link->h < 4) {
>          av_log(ctx, AV_LOG_ERROR, "Video of less than 3 columns or 4
> lines is not supported\n");
> diff --git a/libavfilter/vf_yadif.c b/libavfilter/vf_yadif.c
> index 1be02de1a9..b51d21f6ff 100644
> --- a/libavfilter/vf_yadif.c
> +++ b/libavfilter/vf_yadif.c
> @@ -261,6 +261,7 @@ static av_cold void uninit(AVFilterContext *ctx)
>      av_frame_free(&yadif->prev);
>      av_frame_free(&yadif->cur );
>      av_frame_free(&yadif->next);
> +    av_ccfifo_freep(&yadif->cc_fifo);
>  }
>
>  static const enum AVPixelFormat pix_fmts[] = {
> @@ -293,6 +294,11 @@ static int config_output(AVFilterLink *outlink)
>      if(s->mode & 1)
>          outlink->frame_rate = av_mul_q(ctx->inputs[0]->frame_rate,
>                                      (AVRational){2, 1});
> +    else
> +        outlink->frame_rate = ctx->inputs[0]->frame_rate;
> +
> +    if (!(s->cc_fifo = av_ccfifo_alloc(&outlink->frame_rate, ctx)))
> +        av_log(ctx, AV_LOG_VERBOSE, "Failure to setup CC FIFO queue.
> Captions will be passed through\n");
>
>      if (outlink->w < 3 || outlink->h < 3) {
>          av_log(ctx, AV_LOG_ERROR, "Video of less than 3 columns or lines
> is not supported\n");
> diff --git a/libavfilter/vf_yadif_cuda.c b/libavfilter/vf_yadif_cuda.c
> index 685b8a2035..d96ec391a0 100644
> --- a/libavfilter/vf_yadif_cuda.c
> +++ b/libavfilter/vf_yadif_cuda.c
> @@ -206,6 +206,9 @@ static av_cold void deint_cuda_uninit(AVFilterContext
> *ctx)
>      av_frame_free(&y->cur);
>      av_frame_free(&y->next);
>
> +    if (yadif->cc_fifo)
> +        av_cc_fifo_free(yadif->cc_fifo);
> +
>      av_buffer_unref(&s->device_ref);
>      s->hwctx = NULL;
>      av_buffer_unref(&s->input_frames_ref);
> @@ -291,6 +294,11 @@ static int config_output(AVFilterLink *link)
>      if(y->mode & 1)
>          link->frame_rate = av_mul_q(ctx->inputs[0]->frame_rate,
>                                      (AVRational){2, 1});
> +    else
> +        outlink->frame_rate = ctx->inputs[0]->frame_rate;
> +
> +    if (!(s->cc_fifo = av_cc_fifo_alloc(&outlink->frame_rate, ctx)))
> +        av_log(ctx, AV_LOG_VERBOSE, "Failure to setup CC FIFO queue.
> Captions will be passed through\n");
>
>      if (link->w < 3 || link->h < 3) {
>          av_log(ctx, AV_LOG_ERROR, "Video of less than 3 columns or lines
> is not supported\n");
> diff --git a/libavfilter/yadif.h b/libavfilter/yadif.h
> index c928911b35..ccd4304860 100644
> --- a/libavfilter/yadif.h
> +++ b/libavfilter/yadif.h
> @@ -21,6 +21,7 @@
>
>  #include "libavutil/opt.h"
>  #include "libavutil/pixdesc.h"
> +#include "libavutil/ccfifo.h"
>  #include "avfilter.h"
>
>  enum YADIFMode {
> @@ -76,6 +77,7 @@ typedef struct YADIFContext {
>      int eof;
>      uint8_t *temp_line;
>      int temp_line_size;
> +    AVCCFifo *cc_fifo;
>
>      /*
>       * An algorithm that treats first and/or last fields in a sequence
> diff --git a/libavfilter/yadif_common.c b/libavfilter/yadif_common.c
> index a10cf7a17f..f95b8c10d3 100644
> --- a/libavfilter/yadif_common.c
> +++ b/libavfilter/yadif_common.c
> @@ -43,6 +43,7 @@ static int return_frame(AVFilterContext *ctx, int
> is_second)
>              return AVERROR(ENOMEM);
>
>          av_frame_copy_props(yadif->out, yadif->cur);
> +        av_ccfifo_inject(yadif->cc_fifo, yadif->out);
>          yadif->out->interlaced_frame = 0;
>          if (yadif->current_field == YADIF_FIELD_BACK_END)
>              yadif->current_field = YADIF_FIELD_END;
> @@ -96,6 +97,8 @@ int ff_yadif_filter_frame(AVFilterLink *link, AVFrame
> *frame)
>
>      av_assert0(frame);
>
> +    av_ccfifo_extract(yadif->cc_fifo, frame);
> +
>      if (yadif->frame_pending)
>          return_frame(ctx, 1);
>
> @@ -137,6 +140,7 @@ int ff_yadif_filter_frame(AVFilterLink *link, AVFrame
> *frame)
>          if (!yadif->out)
>              return AVERROR(ENOMEM);
>
> +        av_ccfifo_inject(yadif->cc_fifo, yadif->out);
>          av_frame_free(&yadif->prev);
>          if (yadif->out->pts != AV_NOPTS_VALUE)
>              yadif->out->pts *= 2;
> @@ -148,6 +152,7 @@ int ff_yadif_filter_frame(AVFilterLink *link, AVFrame
> *frame)
>          return AVERROR(ENOMEM);
>
>      av_frame_copy_props(yadif->out, yadif->cur);
> +    av_ccfifo_inject(yadif->cc_fifo, yadif->out);
>      yadif->out->interlaced_frame = 0;
>
>      if (yadif->out->pts != AV_NOPTS_VALUE)
> --
> 2.35.1.655.ga68dfadae5
>


Hello Devin,

How would this be propagated (or replicated) in other deinterlacers, namely
Intel's OneVPL based deinterlace_qsv, vpp_qsv and VAAPI's
deinterlace_vaapi?

>