[FFmpeg-devel] [PATCH v5 1/6] ccfifo: Properly handle CEA-708 captions through framerate conversion
Lance Wang
lance.lmwang at gmail.com
Fri May 5 17:09:52 EEST 2023
On Fri, May 5, 2023 at 5:07 AM Devin Heitmueller <
devin.heitmueller at ltnglobal.com> wrote:
> When transcoding video that contains 708 closed captions, the
> caption data is tied to the frames as side data. Simply dropping
> or adding frames to change the framerate will result in loss of
> data, so the caption data needs to be preserved and reformatted.
>
> For example, without this patch converting 720p59 to 1080i59
> would result in loss of 50% of the caption bytes, resulting in
> garbled 608 captions and 708 probably wouldn't render at all.
> Further, the frames that are there will have an illegal
> cc_count for the target framerate, so some decoders may ignore
> the packets entirely.
>
> Extract the 608 and 708 tuples and insert them onto queues. Then
> after dropping/adding frames, re-write the tuples back into the
> resulting frames at the appropriate rate given the target
> framerate. This includes both having the correct cc_count as
> well as clocking out the 608 pairs at the appropriate rate.
>
> Thanks for Lance Wang <lance.lmwang at gmail.com> and Anton
> Khirnov <anton at khirnov.net> for providing review/feedback.
>
> Signed-off-by: Devin Heitmueller <dheitmueller at ltnglobal.com>
> ---
> libavfilter/Makefile | 1 +
> libavfilter/ccfifo.c | 222
> +++++++++++++++++++++++++++++++++++++++++++++++++++
> libavfilter/ccfifo.h | 110 +++++++++++++++++++++++++
> 3 files changed, 333 insertions(+)
> create mode 100644 libavfilter/ccfifo.c
> create mode 100644 libavfilter/ccfifo.h
>
> diff --git a/libavfilter/Makefile b/libavfilter/Makefile
> index 482aeaf..68c8f14 100644
> --- a/libavfilter/Makefile
> +++ b/libavfilter/Makefile
> @@ -14,6 +14,7 @@ OBJS = allfilters.o
> \
> buffersink.o \
> buffersrc.o \
> colorspace.o \
> + ccfifo.o \
> drawutils.o \
> fifo.o \
> formats.o \
> diff --git a/libavfilter/ccfifo.c b/libavfilter/ccfifo.c
> new file mode 100644
> index 0000000..357f764
> --- /dev/null
> +++ b/libavfilter/ccfifo.c
> @@ -0,0 +1,222 @@
> +/*
> + * CEA-708 Closed Captioning FIFO
> + * Copyright (c) 2023 LTN Global Communications
> + *
> + * Author: Devin Heitmueller <dheitmueller at ltnglobal.com>
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
> 02110-1301 USA
> + */
> +
> +#include "ccfifo.h"
> +
> +struct AVCCFifo {
> + AVFifo *cc_608_fifo;
> + AVFifo *cc_708_fifo;
> + AVRational framerate;
> + int expected_cc_count;
> + int expected_608;
> + int cc_detected;
> + int passthrough;
> + int passthrough_warning;
> + void *log_ctx;
> +};
> +
> +#define MAX_CC_ELEMENTS 128
> +#define CC_BYTES_PER_ENTRY 3
> +
> +struct cc_lookup {
> + int num;
> + int den;
> + int cc_count;
> + int num_608;
> +};
> +
> +const static struct cc_lookup cc_lookup_vals[] = {
> + { 15, 1, 40, 4 },
> + { 24, 1, 25, 3 },
> + { 24000, 1001, 25, 3 },
> + { 30, 1, 20, 2 },
> + { 30000, 1001, 20, 2},
> + { 60, 1, 10, 1 },
> + { 60000, 1001, 10, 1},
> +};
> +
> +void ff_ccfifo_freep(AVCCFifo **ccf)
> +{
> + AVCCFifo *tmp = *ccf;
> + av_fifo_freep2(&tmp->cc_608_fifo);
> + av_fifo_freep2(&tmp->cc_708_fifo);
> + av_freep(*ccf);
> +}
> +
> +AVCCFifo *ff_ccfifo_alloc(AVRational framerate, void *log_ctx)
> +{
> + AVCCFifo *ccf;
> + int i;
> +
> + ccf = av_mallocz(sizeof(*ccf));
> + if (!ccf)
> + return NULL;
> +
> + ccf->log_ctx = log_ctx;
> + ccf->framerate = framerate;
> +
> + if (!(ccf->cc_708_fifo = av_fifo_alloc2(MAX_CC_ELEMENTS,
> CC_BYTES_PER_ENTRY, 0)))
> + goto error;
> +
> + if (!(ccf->cc_608_fifo = av_fifo_alloc2(MAX_CC_ELEMENTS,
> CC_BYTES_PER_ENTRY, 0)))
> + goto error;
> +
> + /* Based on the target FPS, figure out the expected cc_count and
> number of
> + 608 tuples per packet. See ANSI/CTA-708-E Sec 4.3.6.1. */
> + for (i = 0; i < FF_ARRAY_ELEMS(cc_lookup_vals); i++) {
> + if (framerate.num == cc_lookup_vals[i].num &&
> + framerate.den == cc_lookup_vals[i].den) {
> + ccf->expected_cc_count = cc_lookup_vals[i].cc_count;
> + ccf->expected_608 = cc_lookup_vals[i].num_608;
> + break;
> + }
> + }
> +
> + if (ccf->expected_608 == 0) {
> + /* We didn't find an output frame we support. We'll let the call
> succeed
> + and the FIFO to be allocated, but the extract/inject functions
> will simply
> + leave everything the way it is */
> + ccf->passthrough = 1;
> + }
> +
> + return ccf;
> +
> +error:
> + ff_ccfifo_freep(&ccf);
> + return NULL;
> +}
> +
> +int ff_ccfifo_getoutputsize(AVCCFifo *ccf)
> +{
> + return ccf->expected_cc_count * CC_BYTES_PER_ENTRY;
> +}
> +
> +int ff_ccfifo_ccdetected(AVCCFifo *ccf)
> +{
> + return ccf->cc_detected;
> +}
> +
> +int ff_ccfifo_injectbytes(AVCCFifo *ccf, uint8_t *cc_data, size_t len)
> +{
> + int cc_608_tuples = 0;
> + int cc_708_tuples = 0;
> + int cc_filled = 0;
> +
> + if (ccf->passthrough) {
> + return 0;
> + }
> +
> + if (len < ff_ccfifo_getoutputsize(ccf)) {
> + return AVERROR(EINVAL);
> + }
> +
> + /* Insert any available data from the 608 FIFO */
> + if (ccf->expected_608 <= av_fifo_can_read(ccf->cc_608_fifo))
> + cc_608_tuples = ccf->expected_608;
> + else
> + cc_608_tuples = av_fifo_can_read(ccf->cc_608_fifo);
> + av_fifo_read(ccf->cc_608_fifo, cc_data, cc_608_tuples);
> + cc_filled += cc_608_tuples;
> +
> + /* Insert any available data from the 708 FIFO */
> + if ((ccf->expected_cc_count - cc_filled) <=
> av_fifo_can_read(ccf->cc_708_fifo))
> + cc_708_tuples = ccf->expected_cc_count - cc_filled;
> + else
> + cc_708_tuples = av_fifo_can_read(ccf->cc_708_fifo);
> + av_fifo_read(ccf->cc_708_fifo, &cc_data[cc_filled *
> CC_BYTES_PER_ENTRY], cc_708_tuples);
> + cc_filled += cc_708_tuples;
> +
> + /* Insert 708 padding into any remaining fields */
> + while (cc_filled < ccf->expected_cc_count) {
> + cc_data[cc_filled * CC_BYTES_PER_ENTRY] = 0xfa;
> + cc_data[cc_filled * CC_BYTES_PER_ENTRY + 1] = 0x00;
> + cc_data[cc_filled * CC_BYTES_PER_ENTRY + 2] = 0x00;
> + cc_filled++;
> + }
> +
> + return 0;
> +}
> +
> +int ff_ccfifo_inject(AVCCFifo *ccf, AVFrame *frame)
> +{
> + AVFrameSideData *sd;
> + int ret;
> +
> + if (ccf->passthrough == 1 || ccf->cc_detected == 0)
> + return 0;
> +
> + sd = av_frame_new_side_data(frame, AV_FRAME_DATA_A53_CC,
> + ff_ccfifo_getoutputsize(ccf));
> + if (sd) {
> + ret = ff_ccfifo_injectbytes(ccf, sd->data, sd->size);
> + if (ret < 0) {
> + av_frame_remove_side_data(frame, AV_FRAME_DATA_A53_CC);
> + return AVERROR(ENOMEM);
>
prefer to return ret from ff_ccfifo_injectbytes()
> + }
> + }
> +
> + return 0;
> +}
> +
> +int ff_ccfifo_extractbytes(AVCCFifo *ccf, uint8_t *cc_bytes, size_t len)
> +{
> + int cc_count = len / CC_BYTES_PER_ENTRY;
> +
> + if (ccf->passthrough == 1) {
> + av_log_once(ccf->log_ctx, AV_LOG_WARNING, AV_LOG_DEBUG,
> &ccf->passthrough_warning,
> + "cc_fifo cannot transcode captions fps=%d/%d\n",
> + ccf->framerate.num, ccf->framerate.den);
> + return 0;
> + }
> +
> + ccf->cc_detected = 1;
> +
> + for (int i = 0; i < cc_count; i++) {
> + /* See ANSI/CTA-708-E Sec 4.3, Table 3 */
> + uint8_t cc_valid = (cc_bytes[CC_BYTES_PER_ENTRY*i] & 0x04) >> 2;
> + uint8_t cc_type = cc_bytes[CC_BYTES_PER_ENTRY*i] & 0x03;
> + if (cc_type == 0x00 || cc_type == 0x01) {
> + av_fifo_write(ccf->cc_608_fifo,
> &cc_bytes[CC_BYTES_PER_ENTRY*i], 1);
> + } else if (cc_valid && (cc_type == 0x02 || cc_type == 0x03)) {
> + av_fifo_write(ccf->cc_708_fifo,
> &cc_bytes[CC_BYTES_PER_ENTRY*i], 1);
> + }
> + }
> + return 0;
> +}
> +
> +/* Read the A53 side data, discard padding, and put 608/708 into
> + queues so we can ensure they get into the output frames at
> + the correct rate... */
> +int ff_ccfifo_extract(AVCCFifo *ccf, AVFrame *frame)
> +{
> + AVFrameSideData *side_data = av_frame_get_side_data(frame,
> AV_FRAME_DATA_A53_CC);
> + if (side_data) {
> + ff_ccfifo_extractbytes(ccf, side_data->data, side_data->size);
> +
> + /* Remove the side data, as we will re-create it on the
> + output as needed */
> + if (!ccf->passthrough)
> + av_frame_remove_side_data(frame, AV_FRAME_DATA_A53_CC);
> + }
> + return 0;
> +}
> diff --git a/libavfilter/ccfifo.h b/libavfilter/ccfifo.h
> new file mode 100644
> index 0000000..44c9245
> --- /dev/null
> +++ b/libavfilter/ccfifo.h
> @@ -0,0 +1,110 @@
> +/*
> + * CEA-708 Closed Captioning FIFO
> + * Copyright (c) 2023 LTN Global Communications
> + *
> + * Author: Devin Heitmueller <dheitmueller at ltnglobal.com>
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
> 02110-1301 USA
> + */
> +
> +/**
> + * @file
> + * CC FIFO Buffer
> + */
> +
> +#ifndef AVFILTER_CCFIFO_H
> +#define AVFILTER_CCFIFO_H
> +
> +#include "libavutil/avutil.h"
> +#include "libavutil/frame.h"
> +#include "libavutil/fifo.h"
> +
> +typedef struct AVCCFifo AVCCFifo;
> +
> +/**
> + * Allocate an AVCCFifo.
> + *
> + * @param framerate output framerate
> + * @param log_ctx used for any av_log() calls
> + * @return newly allocated AVCCFifo, or NULL on error
> + */
> +AVCCFifo *ff_ccfifo_alloc(AVRational framerate, void *log_ctx);
> +
> +/**
> + * Free an AVCCFifo
> + *
> + * @param ccf Pointer to the pointer to the AVCCFifo which should be freed
> + * @note `*ptr = NULL` is safe and leads to no action.
> + */
> +void ff_ccfifo_freep(AVCCFifo **ccf);
> +
> +
> +/**
> + * Extract CC data from an AVFrame
> + *
> + * Extract CC bytes from the AVFrame, insert them into our queue, and
> + * remove the side data from the AVFrame. The side data is removed
> + * as it will be re-inserted at the appropriate rate later in the
> + * filter.
> + *
> + * @param af AVCCFifo to write to
> + * @param frame AVFrame with the video frame to operate on
> + * @return Zero on success, or negative AVERROR
> + * code on failure.
> + */
> +int ff_ccfifo_extract(AVCCFifo *ccf, AVFrame *frame);
> +
> +/**
> + *Just like ff_ccfifo_extract(), but takes the raw bytes instead of an
> AVFrame
> + */
> +int ff_ccfifo_extractbytes(AVCCFifo *ccf, uint8_t *data, size_t len);
> +
> +/**
> + * Provide the size in bytes of an output buffer to allocate
> + *
> + * Ask for how many bytes the output will contain, so the caller can
> allocate
> + * an appropriately sized buffer and pass it to ff_ccfifo_injectbytes()
> + *
> + */
> +int ff_ccfifo_getoutputsize(AVCCFifo *ccf);
> +
> +/**
> + * Insert CC data from the FIFO into an AVFrame (as side data)
> + *
> + * Dequeue the appropriate number of CC tuples based on the
> + * frame rate, and insert them into the AVFrame
> + *
> + * @param af AVCCFifo to read from
> + * @param frame AVFrame with the video frame to operate on
> + * @return Zero on success, or negative AVERROR
> + * code on failure.
> + */
> +int ff_ccfifo_inject(AVCCFifo *ccf, AVFrame *frame);
> +
> +/**
> + * Just like ff_ccfifo_inject(), but takes the raw bytes to insert the CC
> data
> + * int rather than an AVFrame
> + */
> +int ff_ccfifo_injectbytes(AVCCFifo *ccf, uint8_t *data, size_t len);
> +
> +/**
> + * Returns 1 if captions have been found as a prior call
> + * to ff_ccfifo_extract() or ff_ccfifo_extractbytes()
> + */
> +int ff_ccfifo_ccdetected(AVCCFifo *ccf);
> +
> +#endif /* AVFILTER_CCFIFO_H */
> --
> 1.8.3.1
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel at ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request at ffmpeg.org with subject "unsubscribe".
>
More information about the ffmpeg-devel
mailing list