[FFmpeg-cvslog] fftools/ffmpeg: move sub2video handling to ffmpeg_filter
Anton Khirnov
git at videolan.org
Wed May 31 17:26:13 EEST 2023
ffmpeg | branch: master | Anton Khirnov <anton at khirnov.net> | Wed May 24 10:08:33 2023 +0200| [f8abab673c5494d0fc120e2d27f0e5d49ae3d285] | committer: Anton Khirnov
fftools/ffmpeg: move sub2video handling to ffmpeg_filter
Make all relevant state per-filtergraph input, rather than per-input
stream. Refactor the code to make it work and avoid leaking memory when
a single subtitle stream is sent to multiple filters.
> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=f8abab673c5494d0fc120e2d27f0e5d49ae3d285
---
fftools/ffmpeg.c | 133 ++--------------------------------
fftools/ffmpeg.h | 7 +-
fftools/ffmpeg_dec.c | 9 +--
fftools/ffmpeg_demux.c | 1 -
fftools/ffmpeg_filter.c | 189 ++++++++++++++++++++++++++++++++++++++++++------
5 files changed, 177 insertions(+), 162 deletions(-)
diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c
index e9e60407d2..36b4becaf2 100644
--- a/fftools/ffmpeg.c
+++ b/fftools/ffmpeg.c
@@ -147,139 +147,20 @@ static int restore_tty;
This is a temporary solution until libavfilter gets real subtitles support.
*/
-static int sub2video_get_blank_frame(InputStream *ist)
-{
- int ret;
- AVFrame *frame = ist->sub2video.frame;
-
- av_frame_unref(frame);
- frame->width = ist->sub2video.w;
- frame->height = ist->sub2video.h;
- frame->format = AV_PIX_FMT_RGB32;
- if ((ret = av_frame_get_buffer(frame, 0)) < 0)
- return ret;
- memset(frame->data[0], 0, frame->height * frame->linesize[0]);
- return 0;
-}
-
-static void sub2video_copy_rect(uint8_t *dst, int dst_linesize, int w, int h,
- AVSubtitleRect *r)
-{
- uint32_t *pal, *dst2;
- uint8_t *src, *src2;
- int x, y;
-
- if (r->type != SUBTITLE_BITMAP) {
- av_log(NULL, AV_LOG_WARNING, "sub2video: non-bitmap subtitle\n");
- return;
- }
- if (r->x < 0 || r->x + r->w > w || r->y < 0 || r->y + r->h > h) {
- av_log(NULL, AV_LOG_WARNING, "sub2video: rectangle (%d %d %d %d) overflowing %d %d\n",
- r->x, r->y, r->w, r->h, w, h
- );
- return;
- }
-
- dst += r->y * dst_linesize + r->x * 4;
- src = r->data[0];
- pal = (uint32_t *)r->data[1];
- for (y = 0; y < r->h; y++) {
- dst2 = (uint32_t *)dst;
- src2 = src;
- for (x = 0; x < r->w; x++)
- *(dst2++) = pal[*(src2++)];
- dst += dst_linesize;
- src += r->linesize[0];
- }
-}
-
-static void sub2video_push_ref(InputStream *ist, int64_t pts)
-{
- AVFrame *frame = ist->sub2video.frame;
- int i;
- int ret;
-
- av_assert1(frame->data[0]);
- ist->sub2video.last_pts = frame->pts = pts;
- for (i = 0; i < ist->nb_filters; i++) {
- ret = av_buffersrc_add_frame_flags(ist->filters[i]->filter, frame,
- AV_BUFFERSRC_FLAG_KEEP_REF |
- AV_BUFFERSRC_FLAG_PUSH);
- if (ret != AVERROR_EOF && ret < 0)
- av_log(NULL, AV_LOG_WARNING, "Error while add the frame to buffer source(%s).\n",
- av_err2str(ret));
- }
-}
-
-void sub2video_update(InputStream *ist, int64_t heartbeat_pts,
- const AVSubtitle *sub)
-{
- AVFrame *frame = ist->sub2video.frame;
- int8_t *dst;
- int dst_linesize;
- int num_rects, i;
- int64_t pts, end_pts;
-
- if (!frame)
- return;
- if (sub) {
- pts = av_rescale_q(sub->pts + sub->start_display_time * 1000LL,
- AV_TIME_BASE_Q, ist->st->time_base);
- end_pts = av_rescale_q(sub->pts + sub->end_display_time * 1000LL,
- AV_TIME_BASE_Q, ist->st->time_base);
- num_rects = sub->num_rects;
- } else {
- /* If we are initializing the system, utilize current heartbeat
- PTS as the start time, and show until the following subpicture
- is received. Otherwise, utilize the previous subpicture's end time
- as the fall-back value. */
- pts = ist->sub2video.initialize ?
- heartbeat_pts : ist->sub2video.end_pts;
- end_pts = INT64_MAX;
- num_rects = 0;
- }
- if (sub2video_get_blank_frame(ist) < 0) {
- av_log(NULL, AV_LOG_ERROR,
- "Impossible to get a blank canvas.\n");
- return;
- }
- dst = frame->data [0];
- dst_linesize = frame->linesize[0];
- for (i = 0; i < num_rects; i++)
- sub2video_copy_rect(dst, dst_linesize, frame->width, frame->height, sub->rects[i]);
- sub2video_push_ref(ist, pts);
- ist->sub2video.end_pts = end_pts;
- ist->sub2video.initialize = 0;
-}
-
static void sub2video_heartbeat(InputFile *infile, int64_t pts, AVRational tb)
{
- int i, j, nb_reqs;
- int64_t pts2;
-
/* When a frame is read from a file, examine all sub2video streams in
the same file and send the sub2video frame again. Otherwise, decoded
video frames could be accumulating in the filter graph while a filter
(possibly overlay) is desperately waiting for a subtitle frame. */
- for (i = 0; i < infile->nb_streams; i++) {
- InputStream *ist2 = infile->streams[i];
- if (!ist2->sub2video.frame)
- continue;
- /* subtitles seem to be usually muxed ahead of other streams;
- if not, subtracting a larger time here is necessary */
- pts2 = av_rescale_q(pts, tb, ist2->st->time_base) - 1;
- /* do not send the heartbeat frame if the subtitle is already ahead */
- if (pts2 <= ist2->sub2video.last_pts)
+ for (int i = 0; i < infile->nb_streams; i++) {
+ InputStream *ist = infile->streams[i];
+
+ if (ist->dec_ctx->codec_type != AVMEDIA_TYPE_SUBTITLE)
continue;
- if (pts2 >= ist2->sub2video.end_pts || ist2->sub2video.initialize)
- /* if we have hit the end of the current displayed subpicture,
- or if we need to initialize the system, update the
- overlayed subpicture and its start/end times */
- sub2video_update(ist2, pts2 + 1, NULL);
- for (j = 0, nb_reqs = 0; j < ist2->nb_filters; j++)
- nb_reqs += av_buffersrc_get_nb_failed_requests(ist2->filters[j]->filter);
- if (nb_reqs)
- sub2video_push_ref(ist2, pts2);
+
+ for (int j = 0; j < ist->nb_filters; j++)
+ ifilter_sub2video_heartbeat(ist->filters[j], pts, tb);
}
}
diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
index d4aff5cc7c..75695d3fb5 100644
--- a/fftools/ffmpeg.h
+++ b/fftools/ffmpeg.h
@@ -372,11 +372,7 @@ typedef struct InputStream {
} prev_sub;
struct sub2video {
- int64_t last_pts;
- int64_t end_pts;
- AVFrame *frame;
int w, h;
- unsigned int initialize; ///< marks if sub2video_update should force an initialization
} sub2video;
/* decoded data from this stream goes into all those filters
@@ -741,13 +737,12 @@ int init_simple_filtergraph(InputStream *ist, OutputStream *ost,
char *graph_desc);
int init_complex_filtergraph(FilterGraph *fg);
-void sub2video_update(InputStream *ist, int64_t heartbeat_pts,
- const AVSubtitle *sub);
int copy_av_subtitle(AVSubtitle *dst, const AVSubtitle *src);
int ifilter_send_frame(InputFilter *ifilter, AVFrame *frame, int keep_reference);
int ifilter_send_eof(InputFilter *ifilter, int64_t pts, AVRational tb);
int ifilter_sub2video(InputFilter *ifilter, const AVSubtitle *sub);
+void ifilter_sub2video_heartbeat(InputFilter *ifilter, int64_t pts, AVRational tb);
/**
* Set up fallback filtering parameters from a decoder context. They will only
diff --git a/fftools/ffmpeg_dec.c b/fftools/ffmpeg_dec.c
index a7acaf67c2..30959c64b7 100644
--- a/fftools/ffmpeg_dec.c
+++ b/fftools/ffmpeg_dec.c
@@ -321,13 +321,8 @@ static int video_frame_process(InputStream *ist, AVFrame *frame)
static void sub2video_flush(InputStream *ist)
{
- int i;
- int ret;
-
- if (ist->sub2video.end_pts < INT64_MAX)
- sub2video_update(ist, INT64_MAX, NULL);
- for (i = 0; i < ist->nb_filters; i++) {
- ret = av_buffersrc_add_frame(ist->filters[i]->filter, NULL);
+ for (int i = 0; i < ist->nb_filters; i++) {
+ int ret = ifilter_sub2video(ist->filters[i], NULL);
if (ret != AVERROR_EOF && ret < 0)
av_log(NULL, AV_LOG_WARNING, "Flush the frame error.\n");
}
diff --git a/fftools/ffmpeg_demux.c b/fftools/ffmpeg_demux.c
index 817ccbbedc..5c15b8bad3 100644
--- a/fftools/ffmpeg_demux.c
+++ b/fftools/ffmpeg_demux.c
@@ -817,7 +817,6 @@ static void ist_free(InputStream **pist)
av_dict_free(&ist->decoder_opts);
avsubtitle_free(&ist->prev_sub.subtitle);
- av_frame_free(&ist->sub2video.frame);
av_freep(&ist->filters);
av_freep(&ist->outputs);
av_freep(&ist->hwaccel_device);
diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c
index 4b5ccf50bd..8f839cc273 100644
--- a/fftools/ffmpeg_filter.c
+++ b/fftools/ffmpeg_filter.c
@@ -107,6 +107,14 @@ typedef struct InputFilterPriv {
struct {
///< queue of AVSubtitle* before filter init
AVFifo *queue;
+
+ AVFrame *frame;
+
+ int64_t last_pts;
+ int64_t end_pts;
+
+ ///< marks if sub2video_update should force an initialization
+ unsigned int initialize;
} sub2video;
} InputFilterPriv;
@@ -115,6 +123,111 @@ static InputFilterPriv *ifp_from_ifilter(InputFilter *ifilter)
return (InputFilterPriv*)ifilter;
}
+static int sub2video_get_blank_frame(InputFilterPriv *ifp)
+{
+ AVFrame *frame = ifp->sub2video.frame;
+ int ret;
+
+ av_frame_unref(frame);
+
+ frame->width = ifp->width;
+ frame->height = ifp->height;
+ frame->format = ifp->format;
+
+ ret = av_frame_get_buffer(frame, 0);
+ if (ret < 0)
+ return ret;
+
+ memset(frame->data[0], 0, frame->height * frame->linesize[0]);
+
+ return 0;
+}
+
+static void sub2video_copy_rect(uint8_t *dst, int dst_linesize, int w, int h,
+ AVSubtitleRect *r)
+{
+ uint32_t *pal, *dst2;
+ uint8_t *src, *src2;
+ int x, y;
+
+ if (r->type != SUBTITLE_BITMAP) {
+ av_log(NULL, AV_LOG_WARNING, "sub2video: non-bitmap subtitle\n");
+ return;
+ }
+ if (r->x < 0 || r->x + r->w > w || r->y < 0 || r->y + r->h > h) {
+ av_log(NULL, AV_LOG_WARNING, "sub2video: rectangle (%d %d %d %d) overflowing %d %d\n",
+ r->x, r->y, r->w, r->h, w, h
+ );
+ return;
+ }
+
+ dst += r->y * dst_linesize + r->x * 4;
+ src = r->data[0];
+ pal = (uint32_t *)r->data[1];
+ for (y = 0; y < r->h; y++) {
+ dst2 = (uint32_t *)dst;
+ src2 = src;
+ for (x = 0; x < r->w; x++)
+ *(dst2++) = pal[*(src2++)];
+ dst += dst_linesize;
+ src += r->linesize[0];
+ }
+}
+
+static void sub2video_push_ref(InputFilterPriv *ifp, int64_t pts)
+{
+ AVFrame *frame = ifp->sub2video.frame;
+ int ret;
+
+ av_assert1(frame->data[0]);
+ ifp->sub2video.last_pts = frame->pts = pts;
+ ret = av_buffersrc_add_frame_flags(ifp->ifilter.filter, frame,
+ AV_BUFFERSRC_FLAG_KEEP_REF |
+ AV_BUFFERSRC_FLAG_PUSH);
+ if (ret != AVERROR_EOF && ret < 0)
+ av_log(NULL, AV_LOG_WARNING, "Error while add the frame to buffer source(%s).\n",
+ av_err2str(ret));
+}
+
+static void sub2video_update(InputFilterPriv *ifp, int64_t heartbeat_pts,
+ const AVSubtitle *sub)
+{
+ AVFrame *frame = ifp->sub2video.frame;
+ int8_t *dst;
+ int dst_linesize;
+ int num_rects, i;
+ int64_t pts, end_pts;
+
+ if (sub) {
+ pts = av_rescale_q(sub->pts + sub->start_display_time * 1000LL,
+ AV_TIME_BASE_Q, ifp->time_base);
+ end_pts = av_rescale_q(sub->pts + sub->end_display_time * 1000LL,
+ AV_TIME_BASE_Q, ifp->time_base);
+ num_rects = sub->num_rects;
+ } else {
+ /* If we are initializing the system, utilize current heartbeat
+ PTS as the start time, and show until the following subpicture
+ is received. Otherwise, utilize the previous subpicture's end time
+ as the fall-back value. */
+ pts = ifp->sub2video.initialize ?
+ heartbeat_pts : ifp->sub2video.end_pts;
+ end_pts = INT64_MAX;
+ num_rects = 0;
+ }
+ if (sub2video_get_blank_frame(ifp) < 0) {
+ av_log(NULL, AV_LOG_ERROR,
+ "Impossible to get a blank canvas.\n");
+ return;
+ }
+ dst = frame->data [0];
+ dst_linesize = frame->linesize[0];
+ for (i = 0; i < num_rects; i++)
+ sub2video_copy_rect(dst, dst_linesize, frame->width, frame->height, sub->rects[i]);
+ sub2video_push_ref(ifp, pts);
+ ifp->sub2video.end_pts = end_pts;
+ ifp->sub2video.initialize = 0;
+}
+
// FIXME: YUV420P etc. are actually supported with full color range,
// yet the latter information isn't available here.
static const enum AVPixelFormat *get_compliance_normal_pix_fmts(const AVCodec *codec, const enum AVPixelFormat default_formats[])
@@ -465,6 +578,12 @@ static int ifilter_bind_ist(InputFilter *ifilter, InputStream *ist)
if (ret < 0)
return ret;
+ if (ifp->type_src == AVMEDIA_TYPE_SUBTITLE) {
+ ifp->sub2video.frame = av_frame_alloc();
+ if (!ifp->sub2video.frame)
+ return AVERROR(ENOMEM);
+ }
+
return 0;
}
@@ -610,6 +729,7 @@ void fg_free(FilterGraph **pfg)
avsubtitle_free(&sub);
av_fifo_freep2(&ifp->sub2video.queue);
}
+ av_frame_free(&ifp->sub2video.frame);
av_channel_layout_uninit(&ifp->fallback.ch_layout);
@@ -1108,20 +1228,15 @@ void check_filter_outputs(void)
}
}
-static int sub2video_prepare(InputStream *ist, InputFilter *ifilter)
+static void sub2video_prepare(InputFilterPriv *ifp)
{
- ist->sub2video.frame = av_frame_alloc();
- if (!ist->sub2video.frame)
- return AVERROR(ENOMEM);
- ist->sub2video.last_pts = INT64_MIN;
- ist->sub2video.end_pts = INT64_MIN;
+ ifp->sub2video.last_pts = INT64_MIN;
+ ifp->sub2video.end_pts = INT64_MIN;
/* sub2video structure has been (re-)initialized.
Mark it as such so that the system will be
initialized with the first received heartbeat. */
- ist->sub2video.initialize = 1;
-
- return 0;
+ ifp->sub2video.initialize = 1;
}
static int configure_input_video_filter(FilterGraph *fg, InputFilter *ifilter,
@@ -1156,11 +1271,8 @@ static int configure_input_video_filter(FilterGraph *fg, InputFilter *ifilter,
if (!fr.num)
fr = ist->framerate_guessed;
- if (ist->dec_ctx->codec_type == AVMEDIA_TYPE_SUBTITLE) {
- ret = sub2video_prepare(ist, ifilter);
- if (ret < 0)
- goto fail;
- }
+ if (ifp->type_src == AVMEDIA_TYPE_SUBTITLE)
+ sub2video_prepare(ifp);
ifp->time_base = ist->framerate.num ? av_inv_q(ist->framerate) :
ist->st->time_base;
@@ -1466,12 +1578,13 @@ int configure_filtergraph(FilterGraph *fg)
/* process queued up subtitle packets */
for (i = 0; i < fg->nb_inputs; i++) {
- InputFilterPriv *ifp = ifp_from_ifilter(fg->inputs[i]);
- InputStream *ist = ifp->ist;
- if (ifp->sub2video.queue && ist->sub2video.frame) {
+ InputFilter *ifilter = fg->inputs[i];
+ InputFilterPriv *ifp = ifp_from_ifilter(ifilter);
+
+ if (ifp->type_src == AVMEDIA_TYPE_SUBTITLE && ifp->sub2video.queue) {
AVSubtitle tmp;
while (av_fifo_read(ifp->sub2video.queue, &tmp, 1) >= 0) {
- sub2video_update(ist, INT64_MIN, &tmp);
+ sub2video_update(ifp, INT64_MIN, &tmp);
avsubtitle_free(&tmp);
}
}
@@ -1620,15 +1733,47 @@ int reap_filters(int flush)
return 0;
}
+void ifilter_sub2video_heartbeat(InputFilter *ifilter, int64_t pts, AVRational tb)
+{
+ InputFilterPriv *ifp = ifp_from_ifilter(ifilter);
+ int64_t pts2;
+
+ if (!ifilter->graph->graph)
+ return;
+
+ /* subtitles seem to be usually muxed ahead of other streams;
+ if not, subtracting a larger time here is necessary */
+ pts2 = av_rescale_q(pts, tb, ifp->time_base) - 1;
+
+ /* do not send the heartbeat frame if the subtitle is already ahead */
+ if (pts2 <= ifp->sub2video.last_pts)
+ return;
+
+ if (pts2 >= ifp->sub2video.end_pts || ifp->sub2video.initialize)
+ /* if we have hit the end of the current displayed subpicture,
+ or if we need to initialize the system, update the
+ overlayed subpicture and its start/end times */
+ sub2video_update(ifp, pts2 + 1, NULL);
+
+ if (av_buffersrc_get_nb_failed_requests(ifilter->filter))
+ sub2video_push_ref(ifp, pts2);
+}
+
int ifilter_sub2video(InputFilter *ifilter, const AVSubtitle *subtitle)
{
InputFilterPriv *ifp = ifp_from_ifilter(ifilter);
- InputStream *ist = ifp->ist;
int ret;
- if (ist->sub2video.frame) {
- sub2video_update(ist, INT64_MIN, subtitle);
- } else {
+ if (ifilter->graph->graph) {
+ if (!subtitle) {
+ if (ifp->sub2video.end_pts < INT64_MAX)
+ sub2video_update(ifp, INT64_MAX, NULL);
+
+ return av_buffersrc_add_frame(ifilter->filter, NULL);
+ }
+
+ sub2video_update(ifp, INT64_MIN, subtitle);
+ } else if (subtitle) {
AVSubtitle sub;
if (!ifp->sub2video.queue)
More information about the ffmpeg-cvslog
mailing list