[FFmpeg-devel] [PATCH 2/3] fftools/ffmpeg: attach wallclock timing information to packets and frames
Anton Khirnov
anton at khirnov.net
Tue Dec 12 14:09:57 EET 2023
Will become useful in following commits.
---
fftools/ffmpeg.c | 3 +++
fftools/ffmpeg.h | 13 +++++++++++++
fftools/ffmpeg_dec.c | 14 ++++++++++++--
fftools/ffmpeg_demux.c | 26 ++++++++++++--------------
fftools/ffmpeg_enc.c | 16 ++++++++++++++++
fftools/ffmpeg_filter.c | 9 +++++++++
6 files changed, 65 insertions(+), 16 deletions(-)
diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c
index e7ff9a6d6f..42e675dcb9 100644
--- a/fftools/ffmpeg.c
+++ b/fftools/ffmpeg.c
@@ -416,6 +416,9 @@ static int frame_data_ensure(AVBufferRef **dst, int writable)
fd->dec.frame_num = UINT64_MAX;
fd->dec.pts = AV_NOPTS_VALUE;
+
+ for (unsigned i = 0; i < FF_ARRAY_ELEMS(fd->wallclock); i++)
+ fd->wallclock[i] = INT64_MIN;
} else if (writable)
return av_buffer_make_writable(dst);
diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
index aafb35538e..7f84f3266d 100644
--- a/fftools/ffmpeg.h
+++ b/fftools/ffmpeg.h
@@ -94,6 +94,17 @@ enum PacketOpaque {
PKT_OPAQUE_FIX_SUB_DURATION,
};
+enum LatencyProbe {
+ LATENCY_PROBE_DEMUX,
+ LATENCY_PROBE_DEC_PRE,
+ LATENCY_PROBE_DEC_POST,
+ LATENCY_PROBE_FILTER_PRE,
+ LATENCY_PROBE_FILTER_POST,
+ LATENCY_PROBE_ENC_PRE,
+ LATENCY_PROBE_ENC_POST,
+ LATENCY_PROBE_NB,
+};
+
typedef struct HWDevice {
const char *name;
enum AVHWDeviceType type;
@@ -631,6 +642,8 @@ typedef struct FrameData {
AVRational frame_rate_filter;
int bits_per_raw_sample;
+
+ int64_t wallclock[LATENCY_PROBE_NB];
} FrameData;
extern InputFile **input_files;
diff --git a/fftools/ffmpeg_dec.c b/fftools/ffmpeg_dec.c
index 5dde82a276..ce5aa33f73 100644
--- a/fftools/ffmpeg_dec.c
+++ b/fftools/ffmpeg_dec.c
@@ -22,6 +22,7 @@
#include "libavutil/log.h"
#include "libavutil/pixdesc.h"
#include "libavutil/pixfmt.h"
+#include "libavutil/time.h"
#include "libavutil/timestamp.h"
#include "libavcodec/avcodec.h"
@@ -475,6 +476,13 @@ static int packet_decode(InputStream *ist, AVPacket *pkt, AVFrame *frame)
pkt->dts = AV_NOPTS_VALUE;
}
+ if (pkt) {
+ FrameData *fd = packet_data(pkt);
+ if (!fd)
+ return AVERROR(ENOMEM);
+ fd->wallclock[LATENCY_PROBE_DEC_PRE] = av_gettime_relative();
+ }
+
ret = avcodec_send_packet(dec, pkt);
if (ret < 0 && !(ret == AVERROR_EOF && !pkt)) {
// In particular, we don't expect AVERROR(EAGAIN), because we read all
@@ -528,8 +536,6 @@ static int packet_decode(InputStream *ist, AVPacket *pkt, AVFrame *frame)
return AVERROR_INVALIDDATA;
}
-
- av_assert0(!frame->opaque_ref);
fd = frame_data(frame);
if (!fd) {
av_frame_unref(frame);
@@ -540,6 +546,8 @@ static int packet_decode(InputStream *ist, AVPacket *pkt, AVFrame *frame)
fd->dec.frame_num = dec->frame_num - 1;
fd->bits_per_raw_sample = dec->bits_per_raw_sample;
+ fd->wallclock[LATENCY_PROBE_DEC_POST] = av_gettime_relative();
+
frame->time_base = dec->pkt_timebase;
if (dec->codec_type == AVMEDIA_TYPE_AUDIO) {
@@ -932,6 +940,8 @@ int dec_open(InputStream *ist, Scheduler *sch, unsigned sch_idx)
if (ist->st->disposition & AV_DISPOSITION_ATTACHED_PIC)
av_dict_set(&ist->decoder_opts, "threads", "1", 0);
+ av_dict_set(&ist->decoder_opts, "flags", "+copy_opaque", AV_DICT_MULTIKEY);
+
ret = hw_device_setup_for_decode(ist);
if (ret < 0) {
av_log(ist, AV_LOG_ERROR,
diff --git a/fftools/ffmpeg_demux.c b/fftools/ffmpeg_demux.c
index 392b447338..d2466e885d 100644
--- a/fftools/ffmpeg_demux.c
+++ b/fftools/ffmpeg_demux.c
@@ -271,7 +271,7 @@ static void ts_discontinuity_process(Demuxer *d, InputStream *ist,
ts_discontinuity_detect(d, ist, pkt);
}
-static int ist_dts_update(DemuxStream *ds, AVPacket *pkt)
+static int ist_dts_update(DemuxStream *ds, AVPacket *pkt, FrameData *fd)
{
InputStream *ist = &ds->ist;
const AVCodecParameters *par = ist->par;
@@ -326,21 +326,12 @@ static int ist_dts_update(DemuxStream *ds, AVPacket *pkt)
break;
}
- av_assert0(!pkt->opaque_ref);
- if (ds->streamcopy_needed) {
- FrameData *fd;
-
- fd = packet_data(pkt);
- if (!fd)
- return AVERROR(ENOMEM);
-
- fd->dts_est = ds->dts;
- }
+ fd->dts_est = ds->dts;
return 0;
}
-static int ts_fixup(Demuxer *d, AVPacket *pkt)
+static int ts_fixup(Demuxer *d, AVPacket *pkt, FrameData *fd)
{
InputFile *ifile = &d->f;
InputStream *ist = ifile->streams[pkt->stream_index];
@@ -424,7 +415,7 @@ static int ts_fixup(Demuxer *d, AVPacket *pkt)
ts_discontinuity_process(d, ist, pkt);
// update estimated/predicted dts
- ret = ist_dts_update(ds, pkt);
+ ret = ist_dts_update(ds, pkt, fd);
if (ret < 0)
return ret;
@@ -436,9 +427,14 @@ static int input_packet_process(Demuxer *d, AVPacket *pkt, unsigned *send_flags)
InputFile *f = &d->f;
InputStream *ist = f->streams[pkt->stream_index];
DemuxStream *ds = ds_from_ist(ist);
+ FrameData *fd;
int ret = 0;
- ret = ts_fixup(d, pkt);
+ fd = packet_data(pkt);
+ if (!fd)
+ return AVERROR(ENOMEM);
+
+ ret = ts_fixup(d, pkt, fd);
if (ret < 0)
return ret;
@@ -455,6 +451,8 @@ static int input_packet_process(Demuxer *d, AVPacket *pkt, unsigned *send_flags)
ds->data_size += pkt->size;
ds->nb_packets++;
+ fd->wallclock[LATENCY_PROBE_DEMUX] = av_gettime_relative();
+
if (debug_ts) {
av_log(NULL, AV_LOG_INFO, "demuxer+ffmpeg -> ist_index:%d:%d type:%s pkt_pts:%s pkt_pts_time:%s pkt_dts:%s pkt_dts_time:%s duration:%s duration_time:%s off:%s off_time:%s\n",
f->index, pkt->stream_index,
diff --git a/fftools/ffmpeg_enc.c b/fftools/ffmpeg_enc.c
index 320df2dee7..43e8b077eb 100644
--- a/fftools/ffmpeg_enc.c
+++ b/fftools/ffmpeg_enc.c
@@ -34,6 +34,7 @@
#include "libavutil/log.h"
#include "libavutil/pixdesc.h"
#include "libavutil/rational.h"
+#include "libavutil/time.h"
#include "libavutil/timestamp.h"
#include "libavcodec/avcodec.h"
@@ -615,6 +616,14 @@ static int encode_frame(OutputFile *of, OutputStream *ost, AVFrame *frame,
int ret;
if (frame) {
+ FrameData *fd = frame_data(frame);
+
+ fd = frame_data(frame);
+ if (!fd)
+ return AVERROR(ENOMEM);
+
+ fd->wallclock[LATENCY_PROBE_ENC_PRE] = av_gettime_relative();
+
if (ost->enc_stats_pre.io)
enc_stats_write(ost, &ost->enc_stats_pre, frame, NULL,
ost->frames_encoded);
@@ -644,6 +653,8 @@ static int encode_frame(OutputFile *of, OutputStream *ost, AVFrame *frame,
}
while (1) {
+ FrameData *fd;
+
av_packet_unref(pkt);
ret = avcodec_receive_packet(enc, pkt);
@@ -665,6 +676,11 @@ static int encode_frame(OutputFile *of, OutputStream *ost, AVFrame *frame,
return ret;
}
+ fd = packet_data(pkt);
+ if (!fd)
+ return AVERROR(ENOMEM);
+ fd->wallclock[LATENCY_PROBE_ENC_POST] = av_gettime_relative();
+
if (enc->codec_type == AVMEDIA_TYPE_VIDEO) {
ret = update_video_stats(ost, pkt, !!vstats_filename);
if (ret < 0)
diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c
index ada235b084..21f823be12 100644
--- a/fftools/ffmpeg_filter.c
+++ b/fftools/ffmpeg_filter.c
@@ -36,6 +36,7 @@
#include "libavutil/pixfmt.h"
#include "libavutil/imgutils.h"
#include "libavutil/samplefmt.h"
+#include "libavutil/time.h"
#include "libavutil/timestamp.h"
// FIXME private header, used for mid_pred()
@@ -2364,6 +2365,8 @@ static int fg_output_step(OutputFilterPriv *ofp, FilterGraphThread *fgt,
return AVERROR(ENOMEM);
}
+ fd->wallclock[LATENCY_PROBE_FILTER_POST] = av_gettime_relative();
+
// only use bits_per_raw_sample passed through from the decoder
// if the filtergraph did not touch the frame data
if (!fgp->is_meta)
@@ -2576,6 +2579,7 @@ static int send_frame(FilterGraph *fg, FilterGraphThread *fgt,
InputFilter *ifilter, AVFrame *frame)
{
InputFilterPriv *ifp = ifp_from_ifilter(ifilter);
+ FrameData *fd;
AVFrameSideData *sd;
int need_reinit, ret;
@@ -2651,6 +2655,11 @@ static int send_frame(FilterGraph *fg, FilterGraphThread *fgt,
)
#endif
+ fd = frame_data(frame);
+ if (!fd)
+ return AVERROR(ENOMEM);
+ fd->wallclock[LATENCY_PROBE_FILTER_PRE] = av_gettime_relative();
+
ret = av_buffersrc_add_frame_flags(ifp->filter, frame,
AV_BUFFERSRC_FLAG_PUSH);
if (ret < 0) {
--
2.42.0
More information about the ffmpeg-devel
mailing list