[FFmpeg-devel] [PATCH 3/4 v3] avformat/mov: add support for tile HEIF still images
James Almer
jamrial at gmail.com
Thu Jan 25 21:16:59 EET 2024
Export each tile as its own stream, and the tiling information as a Stream
Group of type TILE_GRID.
This also enables exporting other stream items like thumbnails, which may be
present in non tiled HEIF images too. For those, the primary stream will be
tagged with the default disposition.
Based on a patch by Swaraj Hota
Signed-off-by: James Almer <jamrial at gmail.com>
---
libavcodec/packet.h | 9 ++
libavformat/avformat.h | 6 +
libavformat/dump.c | 6 +
libavformat/isom.h | 7 +-
libavformat/mov.c | 313 ++++++++++++++++++++++++++++++++++++-----
5 files changed, 305 insertions(+), 36 deletions(-)
diff --git a/libavcodec/packet.h b/libavcodec/packet.h
index 2c57d262c6..48ca799334 100644
--- a/libavcodec/packet.h
+++ b/libavcodec/packet.h
@@ -323,6 +323,15 @@ enum AVPacketSideDataType {
*/
AV_PKT_DATA_IAMF_RECON_GAIN_INFO_PARAM,
+ /**
+ * Tile info for image reconstruction, e.g HEIF.
+ * @code
+ * u32le tile number in row major order [0..nb_tiles-1]
+ * u32le nb_tiles
+ * @endcode
+ */
+ AV_PKT_DATA_TILE_INFO,
+
/**
* The number of side data types.
* This is not part of the public API/ABI in the sense that it may
diff --git a/libavformat/avformat.h b/libavformat/avformat.h
index 184bb5b2ec..8199f7b052 100644
--- a/libavformat/avformat.h
+++ b/libavformat/avformat.h
@@ -811,6 +811,12 @@ typedef struct AVIndexEntry {
* The video stream contains still images.
*/
#define AV_DISPOSITION_STILL_IMAGE (1 << 20)
+/**
+ * The video stream is intended to be merged with another stream before
+ * presentation.
+ * Used for example to signal the stream contains a tile from a HEIF grid.
+ */
+#define AV_DISPOSITION_TILE (1 << 21)
/**
* @return The AV_DISPOSITION_* flag corresponding to disp or a negative error
diff --git a/libavformat/dump.c b/libavformat/dump.c
index ef4a8df6fe..6917df96b9 100644
--- a/libavformat/dump.c
+++ b/libavformat/dump.c
@@ -514,6 +514,10 @@ static void dump_sidedata(void *ctx, const AVStream *st, const char *indent,
av_log(ctx, log_level, "SMPTE ST 12-1:2014: ");
dump_s12m_timecode(ctx, st, sd, log_level);
break;
+ case AV_PKT_DATA_TILE_INFO:
+ av_log(ctx, log_level, "HEIF tile info: tile %u/%u",
+ AV_RL32(sd->data) + 1U, AV_RL32(sd->data + 4));
+ break;
default:
av_log(ctx, log_level, "unknown side data type %d "
"(%"SIZE_SPECIFIER" bytes)", sd->type, sd->size);
@@ -640,6 +644,8 @@ static void dump_stream_format(const AVFormatContext *ic, int i,
av_log(NULL, log_level, " (still image)");
if (st->disposition & AV_DISPOSITION_NON_DIEGETIC)
av_log(NULL, log_level, " (non-diegetic)");
+ if (st->disposition & AV_DISPOSITION_TILE)
+ av_log(NULL, log_level, " (tile)");
av_log(NULL, log_level, "\n");
dump_metadata(NULL, st->metadata, extra_indent, log_level);
diff --git a/libavformat/isom.h b/libavformat/isom.h
index 21caaac256..c6a8507a68 100644
--- a/libavformat/isom.h
+++ b/libavformat/isom.h
@@ -267,10 +267,10 @@ typedef struct HEIFItem {
int item_id;
int64_t extent_length;
int64_t extent_offset;
- int64_t size;
int width;
int height;
int type;
+ int is_idat_relative;
} HEIFItem;
typedef struct MOVContext {
@@ -336,6 +336,11 @@ typedef struct MOVContext {
int cur_item_id;
HEIFItem *heif_info;
int heif_info_size;
+ int grid_item_id;
+ int thmb_item_id;
+ int16_t *tile_id_list;
+ int nb_tiles;
+ int64_t idat_offset;
int interleaved_read;
} MOVContext;
diff --git a/libavformat/mov.c b/libavformat/mov.c
index 03ca62895e..3d4de3028b 100644
--- a/libavformat/mov.c
+++ b/libavformat/mov.c
@@ -185,6 +185,30 @@ static int mov_read_mac_string(MOVContext *c, AVIOContext *pb, int len,
return p - dst;
}
+static AVStream *get_curr_st(MOVContext *c)
+{
+ AVStream *st = NULL;
+
+ if (c->fc->nb_streams < 1)
+ return NULL;
+
+ for (int i = 0; i < c->heif_info_size; i++) {
+ HEIFItem *item = &c->heif_info[i];
+
+ if (!item->st)
+ continue;
+ if (item->st->id != c->cur_item_id)
+ continue;
+
+ st = item->st;
+ break;
+ }
+ if (!st)
+ st = c->fc->streams[c->fc->nb_streams-1];
+
+ return st;
+}
+
static int mov_read_covr(MOVContext *c, AVIOContext *pb, int type, int len)
{
AVStream *st;
@@ -1767,9 +1791,9 @@ static int mov_read_colr(MOVContext *c, AVIOContext *pb, MOVAtom atom)
uint16_t color_primaries, color_trc, color_matrix;
int ret;
- if (c->fc->nb_streams < 1)
+ st = get_curr_st(c);
+ if (!st)
return 0;
- st = c->fc->streams[c->fc->nb_streams - 1];
ret = ffio_read_size(pb, color_parameter_type, 4);
if (ret < 0)
@@ -2117,9 +2141,9 @@ static int mov_read_glbl(MOVContext *c, AVIOContext *pb, MOVAtom atom)
AVStream *st;
int ret;
- if (c->fc->nb_streams < 1)
+ st = get_curr_st(c);
+ if (!st)
return 0;
- st = c->fc->streams[c->fc->nb_streams-1];
if ((uint64_t)atom.size > (1<<30))
return AVERROR_INVALIDDATA;
@@ -4946,12 +4970,10 @@ static int heif_add_stream(MOVContext *c, HEIFItem *item)
st->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
st->codecpar->codec_id = mov_codec_id(st, item->type);
sc->ffindex = st->index;
- c->trak_index = st->index;
st->avg_frame_rate.num = st->avg_frame_rate.den = 1;
st->time_base.num = st->time_base.den = 1;
st->nb_frames = 1;
sc->time_scale = 1;
- sc = st->priv_data;
sc->pb = c->fc->pb;
sc->pb_is_copied = 1;
@@ -7779,11 +7801,55 @@ static int mov_read_pitm(MOVContext *c, AVIOContext *pb, MOVAtom atom)
return atom.size;
}
+static int mov_read_idat(MOVContext *c, AVIOContext *pb, MOVAtom atom)
+{
+ c->idat_offset = avio_tell(pb);
+ return 0;
+}
+
+static int read_image_grid(AVFormatContext *s, AVStreamGroupTileGrid *tile_grid,
+ HEIFItem *item)
+{
+ MOVContext *c = s->priv_data;
+ int64_t offset = 0, pos = avio_tell(s->pb);
+ uint8_t flags;
+
+ if (!(s->pb->seekable & AVIO_SEEKABLE_NORMAL)) {
+ av_log(c->fc, AV_LOG_INFO, "grid box with non seekable input\n");
+ return AVERROR_PATCHWELCOME;
+ }
+ if (item->is_idat_relative) {
+ if (!c->idat_offset) {
+ av_log(c->fc, AV_LOG_ERROR, "missing idat box required by the image grid\n");
+ return AVERROR_INVALIDDATA;
+ }
+ offset = c->idat_offset;
+ }
+
+ avio_seek(s->pb, item->extent_offset + offset, SEEK_SET);
+
+ avio_r8(s->pb); /* version */
+ flags = avio_r8(s->pb);
+
+ tile_grid->tile_rows = avio_r8(s->pb) + 1;
+ tile_grid->tile_cols = avio_r8(s->pb) + 1;
+ /* actual width and height of output image */
+ tile_grid->output_width = (flags & 1) ? avio_rb32(s->pb) : avio_rb16(s->pb);
+ tile_grid->output_height = (flags & 1) ? avio_rb32(s->pb) : avio_rb16(s->pb);
+
+ av_log(c->fc, AV_LOG_TRACE, "grid: grid_rows %d grid_cols %d output_width %d output_height %d\n",
+ tile_grid->tile_rows, tile_grid->tile_cols, tile_grid->output_width, tile_grid->output_height);
+
+ avio_seek(s->pb, pos, SEEK_SET);
+
+ return 0;
+}
+
static int mov_read_iloc(MOVContext *c, AVIOContext *pb, MOVAtom atom)
{
int version, offset_size, length_size, base_offset_size, index_size;
int item_count, extent_count;
- uint64_t base_offset, extent_offset, extent_length;
+ int64_t base_offset, extent_offset, extent_length;
uint8_t value;
if (c->found_moov) {
@@ -7830,6 +7896,7 @@ static int mov_read_iloc(MOVContext *c, AVIOContext *pb, MOVAtom atom)
avio_rb16(pb); // data_reference_index.
if (rb_size(pb, &base_offset, base_offset_size) < 0)
return AVERROR_INVALIDDATA;
+ av_log(c->fc, AV_LOG_TRACE, "iloc: base_offset %"PRId64"\n", base_offset);
extent_count = avio_rb16(pb);
if (extent_count > 1) {
// For still AVIF images, we only support one extent item.
@@ -7840,6 +7907,8 @@ static int mov_read_iloc(MOVContext *c, AVIOContext *pb, MOVAtom atom)
if (rb_size(pb, &extent_offset, offset_size) < 0 ||
rb_size(pb, &extent_length, length_size) < 0)
return AVERROR_INVALIDDATA;
+ if (offset_type == 1)
+ c->heif_info[i].is_idat_relative = 1;
c->heif_info[i].extent_length = extent_length;
c->heif_info[i].extent_offset = base_offset + extent_offset;
av_log(c->fc, AV_LOG_TRACE, "iloc: item_idx %d, offset_type %d, "
@@ -7878,10 +7947,6 @@ static int mov_read_infe(MOVContext *c, AVIOContext *pb, MOVAtom atom)
av_log(c->fc, AV_LOG_TRACE, "infe: item_id %d, item_type %s, item_name %s\n",
item_id, av_fourcc2str(item_type), item_name);
- // Skip all but the primary item until support is added
- if (item_id != c->primary_item_id)
- return 0;
-
if (size > 0)
avio_skip(pb, size);
@@ -7895,6 +7960,9 @@ static int mov_read_infe(MOVContext *c, AVIOContext *pb, MOVAtom atom)
if (ret < 0)
return ret;
break;
+ case MKTAG('g','r','i','d'):
+ c->grid_item_id = item_id;
+ break;
default:
av_log(c->fc, AV_LOG_TRACE, "infe: ignoring item_type %s\n", av_fourcc2str(item_type));
break;
@@ -7954,6 +8022,59 @@ static int mov_read_iref(MOVContext *c, AVIOContext *pb, MOVAtom atom)
return mov_read_default(c, pb, atom);
}
+static int mov_read_dimg(MOVContext *c, AVIOContext *pb, MOVAtom atom)
+{
+ int entries, i;
+ int from_item_id = avio_rb16(pb);
+
+ if (c->grid_item_id < 0) {
+ av_log(c->fc, AV_LOG_ERROR, "Missing grid information\n");
+ return AVERROR_INVALIDDATA;
+ }
+ if (from_item_id != c->grid_item_id) {
+ avpriv_request_sample(c->fc, "Derived item of type other than 'grid'");
+ return AVERROR_PATCHWELCOME;
+ }
+ entries = avio_rb16(pb);
+ c->tile_id_list = av_malloc_array(entries, sizeof(*c->tile_id_list));
+ if (!c->tile_id_list)
+ return AVERROR(ENOMEM);
+ /* 'to' item ids */
+ for (i = 0; i < entries; i++)
+ c->tile_id_list[i] = avio_rb16(pb);
+ c->nb_tiles = entries;
+
+ av_log(c->fc, AV_LOG_TRACE, "dimg: from_item_id %d, entries %d\n",
+ from_item_id, entries);
+
+ return 0;
+}
+
+static int mov_read_thmb(MOVContext *c, AVIOContext *pb, MOVAtom atom)
+{
+ int entries;
+ int to_item_id, from_item_id = avio_rb16(pb);
+
+ entries = avio_rb16(pb);
+ if (entries > 1) {
+ avpriv_request_sample(c->fc, "More than one thmb entry");
+ return AVERROR_PATCHWELCOME;
+ }
+ /* 'to' item ids */
+ to_item_id = avio_rb16(pb);
+
+ if (to_item_id != c->primary_item_id ||
+ to_item_id != c->grid_item_id)
+ return 0;
+
+ c->thmb_item_id = from_item_id;
+
+ av_log(c->fc, AV_LOG_TRACE, "thmb: from_item_id %d, entries %d\n",
+ from_item_id, entries);
+
+ return 0;
+}
+
static int mov_read_ispe(MOVContext *c, AVIOContext *pb, MOVAtom atom)
{
uint32_t width, height;
@@ -7969,15 +8090,17 @@ static int mov_read_ispe(MOVContext *c, AVIOContext *pb, MOVAtom atom)
width = avio_rb32(pb);
height = avio_rb32(pb);
- av_log(c->fc, AV_LOG_TRACE, "ispe: item_id %d, width %u, height %u\n",
+ av_log(c->fc, AV_LOG_TRACE, "ispe: cur_item_id %d, width %u, height %u\n",
c->cur_item_id, width, height);
for (int i = 0; i < c->heif_info_size; i++) {
- if (c->heif_info[i].item_id == c->cur_item_id) {
- c->heif_info[i].width = width;
- c->heif_info[i].height = height;
- break;
- }
+ HEIFItem *item = &c->heif_info[i];
+ if (item->item_id != c->cur_item_id)
+ continue;
+
+ item->width = width;
+ item->height = height;
+ break;
}
return 0;
@@ -8074,10 +8197,6 @@ static int mov_read_iprp(MOVContext *c, AVIOContext *pb, MOVAtom atom)
av_log(c->fc, AV_LOG_TRACE, "ipma: property_index %d, item_id %d, item_type %s\n",
index + 1, item_id, av_fourcc2str(ref->type));
- // Skip properties referencing items other than the primary item until support is added
- if (item_id != c->primary_item_id)
- continue;
-
c->cur_item_id = item_id;
ret = mov_read_default(c, &ref->b.pub,
@@ -8205,6 +8324,9 @@ static const MOVParseTableEntry mov_default_parse_table[] = {
{ MKTAG('p','c','m','C'), mov_read_pcmc }, /* PCM configuration box */
{ MKTAG('p','i','t','m'), mov_read_pitm },
{ MKTAG('e','v','c','C'), mov_read_glbl },
+{ MKTAG('d','i','m','g'), mov_read_dimg },
+{ MKTAG('t','h','m','b'), mov_read_thmb },
+{ MKTAG('i','d','a','t'), mov_read_idat },
{ MKTAG('i','r','e','f'), mov_read_iref },
{ MKTAG('i','s','p','e'), mov_read_ispe },
{ MKTAG('i','p','r','p'), mov_read_iprp },
@@ -8713,6 +8835,7 @@ static int mov_read_close(AVFormatContext *s)
av_freep(&mov->aes_decrypt);
av_freep(&mov->chapter_tracks);
av_freep(&mov->heif_info);
+ av_freep(&mov->tile_id_list);
return 0;
}
@@ -8852,6 +8975,116 @@ fail:
return ret;
}
+static int mov_parse_tiles(AVFormatContext *s)
+{
+ MOVContext *mov = s->priv_data;
+ AVStreamGroup *stg = avformat_stream_group_create(s, AV_STREAM_GROUP_PARAMS_TILE_GRID, NULL);
+ AVStreamGroupTileGrid *tile_grid;
+ int err;
+
+ if (!stg)
+ return AVERROR(ENOMEM);
+
+ tile_grid = stg->params.tile_grid;
+
+ av_assert0(mov->grid_item_id >= 0);
+ for (int i = 0; i < mov->heif_info_size; i++) {
+ HEIFItem *item = &mov->heif_info[i];
+
+ if (item->item_id != mov->grid_item_id)
+ continue;
+ err = read_image_grid(s, tile_grid, item);
+ if (err < 0)
+ return err;
+ stg->id = item->item_id;
+ break;
+ }
+
+ for (int i = 0; i < mov->nb_tiles; i++) {
+ int tile_id = mov->tile_id_list[i];
+
+ for (int j = 0; j < mov->heif_info_size; j++) {
+ HEIFItem *item = &mov->heif_info[j];
+ AVStream *st = item->st;
+ AVPacketSideData *sd;
+ MOVStreamContext *sc;
+ int64_t offset = 0;
+
+ if (item->item_id != tile_id)
+ continue;
+ if (!st) {
+ av_log(s, AV_LOG_ERROR, "HEIF tile %d doesn't reference a stream\n", tile_id);
+ return AVERROR_INVALIDDATA;
+ }
+ if (item->is_idat_relative) {
+ if (!mov->idat_offset) {
+ av_log(s, AV_LOG_ERROR, "Missing idat box for HEIF tile %d\n", tile_id);
+ return AVERROR_INVALIDDATA;
+ }
+ offset = mov->idat_offset;
+ }
+
+ st->codecpar->width = item->width;
+ st->codecpar->height = item->height;
+
+ err = avformat_stream_group_add_stream(stg, st);
+ if (err == AVERROR(EEXIST))
+ return AVERROR_INVALIDDATA;
+ else if (err < 0)
+ return err;
+
+ sc = st->priv_data;
+ sc->sample_sizes[0] = item->extent_length;
+ sc->chunk_offsets[0] = item->extent_offset + offset;
+
+ st->disposition |= AV_DISPOSITION_TILE;
+
+ mov_build_index(mov, st);
+
+ sd = av_packet_side_data_new(&st->codecpar->coded_side_data,
+ &st->codecpar->nb_coded_side_data,
+ AV_PKT_DATA_TILE_INFO,
+ sizeof(uint32_t) * 2, 0);
+ if (!sd)
+ return AVERROR(ENOMEM);
+
+ AV_WL32(sd->data, i);
+ AV_WL32(sd->data + 4, mov->nb_tiles);
+ break;
+ }
+ }
+
+ for (int i = 0; i < mov->heif_info_size; i++) {
+ HEIFItem *item = &mov->heif_info[i];
+ AVStream *st = item->st;
+ MOVStreamContext *sc;
+ int64_t offset = 0;
+
+ if (item->item_id != mov->thmb_item_id)
+ continue;
+
+ if (!st) {
+ av_log(s, AV_LOG_ERROR, "HEIF thumbnail doesn't reference a stream\n");
+ return AVERROR_INVALIDDATA;
+ }
+ if (item->is_idat_relative) {
+ if (!mov->idat_offset) {
+ av_log(s, AV_LOG_ERROR, "Missing idat box for HEIF thumbnail\n");
+ return AVERROR_INVALIDDATA;
+ }
+ offset = mov->idat_offset;
+ }
+
+ sc = st->priv_data;
+ sc->sample_sizes[0] = item->extent_length;
+ sc->chunk_offsets[0] = item->extent_offset + offset;
+
+ mov_build_index(mov, st);
+ }
+
+ return 0;
+}
+
static int mov_read_header(AVFormatContext *s)
{
MOVContext *mov = s->priv_data;
@@ -8868,6 +9101,9 @@ static int mov_read_header(AVFormatContext *s)
mov->fc = s;
mov->trak_index = -1;
+ mov->grid_item_id = -1;
+ mov->thmb_item_id = -1;
+ mov->primary_item_id = -1;
/* .mov and .mp4 aren't streamable anyway (only progressive download if moov is before mdat) */
if (pb->seekable & AVIO_SEEKABLE_NORMAL)
atom.size = avio_size(pb);
@@ -8890,23 +9126,30 @@ static int mov_read_header(AVFormatContext *s)
av_log(mov->fc, AV_LOG_TRACE, "on_parse_exit_offset=%"PRId64"\n", avio_tell(pb));
if (mov->found_iloc) {
- for (i = 0; i < mov->heif_info_size; i++) {
- HEIFItem *item = &mov->heif_info[i];
- MOVStreamContext *sc;
- AVStream *st;
+ if (mov->nb_tiles) {
+ err = mov_parse_tiles(s);
+ if (err < 0)
+ return err;
+ } else
+ for (i = 0; i < mov->heif_info_size; i++) {
+ HEIFItem *item = &mov->heif_info[i];
+ AVStream *st = item->st;
+ MOVStreamContext *sc;
- if (!item->st)
- continue;
+ if (!st)
+ continue;
- st = item->st;
- sc = st->priv_data;
- st->codecpar->width = item->width;
- st->codecpar->height = item->height;
- sc->sample_sizes[0] = item->extent_length;
- sc->chunk_offsets[0] = item->extent_offset;
+ sc = st->priv_data;
+ st->codecpar->width = item->width;
+ st->codecpar->height = item->height;
+ sc->sample_sizes[0] = item->extent_length;
+ sc->chunk_offsets[0] = item->extent_offset;
- mov_build_index(mov, st);
- }
+ if (item->item_id == mov->primary_item_id)
+ st->disposition |= AV_DISPOSITION_DEFAULT;
+
+ mov_build_index(mov, st);
+ }
}
if (pb->seekable & AVIO_SEEKABLE_NORMAL) {
--
2.43.0
More information about the ffmpeg-devel
mailing list