[FFmpeg-devel] [PATCH 1/3 v3] avformat: add a Tile Grid stream group type

James Almer jamrial at gmail.com
Mon Jan 22 20:09:00 EET 2024


This will be used to support tiled image formats like HEIF.

Signed-off-by: James Almer <jamrial at gmail.com>
---
Moved the struct to libavformat. It can be moved to libavutil if we decide to
reuse it.

 libavformat/avformat.c |  8 ++++
 libavformat/avformat.h | 86 ++++++++++++++++++++++++++++++++++++++++++
 libavformat/dump.c     | 42 +++++++++++++++++++++
 libavformat/options.c  | 32 ++++++++++++++++
 4 files changed, 168 insertions(+)

diff --git a/libavformat/avformat.c b/libavformat/avformat.c
index 882927f7b1..58dfa0746a 100644
--- a/libavformat/avformat.c
+++ b/libavformat/avformat.c
@@ -100,6 +100,14 @@ void ff_free_stream_group(AVStreamGroup **pstg)
         av_iamf_mix_presentation_free(&stg->params.iamf_mix_presentation);
         break;
     }
+    case AV_STREAM_GROUP_PARAMS_TILE_GRID: {
+        if (stg->params.tile_grid) {
+            av_freep(&stg->params.tile_grid->tile_width);
+            av_freep(&stg->params.tile_grid->tile_height);
+        }
+        av_freep(&stg->params.tile_grid);
+        break;
+    }
     default:
         break;
     }
diff --git a/libavformat/avformat.h b/libavformat/avformat.h
index 5d0fe82250..ab9a3fc6be 100644
--- a/libavformat/avformat.h
+++ b/libavformat/avformat.h
@@ -1022,11 +1022,96 @@ enum AVStreamGroupParamsType {
     AV_STREAM_GROUP_PARAMS_NONE,
     AV_STREAM_GROUP_PARAMS_IAMF_AUDIO_ELEMENT,
     AV_STREAM_GROUP_PARAMS_IAMF_MIX_PRESENTATION,
+    AV_STREAM_GROUP_PARAMS_TILE_GRID,
 };
 
 struct AVIAMFAudioElement;
 struct AVIAMFMixPresentation;
 
+/**
+ * AVTileGrid holds information on how to combine several independent images in
+ * a single grid for presentation.
+ *
+ * Its size is not a part of the ABI. No new fields may be added to this struct
+ * without a major version bump.
+ */
+typedef struct AVTileGrid {
+    const AVClass *av_class;
+
+    /**
+     * Amount of rows in the grid.
+     *
+     * Must be > 0.
+     */
+    int tile_rows;
+    /**
+     * Amount of columns in the grid.
+     *
+     * Must be > 0.
+     */
+    int tile_cols;
+
+    /**
+     * A @ref tile_rows * @ref tile_cols sized array of width values for each
+     * tile in the grid, in row major order.
+     * The sum of tile width values must be consistent across all rows.
+     *
+     * Must be allocated with the av_malloc() family of functions, and will be
+     * freed by av_tile_grid_free().
+     */
+    int *tile_width;
+
+    /**
+     * A @ref tile_rows * @ref tile_cols sized array of height values for each
+     * tile in the grid, in row major order.
+     * The sum of tile height values must be consistent across all columns.
+     *
+     * Must be allocated with the av_malloc() family of functions, and will be
+     * freed by av_tile_grid_free().
+     */
+    int *tile_height;
+
+    /**
+     * Offset in pixels from the left edge of the grid where the actual image
+     * meant for presentation starts.
+     *
+     * This field must be >= 0 and <= the sum of values for a row in the
+     * tile_width array minus @ref output_width.
+     */
+    int horizontal_offset;
+    /**
+     * Offset in pixels from the top edge of the grid where the actual image meant
+     * for presentation starts.
+     *
+     * This field must be >= 0 and <= the sum of values for a column in the
+     * tile_height array minus @ref output_height.
+     */
+    int vertical_offset;
+
+    /**
+     * Width of the final image for presentation.
+     *
+     * Must be > 0 and <= the sum of values for a row in the tile_width array
+     * minus @ref horizontal_offset.
+     * When it's not equal to the sum of values for a row in the tile_width array,
+     * the result of said sum minus output_width minus @ref horizontal_offset is
+     * the amount of pixels to be cropped from the right edge of the final image
+     * before presentation.
+     */
+    int output_width;
+    /**
+     * Height of the final image for presentation.
+     *
+     * Must be > 0 and <= the sum of values for a column in the tile_height array
+     * minus @ref vertical_offset.
+     * When it's not equal to the sum of values for a column in the tile_height,
+     * the result of said sum minus output_height minus @ref vertical_offset is
+     * the amount of pixels to be cropped from the bottom edge of the final image
+     * before presentation.
+     */
+    int output_height;
+} AVTileGrid;
+
 typedef struct AVStreamGroup {
     /**
      * A class for @ref avoptions. Set by avformat_stream_group_create().
@@ -1062,6 +1147,7 @@ typedef struct AVStreamGroup {
     union {
         struct AVIAMFAudioElement *iamf_audio_element;
         struct AVIAMFMixPresentation *iamf_mix_presentation;
+        struct AVTileGrid *tile_grid;
     } params;
 
     /**
diff --git a/libavformat/dump.c b/libavformat/dump.c
index aff51b43f6..5a8b5d7160 100644
--- a/libavformat/dump.c
+++ b/libavformat/dump.c
@@ -22,6 +22,7 @@
 #include <stdio.h>
 #include <stdint.h>
 
+#include "libavutil/avstring.h"
 #include "libavutil/channel_layout.h"
 #include "libavutil/display.h"
 #include "libavutil/iamf.h"
@@ -513,6 +514,10 @@ static void dump_sidedata(void *ctx, const AVStream *st, const char *indent,
             av_log(ctx, log_level, "SMPTE ST 12-1:2014: ");
             dump_s12m_timecode(ctx, st, sd, log_level);
             break;
+        case AV_PKT_DATA_TILE_INFO:
+            av_log(ctx, log_level, "HEIF tile info: tile %u/%u",
+                   AV_RL32(sd->data) + 1U, AV_RL32(sd->data + 4));
+            break;
         default:
             av_log(ctx, log_level, "unknown side data type %d "
                    "(%"SIZE_SPECIFIER" bytes)", sd->type, sd->size);
@@ -639,6 +644,8 @@ static void dump_stream_format(const AVFormatContext *ic, int i,
         av_log(NULL, log_level, " (still image)");
     if (st->disposition & AV_DISPOSITION_NON_DIEGETIC)
         av_log(NULL, log_level, " (non-diegetic)");
+    if (st->disposition & AV_DISPOSITION_TILE)
+        av_log(NULL, log_level, " (tile)");
     av_log(NULL, log_level, "\n");
 
     dump_metadata(NULL, st->metadata, extra_indent, log_level);
@@ -720,6 +727,41 @@ static void dump_stream_group(const AVFormatContext *ic, uint8_t *printed,
             }
         }
         break;
+    case AV_STREAM_GROUP_PARAMS_TILE_GRID: {
+        const AVTileGrid *tile_grid = stg->params.tile_grid;
+        AVCodecContext *avctx = avcodec_alloc_context3(NULL);
+        const char *ptr = NULL;
+        av_log(NULL, AV_LOG_INFO, " Tile Grid:");
+        av_log(NULL, AV_LOG_VERBOSE, " %d rows, %d columns,", tile_grid->tile_rows, tile_grid->tile_cols);
+        if (avctx && stg->nb_streams && !avcodec_parameters_to_context(avctx, stg->streams[0]->codecpar)) {
+            int size = tile_grid->tile_rows * tile_grid->tile_cols;
+            int coded_width = 0, coded_height = 0;
+            avctx->width  = tile_grid->output_width;
+            avctx->height = tile_grid->output_height;
+            for (int j = 0; j < tile_grid->tile_cols; j++)
+                coded_width += tile_grid->tile_width[j];
+            for (int j = 0; j < size; j += tile_grid->tile_cols)
+                coded_height += tile_grid->tile_height[j];
+            avctx->coded_width  = coded_width;
+            avctx->coded_height = coded_height;
+            if (ic->dump_separator)
+                av_opt_set(avctx, "dump_separator", ic->dump_separator, 0);
+            buf[0] = 0;
+            avcodec_string(buf, sizeof(buf), avctx, is_output);
+            ptr = av_stristr(buf, " ");
+        }
+        avcodec_free_context(&avctx);
+        if (ptr)
+            av_log(NULL, AV_LOG_INFO, "%s", ptr);
+        av_log(NULL, AV_LOG_INFO, "\n");
+        dump_metadata(NULL, stg->metadata, "    ", AV_LOG_INFO);
+        for (int i = 0; i < stg->nb_streams; i++) {
+            const AVStream *st = stg->streams[i];
+            dump_stream_format(ic, st->index, i, index, is_output, AV_LOG_VERBOSE);
+            printed[st->index] = 1;
+        }
+        break;
+    }
     }
     default:
         break;
diff --git a/libavformat/options.c b/libavformat/options.c
index 75ec86ce05..31620013c6 100644
--- a/libavformat/options.c
+++ b/libavformat/options.c
@@ -337,6 +337,26 @@ fail:
     return NULL;
 }
 
+#define FLAGS AV_OPT_FLAG_ENCODING_PARAM | AV_OPT_FLAG_VIDEO_PARAM
+#define OFFSET(x) offsetof(AVTileGrid, x)
+static const AVOption tile_grid_options[] = {
+    { "tile_rows", NULL, OFFSET(tile_rows), AV_OPT_TYPE_INT, { .i64 = 1 }, 1, INT_MAX, FLAGS },
+    { "tile_cols", NULL, OFFSET(tile_cols), AV_OPT_TYPE_INT, { .i64 = 1 }, 1, INT_MAX, FLAGS },
+    { "horizontal_offset", NULL, OFFSET(horizontal_offset), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, FLAGS },
+    { "vertical_offset", NULL, OFFSET(vertical_offset),     AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, FLAGS },
+    { "output_size", "size of the output image", OFFSET(output_width), AV_OPT_TYPE_IMAGE_SIZE,
+        { .str = NULL }, 0, INT_MAX, FLAGS },
+    { NULL },
+};
+#undef FLAGS
+#undef OFFSET
+
+static const AVClass tile_grid_class = {
+    .class_name = "AVTileGrid",
+    .version    = LIBAVUTIL_VERSION_INT,
+    .option     = tile_grid_options,
+};
+
 static void *stream_group_child_next(void *obj, void *prev)
 {
     AVStreamGroup *stg = obj;
@@ -346,6 +366,8 @@ static void *stream_group_child_next(void *obj, void *prev)
             return stg->params.iamf_audio_element;
         case AV_STREAM_GROUP_PARAMS_IAMF_MIX_PRESENTATION:
             return stg->params.iamf_mix_presentation;
+        case AV_STREAM_GROUP_PARAMS_TILE_GRID:
+            return stg->params.tile_grid;
         default:
             break;
         }
@@ -368,6 +390,9 @@ static const AVClass *stream_group_child_iterate(void **opaque)
     case AV_STREAM_GROUP_PARAMS_IAMF_MIX_PRESENTATION:
         ret = av_iamf_mix_presentation_get_class();
         break;
+    case AV_STREAM_GROUP_PARAMS_TILE_GRID:
+        ret = &tile_grid_class;
+        break;
     default:
         break;
     }
@@ -429,6 +454,13 @@ AVStreamGroup *avformat_stream_group_create(AVFormatContext *s,
         if (!stg->params.iamf_mix_presentation)
             goto fail;
         break;
+    case AV_STREAM_GROUP_PARAMS_TILE_GRID:
+        stg->params.tile_grid = av_mallocz(sizeof(*stg->params.tile_grid));
+        if (!stg->params.tile_grid)
+            goto fail;
+        stg->params.tile_grid->av_class = &tile_grid_class;
+        av_opt_set_defaults(stg->params.tile_grid);
+        break;
     default:
         goto fail;
     }
-- 
2.43.0



More information about the ffmpeg-devel mailing list