[FFmpeg-devel] [PATCH 3/8] ffmpeg: add support for muxing AVStreamGroups
James Almer
jamrial at gmail.com
Wed Dec 6 00:43:57 EET 2023
Starting with IAMF support.
Signed-off-by: James Almer <jamrial at gmail.com>
---
fftools/ffmpeg.h | 2 +
fftools/ffmpeg_mux_init.c | 335 ++++++++++++++++++++++++++++++++++++++
fftools/ffmpeg_opt.c | 2 +
3 files changed, 339 insertions(+)
diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
index 41935d39d5..057535adbb 100644
--- a/fftools/ffmpeg.h
+++ b/fftools/ffmpeg.h
@@ -262,6 +262,8 @@ typedef struct OptionsContext {
int nb_disposition;
SpecifierOpt *program;
int nb_program;
+ SpecifierOpt *stream_groups;
+ int nb_stream_groups;
SpecifierOpt *time_bases;
int nb_time_bases;
SpecifierOpt *enc_time_bases;
diff --git a/fftools/ffmpeg_mux_init.c b/fftools/ffmpeg_mux_init.c
index 63a25a350f..7648f2a2f1 100644
--- a/fftools/ffmpeg_mux_init.c
+++ b/fftools/ffmpeg_mux_init.c
@@ -39,6 +39,7 @@
#include "libavutil/dict.h"
#include "libavutil/display.h"
#include "libavutil/getenv_utf8.h"
+#include "libavutil/iamf.h"
#include "libavutil/intreadwrite.h"
#include "libavutil/log.h"
#include "libavutil/mem.h"
@@ -1943,6 +1944,336 @@ static int setup_sync_queues(Muxer *mux, AVFormatContext *oc, int64_t buf_size_u
return 0;
}
+static int of_parse_iamf_audio_element_layers(Muxer *mux, AVStreamGroup *stg, char **ptr)
+{
+ AVIAMFAudioElement *audio_element = stg->params.iamf_audio_element;
+ AVDictionary *dict = NULL;
+ const char *token;
+ int ret = 0;
+
+ audio_element->demixing_info =
+ av_iamf_param_definition_alloc(AV_IAMF_PARAMETER_DEFINITION_DEMIXING, 1, NULL);
+ audio_element->recon_gain_info =
+ av_iamf_param_definition_alloc(AV_IAMF_PARAMETER_DEFINITION_RECON_GAIN, 1, NULL);
+
+ if (!audio_element->demixing_info ||
+ !audio_element->recon_gain_info)
+ return AVERROR(ENOMEM);
+
+ /* process manually set layers and parameters */
+ token = av_strtok(NULL, ",", ptr);
+ while (token) {
+ const AVDictionaryEntry *e;
+ int demixing = 0, recon_gain = 0;
+ int layer = 0;
+
+ if (av_strstart(token, "layer=", &token))
+ layer = 1;
+ else if (av_strstart(token, "demixing=", &token))
+ demixing = 1;
+ else if (av_strstart(token, "recon_gain=", &token))
+ recon_gain = 1;
+
+ av_dict_free(&dict);
+ ret = av_dict_parse_string(&dict, token, "=", ":", 0);
+ if (ret < 0) {
+ av_log(mux, AV_LOG_ERROR, "Error parsing audio element specification %s\n", token);
+ goto fail;
+ }
+
+ if (layer) {
+ AVIAMFLayer *audio_layer = av_iamf_audio_element_add_layer(audio_element);
+ if (!audio_layer) {
+ av_log(mux, AV_LOG_ERROR, "Error adding layer to stream group %d\n", stg->index);
+ ret = AVERROR(ENOMEM);
+ goto fail;
+ }
+ av_opt_set_dict(audio_layer, &dict);
+ } else if (demixing || recon_gain) {
+ AVIAMFParamDefinition *param = demixing ? audio_element->demixing_info
+ : audio_element->recon_gain_info;
+ void *subblock = av_iamf_param_definition_get_subblock(param, 0);
+
+ av_opt_set_dict(param, &dict);
+ av_opt_set_dict(subblock, &dict);
+
+ /* Hardcode spec parameters */
+ param->param_definition_mode = 0;
+ param->parameter_rate = stg->streams[0]->codecpar->sample_rate;
+ param->duration =
+ param->constant_subblock_duration = stg->streams[0]->codecpar->frame_size;
+ }
+
+ // make sure that no entries are left in the dict
+ e = NULL;
+ if (e = av_dict_iterate(dict, e)) {
+ av_log(mux, AV_LOG_FATAL, "Unknown layer key %s.\n", e->key);
+ ret = AVERROR(EINVAL);
+ goto fail;
+ }
+ token = av_strtok(NULL, ",", ptr);
+ }
+
+fail:
+ av_dict_free(&dict);
+ if (!ret && !audio_element->nb_layers) {
+ av_log(mux, AV_LOG_ERROR, "No layer in audio element specification\n");
+ ret = AVERROR(EINVAL);
+ }
+
+ return ret;
+}
+
+static int of_parse_iamf_submixes(Muxer *mux, AVStreamGroup *stg, char **ptr)
+{
+ AVFormatContext *oc = mux->fc;
+ AVIAMFMixPresentation *mix = stg->params.iamf_mix_presentation;
+ AVDictionary *dict = NULL;
+ const char *token;
+ char *submix_str = NULL;
+ int ret = 0;
+
+ /* process manually set submixes */
+ token = av_strtok(NULL, ",", ptr);
+ while (token) {
+ AVIAMFSubmix *submix = NULL;
+ const char *subtoken;
+ char *subptr = NULL;
+
+ if (!av_strstart(token, "submix=", &token)) {
+ av_log(mux, AV_LOG_ERROR, "No submix in mix presentation specification \"%s\"\n", token);
+ goto fail;
+ }
+
+ submix_str = av_strdup(token);
+ if (!submix_str)
+ goto fail;
+
+ submix = av_iamf_mix_presentation_add_submix(mix);
+ if (!submix) {
+ av_log(mux, AV_LOG_ERROR, "Error adding submix to stream group %d\n", stg->index);
+ ret = AVERROR(ENOMEM);
+ goto fail;
+ }
+ submix->output_mix_config =
+ av_iamf_param_definition_alloc(AV_IAMF_PARAMETER_DEFINITION_MIX_GAIN, 0, NULL);
+ if (!submix->output_mix_config) {
+ ret = AVERROR(ENOMEM);
+ goto fail;
+ }
+
+ submix->output_mix_config->parameter_rate = stg->streams[0]->codecpar->sample_rate;
+
+ subptr = NULL;
+ subtoken = av_strtok(submix_str, "|", &subptr);
+ while (subtoken) {
+ const AVDictionaryEntry *e;
+ int element = 0, layout = 0;
+
+ if (av_strstart(subtoken, "element=", &subtoken))
+ element = 1;
+ else if (av_strstart(subtoken, "layout=", &subtoken))
+ layout = 1;
+
+ av_dict_free(&dict);
+ ret = av_dict_parse_string(&dict, subtoken, "=", ":", 0);
+ if (ret < 0) {
+ av_log(mux, AV_LOG_ERROR, "Error parsing submix specification \"%s\"\n", subtoken);
+ goto fail;
+ }
+
+ if (element) {
+ AVIAMFSubmixElement *submix_element;
+ int idx = -1;
+
+ if (e = av_dict_get(dict, "stg", NULL, 0))
+ idx = strtol(e->value, NULL, 0);
+ av_dict_set(&dict, "stg", NULL, 0);
+ if (idx < 0 || idx >= oc->nb_stream_groups) {
+ av_log(mux, AV_LOG_ERROR, "Invalid or missing stream group index in "
+ "submix element specification \"%s\"\n", subtoken);
+ ret = AVERROR(EINVAL);
+ goto fail;
+ }
+ submix_element = av_iamf_submix_add_element(submix);
+ if (!submix_element) {
+ av_log(mux, AV_LOG_ERROR, "Error adding element to submix\n");
+ ret = AVERROR(ENOMEM);
+ goto fail;
+ }
+
+ submix_element->audio_element_id = oc->stream_groups[idx]->id;
+
+ submix_element->element_mix_config =
+ av_iamf_param_definition_alloc(AV_IAMF_PARAMETER_DEFINITION_MIX_GAIN, 0, NULL);
+ if (!submix_element->element_mix_config)
+ ret = AVERROR(ENOMEM);
+ av_opt_set_dict2(submix_element, &dict, AV_OPT_SEARCH_CHILDREN);
+ submix_element->element_mix_config->parameter_rate = stg->streams[0]->codecpar->sample_rate;
+ } else if (layout) {
+ AVIAMFSubmixLayout *submix_layout = av_iamf_submix_add_layout(submix);
+ if (!submix_layout) {
+ av_log(mux, AV_LOG_ERROR, "Error adding layout to submix\n");
+ ret = AVERROR(ENOMEM);
+ goto fail;
+ }
+ av_opt_set_dict(submix_layout, &dict);
+ } else
+ av_opt_set_dict2(submix, &dict, AV_OPT_SEARCH_CHILDREN);
+
+ if (ret < 0) {
+ goto fail;
+ }
+
+ // make sure that no entries are left in the dict
+ e = NULL;
+ while (e = av_dict_iterate(dict, e)) {
+ av_log(mux, AV_LOG_FATAL, "Unknown submix key %s.\n", e->key);
+ ret = AVERROR(EINVAL);
+ goto fail;
+ }
+ subtoken = av_strtok(NULL, "|", &subptr);
+ }
+ av_freep(&submix_str);
+
+ if (!submix->nb_elements) {
+ av_log(mux, AV_LOG_ERROR, "No audio elements in submix specification \"%s\"\n", token);
+ ret = AVERROR(EINVAL);
+ }
+ token = av_strtok(NULL, ",", ptr);
+ }
+
+fail:
+ av_dict_free(&dict);
+ av_free(submix_str);
+
+ return ret;
+}
+
+static int of_add_groups(Muxer *mux, const OptionsContext *o)
+{
+ AVFormatContext *oc = mux->fc;
+ int ret;
+
+ /* process manually set groups */
+ for (int i = 0; i < o->nb_stream_groups; i++) {
+ AVDictionary *dict = NULL, *tmp = NULL;
+ const AVDictionaryEntry *e;
+ AVStreamGroup *stg = NULL;
+ int type;
+ const char *token;
+ char *str, *ptr = NULL;
+ const AVOption opts[] = {
+ { "type", "Set group type", offsetof(AVStreamGroup, type), AV_OPT_TYPE_INT,
+ { .i64 = 0 }, 0, INT_MAX, AV_OPT_FLAG_ENCODING_PARAM, "type" },
+ { "iamf_audio_element", NULL, 0, AV_OPT_TYPE_CONST,
+ { .i64 = AV_STREAM_GROUP_PARAMS_IAMF_AUDIO_ELEMENT }, .unit = "type" },
+ { "iamf_mix_presentation", NULL, 0, AV_OPT_TYPE_CONST,
+ { .i64 = AV_STREAM_GROUP_PARAMS_IAMF_MIX_PRESENTATION }, .unit = "type" },
+ { NULL },
+ };
+ const AVClass class = {
+ .class_name = "StreamGroupType",
+ .item_name = av_default_item_name,
+ .option = opts,
+ .version = LIBAVUTIL_VERSION_INT,
+ };
+ const AVClass *pclass = &class;
+
+ str = av_strdup(o->stream_groups[i].u.str);
+ if (!str)
+ goto end;
+
+ token = av_strtok(str, ",", &ptr);
+ if (token) {
+ ret = av_dict_parse_string(&dict, token, "=", ":", AV_DICT_MULTIKEY);
+ if (ret < 0) {
+ av_log(mux, AV_LOG_ERROR, "Error parsing group specification %s\n", token);
+ goto end;
+ }
+
+ // "type" is not a user settable option in AVStreamGroup
+ e = av_dict_get(dict, "type", NULL, 0);
+ if (!e) {
+ av_log(mux, AV_LOG_ERROR, "No type define for Steam Group %d\n", i);
+ ret = AVERROR(EINVAL);
+ goto end;
+ }
+
+ ret = av_opt_eval_int(&pclass, opts, e->value, &type);
+ if (ret < 0 || type == AV_STREAM_GROUP_PARAMS_NONE) {
+ av_log(mux, AV_LOG_ERROR, "Invalid group type \"%s\"\n", e->value);
+ goto end;
+ }
+
+ av_dict_copy(&tmp, dict, 0);
+ stg = avformat_stream_group_create(oc, type, &tmp);
+ if (!stg) {
+ ret = AVERROR(ENOMEM);
+ goto end;
+ }
+ av_dict_set(&tmp, "type", NULL, 0);
+
+ e = NULL;
+ while (e = av_dict_get(dict, "st", e, 0)) {
+ unsigned int idx = strtol(e->value, NULL, 0);
+ if (idx >= oc->nb_streams) {
+ av_log(mux, AV_LOG_ERROR, "Invalid stream index %d\n", idx);
+ ret = AVERROR(EINVAL);
+ goto end;
+ }
+ avformat_stream_group_add_stream(stg, oc->streams[idx]);
+ }
+ while (e = av_dict_get(dict, "stg", e, 0)) {
+ unsigned int idx = strtol(e->value, NULL, 0);
+ if (idx >= oc->nb_stream_groups || idx == stg->index) {
+ av_log(mux, AV_LOG_ERROR, "Invalid stream group index %d\n", idx);
+ ret = AVERROR(EINVAL);
+ goto end;
+ }
+ for (int j = 0; j < oc->stream_groups[idx]->nb_streams; j++)
+ avformat_stream_group_add_stream(stg, oc->stream_groups[idx]->streams[j]);
+ }
+
+ switch(type) {
+ case AV_STREAM_GROUP_PARAMS_IAMF_AUDIO_ELEMENT:
+ ret = of_parse_iamf_audio_element_layers(mux, stg, &ptr);
+ break;
+ case AV_STREAM_GROUP_PARAMS_IAMF_MIX_PRESENTATION:
+ ret = of_parse_iamf_submixes(mux, stg, &ptr);
+ break;
+ default:
+ av_log(mux, AV_LOG_FATAL, "Unknown group type %d.\n", type);
+ ret = AVERROR(EINVAL);
+ break;
+ }
+
+ if (ret < 0)
+ goto end;
+
+ // make sure that nothing but "st" and "stg" entries are left in the dict
+ e = NULL;
+ while (e = av_dict_iterate(tmp, e)) {
+ if (!strcmp(e->key, "st") || !strcmp(e->key, "stg"))
+ continue;
+
+ av_log(mux, AV_LOG_FATAL, "Unknown group key %s.\n", e->key);
+ ret = AVERROR(EINVAL);
+ goto end;
+ }
+ }
+
+end:
+ av_dict_free(&dict);
+ av_dict_free(&tmp);
+ av_free(str);
+ if (ret < 0)
+ return ret;
+ }
+
+ return 0;
+}
+
static int of_add_programs(Muxer *mux, const OptionsContext *o)
{
AVFormatContext *oc = mux->fc;
@@ -2740,6 +3071,10 @@ int of_open(const OptionsContext *o, const char *filename)
if (err < 0)
return err;
+ err = of_add_groups(mux, o);
+ if (err < 0)
+ return err;
+
err = of_add_programs(mux, o);
if (err < 0)
return err;
diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c
index 304471dd03..1144f64f89 100644
--- a/fftools/ffmpeg_opt.c
+++ b/fftools/ffmpeg_opt.c
@@ -1491,6 +1491,8 @@ const OptionDef options[] = {
"add metadata", "string=string" },
{ "program", HAS_ARG | OPT_STRING | OPT_SPEC | OPT_OUTPUT, { .off = OFFSET(program) },
"add program with specified streams", "title=string:st=number..." },
+ { "stream_group", HAS_ARG | OPT_STRING | OPT_SPEC | OPT_OUTPUT, { .off = OFFSET(stream_groups) },
+ "add stream group with specified streams and group type-specific arguments", "id=number:st=number..." },
{ "dframes", HAS_ARG | OPT_PERFILE | OPT_EXPERT |
OPT_OUTPUT, { .func_arg = opt_data_frames },
"set the number of data frames to output", "number" },
--
2.43.0
More information about the ffmpeg-devel
mailing list