[FFmpeg-devel] [PATCH] Use channel count if channel layout is undefined

[Marcin Gorzel]

Rematrixing supports up to 64 channels but there is only a limited number of channel layouts defined. Currently, in/out channel count is obtained from the channel layout so if the channel layout is undefined (e.g. for 9, 10, 11 channels etc.) the in/out channel count will be 0 and the rematrixing will fail. This change adds a check if the channel layout is non-zero, and if not, prefers to use the in|out_ch_count instead. This seems to be related to ticket #6790.
---
 libavcodec/utils.c                |  1 +
 libswresample/rematrix.c          | 18 ++++++++++++------
 libswresample/swresample.c        | 10 ++++++++++
 libswresample/x86/rematrix_init.c |  8 ++++++--
 4 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/libavcodec/utils.c b/libavcodec/utils.c
index 59d41ccbb6..728f2b3355 100644
--- a/libavcodec/utils.c
+++ b/libavcodec/utils.c
@@ -674,6 +674,7 @@ int attribute_align_arg avcodec_open2(AVCodecContext *avctx, const AVCodec *code
         av_freep(&avctx->subtitle_header);
 
     if (avctx->channels > FF_SANE_NB_CHANNELS) {
+        av_log(avctx, AV_LOG_ERROR, "Unsupported number of channels: %d.\n", avctx->channels);
         ret = AVERROR(EINVAL);
         goto free_and_end;
     }
diff --git a/libswresample/rematrix.c b/libswresample/rematrix.c
index 8227730056..8c9fbf3804 100644
--- a/libswresample/rematrix.c
+++ b/libswresample/rematrix.c
@@ -69,10 +69,12 @@ int swr_set_matrix(struct SwrContext *s, const double *matrix, int stride)
         return AVERROR(EINVAL);
     memset(s->matrix, 0, sizeof(s->matrix));
     memset(s->matrix_flt, 0, sizeof(s->matrix_flt));
-    nb_in = (s->user_in_ch_count > 0) ? s->user_in_ch_count :
-        av_get_channel_layout_nb_channels(s->user_in_ch_layout);
-    nb_out = (s->user_out_ch_count > 0) ? s->user_out_ch_count :
-        av_get_channel_layout_nb_channels(s->user_out_ch_layout);
+    nb_in = s->user_in_ch_layout != 0
+        ? av_get_channel_layout_nb_channels(s->user_in_ch_layout)
+        : s->user_in_ch_count;
+    nb_out = s->user_out_ch_layout != 0
+        ? av_get_channel_layout_nb_channels(s->user_out_ch_layout)
+        : s->user_out_ch_count;
     for (out = 0; out < nb_out; out++) {
         for (in = 0; in < nb_in; in++)
             s->matrix_flt[out][in] = s->matrix[out][in] = matrix[in];
@@ -384,8 +386,12 @@ av_cold static int auto_matrix(SwrContext *s)
 
 av_cold int swri_rematrix_init(SwrContext *s){
     int i, j;
-    int nb_in  = av_get_channel_layout_nb_channels(s->in_ch_layout);
-    int nb_out = av_get_channel_layout_nb_channels(s->out_ch_layout);
+    int nb_in  = s->in_ch_layout != 0
+        ? av_get_channel_layout_nb_channels(s->in_ch_layout)
+        : s->user_in_ch_count;
+    int nb_out = s->out_ch_layout != 0
+        ? av_get_channel_layout_nb_channels(s->out_ch_layout)
+        : s->user_out_ch_count;
 
     s->mix_any_f = NULL;
 
diff --git a/libswresample/swresample.c b/libswresample/swresample.c
index 5bd39caac4..ecb9d97bc9 100644
--- a/libswresample/swresample.c
+++ b/libswresample/swresample.c
@@ -168,6 +168,16 @@ av_cold int swr_init(struct SwrContext *s){
     s-> in.ch_count  = s->  user_in_ch_count;
     s->used_ch_count = s->user_used_ch_count;
 
+    if(s->user_out_ch_count > SWR_CH_MAX) {
+        av_log(s, AV_LOG_ERROR, "Output channel count %d is unsupported.\n", s->user_out_ch_count);
+        return AVERROR(EINVAL);
+    }
+
+    if(s->user_in_ch_count > SWR_CH_MAX) {
+        av_log(s, AV_LOG_ERROR, "Input channel count %d is unsupported.\n", s->user_in_ch_count);
+        return AVERROR(EINVAL);
+    }
+
     s-> in_ch_layout = s-> user_in_ch_layout;
     s->out_ch_layout = s->user_out_ch_layout;
 
diff --git a/libswresample/x86/rematrix_init.c b/libswresample/x86/rematrix_init.c
index d71b41a73e..a6ae074926 100644
--- a/libswresample/x86/rematrix_init.c
+++ b/libswresample/x86/rematrix_init.c
@@ -33,8 +33,12 @@ D(int16, sse2)
 av_cold int swri_rematrix_init_x86(struct SwrContext *s){
 #if HAVE_X86ASM
     int mm_flags = av_get_cpu_flags();
-    int nb_in  = av_get_channel_layout_nb_channels(s->in_ch_layout);
-    int nb_out = av_get_channel_layout_nb_channels(s->out_ch_layout);
+    int nb_in  = s->in_ch_layout != 0
+        ? av_get_channel_layout_nb_channels(s->in_ch_layout)
+        : s->user_in_ch_count;
+    int nb_out = s->out_ch_layout != 0
+        ? av_get_channel_layout_nb_channels(s->out_ch_layout)
+        : s->user_out_ch_count;
     int num    = nb_in * nb_out;
     int i,j;
 
-- 
2.18.0.203.gfac676dfb9-goog