[FFmpeg-devel] [PATCH] avutil/avframe: fix channel layout checks in av_frame_copy()

[James Almer]

Normally, both the source and dest frame would have only the old API fields
set, only the new API fields set, or both set. But in some cases, like when
calling av_frame_ref() using a non reference counted source frame where only
the old channel layout API fields were populated, the result would be the dst
frame having both the new and old fields populated.

This commit takes this into account and fixes the checks by calling
av_channel_layout_compare() only if the source frame has the new API fields
set, and doing sanity checks for the source frame old API fields if the new
ones are not set.

Signed-off-by: James Almer <jamrial at gmail.com>
---
 libavutil/frame.c | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/libavutil/frame.c b/libavutil/frame.c
index fbb869fffa..0da4bb4768 100644
--- a/libavutil/frame.c
+++ b/libavutil/frame.c
@@ -727,19 +727,30 @@ static int frame_copy_audio(AVFrame *dst, const AVFrame *src)
 
 #if FF_API_OLD_CHANNEL_LAYOUT
 FF_DISABLE_DEPRECATION_WARNINGS
+    int channel_layout = dst->ch_layout.order == AV_CHANNEL_ORDER_NATIVE ?
+                         dst->ch_layout.u.mask : 0;
     if (!channels) {
-        if (dst->channels       != src->channels ||
-            dst->channel_layout != src->channel_layout)
-            return AVERROR(EINVAL);
         channels = dst->channels;
+        channel_layout = dst->channel_layout;
         planes = planar ? channels : 1;
+    }
+    if (!src->ch_layout.nb_channels) {
+        if (channels       != src->channels ||
+            channel_layout != src->channel_layout)
+            return AVERROR(EINVAL);
         CHECK_CHANNELS_CONSISTENCY(src);
     }
 FF_ENABLE_DEPRECATION_WARNINGS
 #endif
 
     if (dst->nb_samples != src->nb_samples ||
+#if FF_API_OLD_CHANNEL_LAYOUT
+        (av_channel_layout_check(&src->ch_layout) &&
+#endif
         av_channel_layout_compare(&dst->ch_layout, &src->ch_layout))
+#if FF_API_OLD_CHANNEL_LAYOUT
+        )
+#endif
         return AVERROR(EINVAL);
 
     for (i = 0; i < planes; i++)
@@ -763,7 +774,7 @@ FF_DISABLE_DEPRECATION_WARNINGS
     else if (dst->nb_samples > 0 &&
              (av_channel_layout_check(&dst->ch_layout)
 #if FF_API_OLD_CHANNEL_LAYOUT
-              || dst->channel_layout || dst->channels
+              || dst->channels > 0
 #endif
             ))
         return frame_copy_audio(dst, src);
-- 
2.36.1