[FFmpeg-devel] [PATCH] lavc/utils: use AVPixFmtDescriptor to probe palette formats

[Matthieu Bouron]

From: Matthieu Bouron <matthieu.bouron at stupeflix.com>

Also use the input frame format instead of the AVCodecContext one according
to the documentation of AVCodecContext.get_buffer2().
---

The current code rely on the fact that avpriv_set_systematic_pal2
will error out if the format is not a palette one (and that the error
code is not checked).
I find it makes more sense to probe this kind of formats using the
AVPixFmtDescriptor flags.

Also the documentation of AVCodecContext.get_buffer2() mention that the
frame fields must be used instead of the AVCodecContext one.

Matthieu

---
 libavcodec/utils.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/libavcodec/utils.c b/libavcodec/utils.c
index 9a7ffde..94ec2f6 100644
--- a/libavcodec/utils.c
+++ b/libavcodec/utils.c
@@ -646,6 +646,7 @@ fail:
 static int video_get_buffer(AVCodecContext *s, AVFrame *pic)
 {
     FramePool *pool = s->internal->pool;
+    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pic->format);
     int i;
 
     if (pic->data[0]) {
@@ -653,6 +654,13 @@ static int video_get_buffer(AVCodecContext *s, AVFrame *pic)
         return -1;
     }
 
+    if (!desc) {
+        av_log(s, AV_LOG_ERROR,
+            "Unable to get pixel format descriptor for format %s\n",
+            av_get_pix_fmt_name(pic->format));
+        return AVERROR(EINVAL);
+    }
+
     memset(pic->data, 0, sizeof(pic->data));
     pic->extended_data = pic->data;
 
@@ -669,8 +677,9 @@ static int video_get_buffer(AVCodecContext *s, AVFrame *pic)
         pic->data[i] = NULL;
         pic->linesize[i] = 0;
     }
-    if (pic->data[1] && !pic->data[2])
-        avpriv_set_systematic_pal2((uint32_t *)pic->data[1], s->pix_fmt);
+    if (desc->flags & AV_PIX_FMT_FLAG_PAL ||
+        desc->flags & AV_PIX_FMT_FLAG_PSEUDOPAL)
+        avpriv_set_systematic_pal2((uint32_t *)pic->data[1], pic->format);
 
     if (s->debug & FF_DEBUG_BUFFERS)
         av_log(s, AV_LOG_DEBUG, "default_get_buffer called on pic %p\n", pic);
-- 
2.6.4