[FFmpeg-devel] [PATCH] Correct space color when using libopenjpeg

[Wang Chuan]

When decoding jp2 with palette, [color space] is determined when calling
[opj_decode]. Because of this, [pix_fmt] should be set after decoding.
Otherwise, [pix_fmt] will be set to AV_PIX_FMT_GRAY8 and output an wrong
image.

Signed-off-by: Wang Chuan <ouchuanm at outlook.com>
---
  libavcodec/libopenjpegdec.c | 35 ++++++++++++++++++-----------------
  1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/libavcodec/libopenjpegdec.c b/libavcodec/libopenjpegdec.c
index 206db07ec7..1a8d599840 100644
--- a/libavcodec/libopenjpegdec.c
+++ b/libavcodec/libopenjpegdec.c
@@ -399,6 +399,23 @@ static int libopenjpeg_decode_frame(AVCodecContext 
*avctx, AVFrame *picture,
      if (ret < 0)
          goto done;
  +    ret = !opj_decode(dec, stream, image);
+
+    if (ret) {
+        av_log(avctx, AV_LOG_ERROR, "Error decoding codestream.\n");
+        ret = AVERROR_EXTERNAL;
+        goto done;
+    }
+
+    for (i = 0; i < image->numcomps; i++) {
+        if (!image->comps[i].data) {
+            av_log(avctx, AV_LOG_ERROR,
+                   "Image component %d contains no data.\n", i);
+            ret = AVERROR_INVALIDDATA;
+            goto done;
+        }
+    }
+
      if (avctx->pix_fmt != AV_PIX_FMT_NONE)
          if (!libopenjpeg_matches_pix_fmt(image, avctx->pix_fmt))
              avctx->pix_fmt = AV_PIX_FMT_NONE;
@@ -411,6 +428,7 @@ static int libopenjpeg_decode_frame(AVCodecContext 
*avctx, AVFrame *picture,
          ret = AVERROR_UNKNOWN;
          goto done;
      }
+
      for (i = 0; i < image->numcomps; i++)
          if (image->comps[i].prec > avctx->bits_per_raw_sample)
              avctx->bits_per_raw_sample = image->comps[i].prec;
@@ -418,23 +436,6 @@ static int libopenjpeg_decode_frame(AVCodecContext 
*avctx, AVFrame *picture,
      if ((ret = ff_thread_get_buffer(avctx, picture, 0)) < 0)
          goto done;
  -    ret = !opj_decode(dec, stream, image);
-
-    if (ret) {
-        av_log(avctx, AV_LOG_ERROR, "Error decoding codestream.\n");
-        ret = AVERROR_EXTERNAL;
-        goto done;
-    }
-
-    for (i = 0; i < image->numcomps; i++) {
-        if (!image->comps[i].data) {
-            av_log(avctx, AV_LOG_ERROR,
-                   "Image component %d contains no data.\n", i);
-            ret = AVERROR_INVALIDDATA;
-            goto done;
-        }
-    }
-
      desc       = av_pix_fmt_desc_get(avctx->pix_fmt);
      pixel_size = desc->comp[0].step;
      ispacked   = libopenjpeg_ispacked(avctx->pix_fmt);
-- 
2.37.1 (Apple Git-137.1)