[FFmpeg-devel] [PATCH] swscale/swscale_unscaled: fix planarRgbToplanarRgbWrapper() for formats with bpc between 9-14 bits

[Ramiro Polla]

Currently, planarRgbToplanarRgbWrapper() always sets the alpha value to 255,
without taking the bit depth into consideration.

This commit restricts the alpha value to the bit depth.
---
 libswscale/swscale_unscaled.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/libswscale/swscale_unscaled.c b/libswscale/swscale_unscaled.c
index 488db43e0b..61073c6c0a 100644
--- a/libswscale/swscale_unscaled.c
+++ b/libswscale/swscale_unscaled.c
@@ -1378,8 +1378,15 @@ static int planarRgbToplanarRgbWrapper(SwsInternal *c,
                  dst[1], dstStride[1]);
     ff_copyPlane(src[2], srcStride[2], srcSliceY, srcSliceH, c->opts.src_w,
                  dst[2], dstStride[2]);
-    if (dst[3])
-        fillPlane(dst[3], dstStride[3], c->opts.src_w, srcSliceH, srcSliceY, 255);
+    if (dst[3]) {
+        if (is16BPS(c->opts.dst_format) || isNBPS(c->opts.dst_format)) {
+            const AVPixFmtDescriptor *desc_dst = av_pix_fmt_desc_get(c->opts.dst_format);
+            fillPlane16(dst[3], dstStride[3], c->opts.src_w, srcSliceH, srcSliceY, 1,
+                        desc_dst->comp[3].depth, isBE(c->opts.dst_format));
+        } else {
+            fillPlane(dst[3], dstStride[3], c->opts.src_w, srcSliceH, srcSliceY, 255);
+        }
+    }
 
     return srcSliceH;
 }
-- 
2.30.2