[FFmpeg-devel] [PATCH 2/3] swscale/swscale_unscaled: fix packed30togbra10() for formats with bpc between 9-14 bits

[Ramiro Polla]

Currently, packed30togbra10() always sets the alpha value to 0xFFFF,
without taking the bit depth into consideration.

This commit restricts the alpha value to the bit depth.
---
 libswscale/swscale_unscaled.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/libswscale/swscale_unscaled.c b/libswscale/swscale_unscaled.c
index 307e5471a9..8d71a88c23 100644
--- a/libswscale/swscale_unscaled.c
+++ b/libswscale/swscale_unscaled.c
@@ -818,6 +818,7 @@ static void packed30togbra10(const uint8_t *src, int srcStride,
     int x, h, i;
     int dst_alpha = dst[3] != NULL;
     int scale_high = bpc - 10, scale_low = 10 - scale_high;
+    uint16_t alpha_val = (1U << bpc) - 1;
     for (h = 0; h < srcSliceH; h++) {
         uint32_t *src_line = (uint32_t *)(src + srcStride * h);
         unsigned component;
@@ -834,7 +835,7 @@ static void packed30togbra10(const uint8_t *src, int srcStride,
                     dst[1][x] = av_bswap16(component << scale_high | component >> scale_low);
                     component =  p        & 0x3FF;
                     dst[2][x] = av_bswap16(component << scale_high | component >> scale_low);
-                    dst[3][x] = 0xFFFF;
+                    dst[3][x] = av_bswap16(alpha_val);
                     src_line++;
                 }
             } else {
@@ -860,7 +861,7 @@ static void packed30togbra10(const uint8_t *src, int srcStride,
                     dst[1][x] = component << scale_high | component >> scale_low;
                     component =  p        & 0x3FF;
                     dst[2][x] = component << scale_high | component >> scale_low;
-                    dst[3][x] = 0xFFFF;
+                    dst[3][x] = alpha_val;
                     src_line++;
                 }
             } else {
-- 
2.30.2