[FFmpeg-devel] [PATCH] avfilter/vf_scale: set RGB to always be full range

[Jan Ekström]

This value - while it looks like the actual range of the content -
is nothing but the internal value of swscale.

Thus, if we have RGB content, force the value to 1. Swscale will
ignore it, but at least the value of the output AVFrame will now
properly be "full range" instead of "limited range", as it is right
now.
---
 libavfilter/vf_scale.c | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/libavfilter/vf_scale.c b/libavfilter/vf_scale.c
index 58eee96744..12df27edf4 100644
--- a/libavfilter/vf_scale.c
+++ b/libavfilter/vf_scale.c
@@ -751,6 +751,15 @@ scale:
         || scale->out_range != AVCOL_RANGE_UNSPECIFIED) {
         int in_full, out_full, brightness, contrast, saturation;
         const int *inv_table, *table;
+        const AVPixFmtDescriptor *out_desc = av_pix_fmt_desc_get(out->format);
+        if (!out_desc) {
+            av_log(ctx, AV_LOG_ERROR,
+                   "Failed to get the pixel format descriptor for format %d!\n",
+                   out->format);
+            av_frame_free(&in);
+            av_frame_free(frame_out);
+            return AVERROR_INVALIDDATA;
+        }
 
         sws_getColorspaceDetails(scale->sws, (int **)&inv_table, &in_full,
                                  (int **)&table, &out_full,
@@ -768,7 +777,15 @@ scale:
         else if (in_range != AVCOL_RANGE_UNSPECIFIED)
             in_full  = (in_range == AVCOL_RANGE_JPEG);
         if (scale->out_range != AVCOL_RANGE_UNSPECIFIED)
+            // note: this can be silently overridden by
+            //       sws_setColorspaceDetails for non-YCbCr formats
             out_full = (scale->out_range == AVCOL_RANGE_JPEG);
+        else if (out_desc->flags & AV_PIX_FMT_FLAG_RGB)
+            // the range values received from swscale are its internal
+            // identifiers, and will not be nonzero for any sort of RGB.
+            // thus, if we do not set it to nonzero ourselves, this will
+            // be incorrect.
+            out_full = 1;
 
         sws_setColorspaceDetails(scale->sws, inv_table, in_full,
                                  table, out_full,
-- 
2.26.2