[FFmpeg-devel] [PATCH 6/8] avfilter/scale_eval: Use 64bit, check values in ff_scale_adjust_dimensions()

[Michael Niedermayer]

Found by reviewing CID1513722 Operands don't affect result

Sponsored-by: Sovereign Tech Fund
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
---
 libavfilter/scale_eval.c | 9 ++++++---
 libavfilter/scale_eval.h | 2 +-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/libavfilter/scale_eval.c b/libavfilter/scale_eval.c
index 75ed503f15f..dc8d522b1e3 100644
--- a/libavfilter/scale_eval.c
+++ b/libavfilter/scale_eval.c
@@ -114,7 +114,7 @@ int ff_scale_adjust_dimensions(AVFilterLink *inlink,
     int *ret_w, int *ret_h,
     int force_original_aspect_ratio, int force_divisible_by)
 {
-    int w, h;
+    int64_t w, h;
     int factor_w, factor_h;
 
     w = *ret_w;
@@ -149,9 +149,9 @@ int ff_scale_adjust_dimensions(AVFilterLink *inlink,
      * unless force_divisible_by is defined as well */
     if (force_original_aspect_ratio) {
         // Including force_divisible_by here rounds to the nearest multiple of it.
-        int tmp_w = av_rescale(h, inlink->w, inlink->h * (int64_t)force_divisible_by)
+        int64_t tmp_w = av_rescale(h, inlink->w, inlink->h * (int64_t)force_divisible_by)
                     * force_divisible_by;
-        int tmp_h = av_rescale(w, inlink->h, inlink->w * (int64_t)force_divisible_by)
+        int64_t tmp_h = av_rescale(w, inlink->h, inlink->w * (int64_t)force_divisible_by)
                     * force_divisible_by;
 
         if (force_original_aspect_ratio == 1) {
@@ -173,6 +173,9 @@ int ff_scale_adjust_dimensions(AVFilterLink *inlink,
         }
     }
 
+    if ((int32_t)w != w || (int32_t)h != h)
+        return AVERROR(EINVAL);
+
     *ret_w = w;
     *ret_h = h;
 
diff --git a/libavfilter/scale_eval.h b/libavfilter/scale_eval.h
index 2eb6970aad4..b4895284043 100644
--- a/libavfilter/scale_eval.h
+++ b/libavfilter/scale_eval.h
@@ -41,7 +41,7 @@ int ff_scale_eval_dimensions(void *ctx,
  * force_original_aspect_ratio is set. force_divisible_by is used only when
  * force_original_aspect_ratio is set and must be at least 1.
  *
- * Returns 0.
+ * Returns negative error code on error or non negative on success
  */
 int ff_scale_adjust_dimensions(AVFilterLink *inlink,
     int *ret_w, int *ret_h,
-- 
2.45.2