[FFmpeg-devel] [PATCH] Reduce rounding error in ff_scale_adjust_dimensions.

[Tristan Schmelcher]

When force_original_aspect_ratio and force_divisible_by are both
used, dimensions are now rounded to the nearest allowed multiple of
force_divisible_by rather than first rounding to the nearest integer and
then rounding in a static direction. This results in less distortion of
the aspect ratio.

Reviewed-by: Thierry Foucu <tfoucu at google.com>
Signed-off-by: Tristan Schmelcher <tschmelcher at google.com>
---
 libavfilter/scale_eval.c | 11 +++++++----
 libavfilter/scale_eval.h |  3 ++-
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/libavfilter/scale_eval.c b/libavfilter/scale_eval.c
index dfec081e15..75ed503f15 100644
--- a/libavfilter/scale_eval.c
+++ b/libavfilter/scale_eval.c
@@ -148,14 +148,17 @@ int ff_scale_adjust_dimensions(AVFilterLink *inlink,
      * dimensions so that it is not divisible by the set factors anymore
      * unless force_divisible_by is defined as well */
     if (force_original_aspect_ratio) {
-        int tmp_w = av_rescale(h, inlink->w, inlink->h);
-        int tmp_h = av_rescale(w, inlink->h, inlink->w);
+        // Including force_divisible_by here rounds to the nearest multiple of it.
+        int tmp_w = av_rescale(h, inlink->w, inlink->h * (int64_t)force_divisible_by)
+                    * force_divisible_by;
+        int tmp_h = av_rescale(w, inlink->h, inlink->w * (int64_t)force_divisible_by)
+                    * force_divisible_by;
 
         if (force_original_aspect_ratio == 1) {
              w = FFMIN(tmp_w, w);
              h = FFMIN(tmp_h, h);
              if (force_divisible_by > 1) {
-                 // round down
+                 // round down in case provided w or h is not divisible.
                  w = w / force_divisible_by * force_divisible_by;
                  h = h / force_divisible_by * force_divisible_by;
              }
@@ -163,7 +166,7 @@ int ff_scale_adjust_dimensions(AVFilterLink *inlink,
              w = FFMAX(tmp_w, w);
              h = FFMAX(tmp_h, h);
              if (force_divisible_by > 1) {
-                 // round up
+                 // round up in case provided w or h is not divisible.
                  w = (w + force_divisible_by - 1) / force_divisible_by * force_divisible_by;
                  h = (h + force_divisible_by - 1) / force_divisible_by * force_divisible_by;
              }
diff --git a/libavfilter/scale_eval.h b/libavfilter/scale_eval.h
index fceb023fec..2eb6970aad 100644
--- a/libavfilter/scale_eval.h
+++ b/libavfilter/scale_eval.h
@@ -38,7 +38,8 @@ int ff_scale_eval_dimensions(void *ctx,
  * Transform evaluated width and height obtained from ff_scale_eval_dimensions
  * into actual target width and height for scaling. Adjustment can occur if one
  * or both of the evaluated values are of the form '-n' or if
- * force_original_aspect_ratio is set.
+ * force_original_aspect_ratio is set. force_divisible_by is used only when
+ * force_original_aspect_ratio is set and must be at least 1.
  *
  * Returns 0.
  */
-- 
2.37.3.968.ga6b4b080e4-goog