avfilter/vf_scale: do aspect ratio and scale factor compensation together

Fixes rounding error

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

diff --git a/libavfilter/vf_scale.c b/libavfilter/vf_scale.c
index 945ed121ddaff555b71340bd871ea1e6e07f2e45..bb5cd39a78c341a1b9f045e909f39e59d3e9cbe8 100644 (file)
--- a/libavfilter/vf_scale.c
+++ b/libavfilter/vf_scale.c
@@ -295,16 +295,14 @@ static int config_props(AVFilterLink *outlink)
         w = inlink->w;
     if (!(h = scale->h))
         h = inlink->h;
-    if (w == -1)
-        w = av_rescale(h, inlink->w, inlink->h);
-    if (h == -1)
-        h = av_rescale(w, inlink->h, inlink->w);
 
     /* Make sure that the result is divisible by the factor we determined
      * earlier. If no factor was set, it is nothing will happen as the default
      * factor is 1 */
-    w = (w / factor_w) * factor_w;
-    h = (h / factor_h) * factor_h;
+    if (w == -1)
+        w = av_rescale(h, inlink->w, inlink->h * factor_w) * factor_w;
+    if (h == -1)
+        h = av_rescale(w, inlink->h, inlink->w * factor_h) * factor_h;
 
     /* Note that force_original_aspect_ratio may overwrite the previous set
      * dimensions so that it is not divisible by the set factors anymore. */