avcodec/sbrdsp_fixed: Fix assertion failure in sbr_sum_square_c()

This also increases the range of input values supported as well as
decreasing the operation dependencies in the main loop, improving
speed on modern CPUs.

Fixes part of: 2045/clusterfuzz-testcase-minimized-6751255865065472

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

diff --git a/libavcodec/sbrdsp_fixed.c b/libavcodec/sbrdsp_fixed.c
index 59263478d89a9891e1abc6ac83d09d59c984b31d..748772102eee5372275d631c1d64c38f6dbfa8d5 100644 (file)
--- a/libavcodec/sbrdsp_fixed.c
+++ b/libavcodec/sbrdsp_fixed.c
@@ -34,38 +34,47 @@
 static SoftFloat sbr_sum_square_c(int (*x)[2], int n)
 {
     SoftFloat ret;
-    uint64_t accu = 0, round;
-    int i, nz;
+    uint64_t accu, round;
+    uint64_t accu0 = 0, accu1 = 0, accu2 = 0, accu3 = 0;
+    int i, nz, nz0;
     unsigned u;
 
     for (i = 0; i < n; i += 2) {
         // Larger values are inavlid and could cause overflows of accu.
-        av_assert2(FFABS(x[i + 0][0]) >> 29 == 0);
-        accu += (int64_t)x[i + 0][0] * x[i + 0][0];
-        av_assert2(FFABS(x[i + 0][1]) >> 29 == 0);
-        accu += (int64_t)x[i + 0][1] * x[i + 0][1];
-        av_assert2(FFABS(x[i + 1][0]) >> 29 == 0);
-        accu += (int64_t)x[i + 1][0] * x[i + 1][0];
-        av_assert2(FFABS(x[i + 1][1]) >> 29 == 0);
-        accu += (int64_t)x[i + 1][1] * x[i + 1][1];
+        av_assert2(FFABS(x[i + 0][0]) >> 30 == 0);
+        accu0 += (int64_t)x[i + 0][0] * x[i + 0][0];
+        av_assert2(FFABS(x[i + 0][1]) >> 30 == 0);
+        accu1 += (int64_t)x[i + 0][1] * x[i + 0][1];
+        av_assert2(FFABS(x[i + 1][0]) >> 30 == 0);
+        accu2 += (int64_t)x[i + 1][0] * x[i + 1][0];
+        av_assert2(FFABS(x[i + 1][1]) >> 30 == 0);
+        accu3 += (int64_t)x[i + 1][1] * x[i + 1][1];
     }
 
+    nz0 = 15;
+    while ((accu0|accu1|accu2|accu3) >> 62) {
+        accu0 >>= 1;
+        accu1 >>= 1;
+        accu2 >>= 1;
+        accu3 >>= 1;
+        nz0 --;
+    }
+    accu = accu0 + accu1 + accu2 + accu3;
+
     u = accu >> 32;
-    if (u == 0) {
-        nz = 1;
-    } else {
-        nz = -1;
+    if (u) {
+        nz = 33;
         while (u < 0x80000000U) {
             u <<= 1;
-            nz++;
+            nz--;
         }
-        nz = 32 - nz;
-    }
+    } else
+        nz = 1;
 
     round = 1ULL << (nz-1);
     u = ((accu + round) >> nz);
     u >>= 1;
-    ret = av_int2sf(u, 15 - nz);
+    ret = av_int2sf(u, nz0 - nz);
 
     return ret;
 }