avcodec/ffv1enc: Fix out-of-bounds-array access

libavcodec/ffv1enc.c accessed an array of uint8_t [32] via array[0][j]
in order to loop over all the uint8_t in this array of arrays. Of course
this implied an out-of-bounds access for array[0] and UBSan complained
about this. So replace this with nested loops; furthermore, factor this
out into a function of its own to easily break out of the nested loops.

This affected the FATE-tests vsynth1-ffv1, vsynth1-ffv1-v3-yuv420p,
vsynth1-ffv1-v3-yuv422p10, vsynth1-ffv1-v3-yuv444p16,
vsynth1-ffv1-v3-bgr0, vsynth1-ffv1-ffv1-v3-rgb48 as well as the
corresponding vsynth2-*, vsynth3-* and the vsynth_lena-* tests.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@gmail.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

diff --git a/libavcodec/ffv1enc.c b/libavcodec/ffv1enc.c
index 796d81f7c6541ef88b6426b487938d533166aa4f..c521b7d445f50601f69f4be3cded48abf879f29f 100644 (file)
--- a/libavcodec/ffv1enc.c
+++ b/libavcodec/ffv1enc.c
@@ -334,6 +334,18 @@ static void write_quant_tables(RangeCoder *c,
         write_quant_table(c, quant_table[i]);
 }
 
+static int contains_non_128(uint8_t (*initial_state)[CONTEXT_SIZE],
+                            int nb_contexts)
+{
+    if (!initial_state)
+        return 0;
+    for (int i = 0; i < nb_contexts; i++)
+        for (int j = 0; j < CONTEXT_SIZE; j++)
+            if (initial_state[i][j] != 128)
+                return 1;
+    return 0;
+}
+
 static void write_header(FFV1Context *f)
 {
     uint8_t state[CONTEXT_SIZE];
@@ -428,10 +440,7 @@ static int write_extradata(FFV1Context *f)
         write_quant_tables(c, f->quant_tables[i]);
 
     for (i = 0; i < f->quant_table_count; i++) {
-        for (j = 0; j < f->context_count[i] * CONTEXT_SIZE; j++)
-            if (f->initial_states[i] && f->initial_states[i][0][j] != 128)
-                break;
-        if (j < f->context_count[i] * CONTEXT_SIZE) {
+        if (contains_non_128(f->initial_states[i], f->context_count[i])) {
             put_rac(c, state, 1);
             for (j = 0; j < f->context_count[i]; j++)
                 for (k = 0; k < CONTEXT_SIZE; k++) {