j2kdec: Fix integer overflow leading to a segfault

Fixes Ticket776
Bug found by: Diana Elena Muscalu

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

diff --git a/libavcodec/j2k_dwt.c b/libavcodec/j2k_dwt.c
index ab7a1ab757a8be6bb56dff2e4dfd5588c307c7bb..48aa33735eb40bb6cb095eadc1c24668b3ea1bdf 100644 (file)
--- a/libavcodec/j2k_dwt.c
+++ b/libavcodec/j2k_dwt.c
@@ -321,7 +321,7 @@ int ff_j2k_dwt_init(DWTContext *s, uint16_t border[2][2], int decomp_levels, int
     int i, j, lev = decomp_levels, maxlen,
         b[2][2];
 
-    if (decomp_levels >= FF_DWT_MAX_DECLVLS)
+    if ((unsigned)decomp_levels >= FF_DWT_MAX_DECLVLS)
         return AVERROR_INVALIDDATA;
     s->ndeclevels = decomp_levels;
     s->type = type;