avfilter/vf_eq: Fix clipping code

Found-by: Christophe Gisquet <christophe.gisquet@gmail.com>
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

diff --git a/libavfilter/vf_eq.c b/libavfilter/vf_eq.c
index 0c2f0cef4ef04fc53ac4e13063b5c143ea777576..7bbbaee2c5bd245886a0ee173c3583ccb83ea4fd 100644 (file)
--- a/libavfilter/vf_eq.c
+++ b/libavfilter/vf_eq.c
@@ -91,7 +91,7 @@ static void process_c(EQParameters *param, uint8_t *dst, int dst_stride,
         for (x = 0; x < w; x++) {
             pel = ((src[y * src_stride + x] * contrast) >> 12) + brightness;
 
-            if (pel & 768)
+            if (pel & ~255)
                 pel = (-pel) >> 31;
 
             dst[y * dst_stride + x] = pel;

diff --git a/libavfilter/x86/vf_eq.c b/libavfilter/x86/vf_eq.c
index cfcc18f495c0a96894d789ac7d60de823e53e7e5..2e42cdf5ba12b205098190d79e420048e4cd412f 100644 (file)
--- a/libavfilter/x86/vf_eq.c
+++ b/libavfilter/x86/vf_eq.c
@@ -72,7 +72,7 @@ static void process_MMX(EQParameters *param, uint8_t *dst, int dst_stride,
 
                 for (i = w&7; i; i--) {
                         pel = ((*src++ * contrast) >> 12) + brightness;
-                        if (pel & 768)
+                        if (pel & ~255)
                             pel = (-pel) >> 31;
                         *dst++ = pel;
                 }