avfilter/vf_lut3d: fix sanitizef INF handling

Signed-off-by: Anton Khirnov <anton@khirnov.net>

diff --git a/libavfilter/vf_lut3d.c b/libavfilter/vf_lut3d.c
index 988f6c8b551ede23e4bac43761084a296bf1f44a..172d6df0c8a3d3839120ac363bb4efc2bc62fadc 100644 (file)
--- a/libavfilter/vf_lut3d.c
+++ b/libavfilter/vf_lut3d.c
@@ -107,7 +107,7 @@ typedef struct ThreadData {
 
 #define EXPONENT_MASK 0x7F800000
 #define MANTISSA_MASK 0x007FFFFF
-#define SIGN_MASK     0x7FFFFFFF
+#define SIGN_MASK     0x80000000
 
 static inline float sanitizef(float f)
 {
@@ -120,7 +120,7 @@ static inline float sanitizef(float f)
             return 0.0f;
         } else if (t.i & SIGN_MASK) {
             // -INF
-            return FLT_MIN;
+            return -FLT_MAX;
         } else {
             // +INF
             return FLT_MAX;