fft: avoid a signed overflow

[ffmpeg] / libavcodec / pnm.c

diff --git a/libavcodec/pnm.c b/libavcodec/pnm.c
index 4fd9c61a24e69647a45dbcd004f714ca5d2505f8..54b55cfa5319f3815cab47e1fa5f5a72b51060b4 100644 (file)
--- a/libavcodec/pnm.c
+++ b/libavcodec/pnm.c
@@ -2,23 +2,24 @@
  * PNM image format
  * Copyright (c) 2002, 2003 Fabrice Bellard
  *
- * This file is part of FFmpeg.
+ * This file is part of Libav.
  *
- * FFmpeg is free software; you can redistribute it and/or
+ * Libav is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  *
- * FFmpeg is distributed in the hope that it will be useful,
+ * Libav is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  *
  * You should have received a copy of the GNU Lesser General Public
- * License along with FFmpeg; if not, write to the Free Software
+ * License along with Libav; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
 
+#include "libavutil/imgutils.h"
 #include "avcodec.h"
 #include "pnm.h"
 
@@ -59,16 +60,20 @@ int ff_pnm_decode_header(AVCodecContext *avctx, PNMContext * const s)
     int h, w, depth, maxval;
 
     pnm_get(s, buf1, sizeof(buf1));
-    if (!strcmp(buf1, "P4")) {
+    s->type= buf1[1]-'0';
+    if(buf1[0] != 'P')
+        return -1;
+
+    if (s->type==1 || s->type==4) {
         avctx->pix_fmt = PIX_FMT_MONOWHITE;
-    } else if (!strcmp(buf1, "P5")) {
+    } else if (s->type==2 || s->type==5) {
         if (avctx->codec_id == CODEC_ID_PGMYUV)
             avctx->pix_fmt = PIX_FMT_YUV420P;
         else
             avctx->pix_fmt = PIX_FMT_GRAY8;
-    } else if (!strcmp(buf1, "P6")) {
+    } else if (s->type==3 || s->type==6) {
         avctx->pix_fmt = PIX_FMT_RGB24;
-    } else if (!strcmp(buf1, "P7")) {
+    } else if (s->type==7) {
         w      = -1;
         h      = -1;
         maxval = -1;
@@ -97,7 +102,7 @@ int ff_pnm_decode_header(AVCodecContext *avctx, PNMContext * const s)
             }
         }
         /* check that all tags are present */
-        if (w <= 0 || h <= 0 || maxval <= 0 || depth <= 0 || tuple_type[0] == '\0' || avcodec_check_dimensions(avctx, w, h))
+        if (w <= 0 || h <= 0 || maxval <= 0 || depth <= 0 || tuple_type[0] == '\0' || av_image_check_size(w, h, 0, avctx))
             return -1;
 
         avctx->width  = w;
@@ -130,11 +135,15 @@ int ff_pnm_decode_header(AVCodecContext *avctx, PNMContext * const s)
         return -1;
     pnm_get(s, buf1, sizeof(buf1));
     avctx->height = atoi(buf1);
-    if(avcodec_check_dimensions(avctx, avctx->width, avctx->height))
+    if(av_image_check_size(avctx->width, avctx->height, 0, avctx))
         return -1;
     if (avctx->pix_fmt != PIX_FMT_MONOWHITE) {
         pnm_get(s, buf1, sizeof(buf1));
         s->maxval = atoi(buf1);
+        if (s->maxval <= 0) {
+            av_log(avctx, AV_LOG_ERROR, "Invalid maxval: %d\n", s->maxval);
+            s->maxval = 255;
+        }
         if (s->maxval >= 256) {
             if (avctx->pix_fmt == PIX_FMT_GRAY8) {
                 avctx->pix_fmt = PIX_FMT_GRAY16BE;
@@ -149,7 +158,8 @@ int ff_pnm_decode_header(AVCodecContext *avctx, PNMContext * const s)
                 return -1;
             }
         }
-    }
+    }else
+        s->maxval=1;
     /* more check if YUV420 */
     if (avctx->pix_fmt == PIX_FMT_YUV420P) {
         if ((avctx->width & 1) != 0)