Merge commit '8d929afd256069aa881f2bf58ef9f0ffce2d6b7e'

[ffmpeg] / libavcodec / h264_mp4toannexb_bsf.c

diff --git a/libavcodec/h264_mp4toannexb_bsf.c b/libavcodec/h264_mp4toannexb_bsf.c
index 86b948a31740085e1490d778653568d5566935b5..a28859c9f571e7f47fc00d979ff36cbda71c6441 100644 (file)
--- a/libavcodec/h264_mp4toannexb_bsf.c
+++ b/libavcodec/h264_mp4toannexb_bsf.c
@@ -2,20 +2,20 @@
  * H.264 MP4 to Annex B byte stream format filter
  * Copyright (c) 2007 Benoit Fouet <benoit.fouet@free.fr>
  *
- * This file is part of Libav.
+ * This file is part of FFmpeg.
  *
- * Libav is free software; you can redistribute it and/or
+ * FFmpeg is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
  * License as published by the Free Software Foundation; either
  * version 2.1 of the License, or (at your option) any later version.
  *
- * Libav is distributed in the hope that it will be useful,
+ * FFmpeg is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  * Lesser General Public License for more details.
  *
  * You should have received a copy of the GNU Lesser General Public
- * License along with Libav; if not, write to the Free Software
+ * License along with FFmpeg; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
 
@@ -68,17 +68,10 @@ static int h264_extradata_to_annexb(AVCodecContext *avctx, const int padding)
     static const uint8_t nalu_header[4] = { 0, 0, 0, 1 };
     int length_size = (*extradata++ & 0x3) + 1; // retrieve length coded size
 
-    if (length_size == 3)
-        return AVERROR(EINVAL);
-
     /* retrieve sps and pps unit(s) */
     unit_nb = *extradata++ & 0x1f; /* number of sps unit(s) */
     if (!unit_nb) {
-        unit_nb = *extradata++; /* number of pps unit(s) */
-        sps_done++;
-
-        if (unit_nb)
-            pps_seen = 1;
+        goto pps;
     } else {
         sps_seen = 1;
     }
@@ -103,7 +96,7 @@ static int h264_extradata_to_annexb(AVCodecContext *avctx, const int padding)
         memcpy(out + total_size - unit_size - 4, nalu_header, 4);
         memcpy(out + total_size - unit_size, extradata + 2, unit_size);
         extradata += 2 + unit_size;
-
+pps:
         if (!unit_nb && !sps_done++) {
             unit_nb = *extradata++; /* number of pps unit(s) */
             if (unit_nb)
@@ -138,6 +131,7 @@ static int h264_mp4toannexb_filter(AVBitStreamFilterContext *bsfc,
                                    int keyframe)
 {
     H264BSFContext *ctx = bsfc->priv_data;
+    int i;
     uint8_t unit_type;
     int32_t nal_size;
     uint32_t cumul_size    = 0;
@@ -164,15 +158,12 @@ static int h264_mp4toannexb_filter(AVBitStreamFilterContext *bsfc,
     *poutbuf_size = 0;
     *poutbuf      = NULL;
     do {
+        ret= AVERROR(EINVAL);
         if (buf + ctx->length_size > buf_end)
             goto fail;
 
-        if (ctx->length_size == 1) {
-            nal_size = buf[0];
-        } else if (ctx->length_size == 2) {
-            nal_size = AV_RB16(buf);
-        } else
-            nal_size = AV_RB32(buf);
+        for (nal_size = 0, i = 0; i<ctx->length_size; i++)
+            nal_size = (nal_size << 8) | buf[i];
 
         buf      += ctx->length_size;
         unit_type = *buf & 0x1f;
@@ -182,14 +173,14 @@ static int h264_mp4toannexb_filter(AVBitStreamFilterContext *bsfc,
 
         /* prepend only to the first type 5 NAL unit of an IDR picture */
         if (ctx->first_idr && unit_type == 5) {
-            if (alloc_and_copy(poutbuf, poutbuf_size,
+            if ((ret=alloc_and_copy(poutbuf, poutbuf_size,
                                avctx->extradata, avctx->extradata_size,
-                               buf, nal_size) < 0)
+                               buf, nal_size)) < 0)
                 goto fail;
             ctx->first_idr = 0;
         } else {
-            if (alloc_and_copy(poutbuf, poutbuf_size,
-                               NULL, 0, buf, nal_size) < 0)
+            if ((ret=alloc_and_copy(poutbuf, poutbuf_size,
+                               NULL, 0, buf, nal_size)) < 0)
                 goto fail;
             if (!ctx->first_idr && unit_type == 1)
                 ctx->first_idr = 1;
@@ -204,7 +195,7 @@ static int h264_mp4toannexb_filter(AVBitStreamFilterContext *bsfc,
 fail:
     av_freep(poutbuf);
     *poutbuf_size = 0;
-    return AVERROR(EINVAL);
+    return ret;
 }
 
 AVBitStreamFilter ff_h264_mp4toannexb_bsf = {