]> git.sesse.net Git - ffmpeg/blobdiff - libavcodec/snowdec.c
snowdec: increase stack size
[ffmpeg] / libavcodec / snowdec.c
index f622252c690690d55b8aa009e24f650c73d2fc0a..0a7f9df1461fe5b6ed633afdbf721dbf024aee25 100644 (file)
@@ -41,9 +41,10 @@ static av_always_inline void predict_slice_buffered(SnowContext *s, slice_buffer
     const int mb_h= s->b_height << s->block_max_depth;
     int x, y, mb_x;
     int block_size = MB_SIZE >> s->block_max_depth;
-    int block_w    = plane_index ? block_size/2 : block_size;
-    const uint8_t *obmc  = plane_index ? ff_obmc_tab[s->block_max_depth+1] : ff_obmc_tab[s->block_max_depth];
-    int obmc_stride= plane_index ? block_size : 2*block_size;
+    int block_w    = plane_index ? block_size>>s->chroma_h_shift : block_size;
+    int block_h    = plane_index ? block_size>>s->chroma_v_shift : block_size;
+    const uint8_t *obmc  = plane_index ? ff_obmc_tab[s->block_max_depth+s->chroma_h_shift] : ff_obmc_tab[s->block_max_depth];
+    int obmc_stride= plane_index ? (2*block_size)>>s->chroma_h_shift : 2*block_size;
     int ref_stride= s->current_picture.linesize[plane_index];
     uint8_t *dst8= s->current_picture.data[plane_index];
     int w= p->width;
@@ -54,7 +55,7 @@ static av_always_inline void predict_slice_buffered(SnowContext *s, slice_buffer
             return;
 
         if(add){
-            for(y=block_w*mb_y; y<FFMIN(h,block_w*(mb_y+1)); y++){
+            for(y=block_h*mb_y; y<FFMIN(h,block_h*(mb_y+1)); y++){
 //                DWTELEM * line = slice_buffer_get_line(sb, y);
                 IDWTELEM * line = sb->line[y];
                 for(x=0; x<w; x++){
@@ -66,7 +67,7 @@ static av_always_inline void predict_slice_buffered(SnowContext *s, slice_buffer
                 }
             }
         }else{
-            for(y=block_w*mb_y; y<FFMIN(h,block_w*(mb_y+1)); y++){
+            for(y=block_h*mb_y; y<FFMIN(h,block_h*(mb_y+1)); y++){
 //                DWTELEM * line = slice_buffer_get_line(sb, y);
                 IDWTELEM * line = sb->line[y];
                 for(x=0; x<w; x++){
@@ -82,8 +83,8 @@ static av_always_inline void predict_slice_buffered(SnowContext *s, slice_buffer
     for(mb_x=0; mb_x<=mb_w; mb_x++){
         add_yblock(s, 1, sb, old_buffer, dst8, obmc,
                    block_w*mb_x - block_w/2,
-                   block_w*mb_y - block_w/2,
-                   block_w, block_w,
+                   block_h*mb_y - block_h/2,
+                   block_w, block_h,
                    w, h,
                    w, ref_stride, obmc_stride,
                    mb_x - 1, mb_y - 1,
@@ -289,6 +290,20 @@ static int decode_header(SnowContext *s){
         s->colorspace_type= get_symbol(&s->c, s->header_state, 0);
         s->chroma_h_shift= get_symbol(&s->c, s->header_state, 0);
         s->chroma_v_shift= get_symbol(&s->c, s->header_state, 0);
+
+        if(s->chroma_h_shift == 1 && s->chroma_v_shift==1){
+            s->avctx->pix_fmt= PIX_FMT_YUV420P;
+        }else if(s->chroma_h_shift == 0 && s->chroma_v_shift==0){
+            s->avctx->pix_fmt= PIX_FMT_YUV444P;
+        }else if(s->chroma_h_shift == 2 && s->chroma_v_shift==2){
+            s->avctx->pix_fmt= PIX_FMT_YUV410P;
+        } else {
+            av_log(s, AV_LOG_ERROR, "unsupported color subsample mode %d %d\n", s->chroma_h_shift, s->chroma_v_shift);
+            s->chroma_h_shift = s->chroma_v_shift = 1;
+            s->avctx->pix_fmt= PIX_FMT_YUV420P;
+            return AVERROR_INVALIDDATA;
+        }
+
         s->spatial_scalability= get_rac(&s->c, s->header_state);
 //        s->rate_scalability= get_rac(&s->c, s->header_state);
         GET_S(s->max_ref_frames, tmp < (unsigned)MAX_REF_FRAMES)
@@ -334,10 +349,6 @@ static int decode_header(SnowContext *s){
         return -1;
     }
 
-    if (s->chroma_h_shift != 1 || s->chroma_v_shift != 1) {
-        av_log(s->avctx, AV_LOG_ERROR, "Invalid chroma shift\n");
-        return AVERROR_PATCHWELCOME;
-    }
 
     s->qlog           += get_symbol(&s->c, s->header_state, 1);
     s->mv_scale       += get_symbol(&s->c, s->header_state, 1);
@@ -354,9 +365,12 @@ static int decode_header(SnowContext *s){
 
 static av_cold int decode_init(AVCodecContext *avctx)
 {
-    avctx->pix_fmt= PIX_FMT_YUV420P;
+    int ret;
 
-    ff_snow_common_init(avctx);
+    if ((ret = ff_snow_common_init(avctx)) < 0) {
+        ff_snow_common_end(avctx->priv_data);
+        return ret;
+    }
 
     return 0;
 }
@@ -392,13 +406,14 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac
     s->current_picture.pict_type= AV_PICTURE_TYPE_I; //FIXME I vs. P
     if(decode_header(s)<0)
         return -1;
-    ff_snow_common_init_after_header(avctx);
+    if ((res=ff_snow_common_init_after_header(avctx)) < 0)
+        return res;
 
     // realloc slice buffer for the case that spatial_decomposition_count changed
     ff_slice_buffer_destroy(&s->sb);
     if ((res = ff_slice_buffer_init(&s->sb, s->plane[0].height,
                                     (MB_SIZE >> s->block_max_depth) +
-                                    s->spatial_decomposition_count * 8 + 1,
+                                    s->spatial_decomposition_count * 11 + 1,
                                     s->plane[0].width,
                                     s->spatial_idwt_buffer)) < 0)
         return res;
@@ -452,7 +467,8 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac
         {
         const int mb_h= s->b_height << s->block_max_depth;
         const int block_size = MB_SIZE >> s->block_max_depth;
-        const int block_w    = plane_index ? block_size/2 : block_size;
+        const int block_w    = plane_index ? block_size>>s->chroma_h_shift : block_size;
+        const int block_h    = plane_index ? block_size>>s->chroma_v_shift : block_size;
         int mb_y;
         DWTCompose cs[MAX_DECOMPOSITIONS];
         int yd=0, yq=0;
@@ -462,11 +478,12 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac
         ff_spatial_idwt_buffered_init(cs, &s->sb, w, h, 1, s->spatial_decomposition_type, s->spatial_decomposition_count);
         for(mb_y=0; mb_y<=mb_h; mb_y++){
 
-            int slice_starty = block_w*mb_y;
-            int slice_h = block_w*(mb_y+1);
+            int slice_starty = block_h*mb_y;
+            int slice_h = block_h*(mb_y+1);
+
             if (!(s->keyframe || s->avctx->debug&512)){
-                slice_starty = FFMAX(0, slice_starty - (block_w >> 1));
-                slice_h -= (block_w >> 1);
+                slice_starty = FFMAX(0, slice_starty - (block_h >> 1));
+                slice_h -= (block_h >> 1);
             }
 
             for(level=0; level<s->spatial_decomposition_count; level++){
@@ -477,11 +494,11 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac
                     int our_mb_start = mb_y;
                     int our_mb_end = (mb_y + 1);
                     const int extra= 3;
-                    start_y = (mb_y ? ((block_w * our_mb_start) >> (s->spatial_decomposition_count - level)) + s->spatial_decomposition_count - level + extra: 0);
-                    end_y = (((block_w * our_mb_end) >> (s->spatial_decomposition_count - level)) + s->spatial_decomposition_count - level + extra);
+                    start_y = (mb_y ? ((block_h * our_mb_start) >> (s->spatial_decomposition_count - level)) + s->spatial_decomposition_count - level + extra: 0);
+                    end_y = (((block_h * our_mb_end) >> (s->spatial_decomposition_count - level)) + s->spatial_decomposition_count - level + extra);
                     if (!(s->keyframe || s->avctx->debug&512)){
-                        start_y = FFMAX(0, start_y - (block_w >> (1+s->spatial_decomposition_count - level)));
-                        end_y = FFMAX(0, end_y - (block_w >> (1+s->spatial_decomposition_count - level)));
+                        start_y = FFMAX(0, start_y - (block_h >> (1+s->spatial_decomposition_count - level)));
+                        end_y = FFMAX(0, end_y - (block_h >> (1+s->spatial_decomposition_count - level)));
                     }
                     start_y = FFMIN(b->height, start_y);
                     end_y = FFMIN(b->height, end_y);
@@ -502,7 +519,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac
             }
 
             for(; yd<slice_h; yd+=4){
-                ff_spatial_idwt_buffered_slice(&s->dwt, cs, &s->sb, w, h, 1, s->spatial_decomposition_type, s->spatial_decomposition_count, yd);
+                ff_spatial_idwt_buffered_slice(&s->dwt, cs, &s->sb, s->temp_idwt_buffer, w, h, 1, s->spatial_decomposition_type, s->spatial_decomposition_count, yd);
             }
 
             if(s->qlog == LOSSLESS_QLOG){
@@ -558,7 +575,7 @@ static av_cold int decode_end(AVCodecContext *avctx)
 AVCodec ff_snow_decoder = {
     .name           = "snow",
     .type           = AVMEDIA_TYPE_VIDEO,
-    .id             = CODEC_ID_SNOW,
+    .id             = AV_CODEC_ID_SNOW,
     .priv_data_size = sizeof(SnowContext),
     .init           = decode_init,
     .close          = decode_end,