]> git.sesse.net Git - ffmpeg/blob - libavcodec/cfhd.c
avcodec/diracdec: Fix integer overflow in divide3()
[ffmpeg] / libavcodec / cfhd.c
1 /*
2  * Copyright (c) 2015-2016 Kieran Kunhya <kieran@kunhya.com>
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20
21 /**
22  * @file
23  * CFHD Video Decoder
24  */
25
26 #include "libavutil/buffer.h"
27 #include "libavutil/common.h"
28 #include "libavutil/intreadwrite.h"
29 #include "libavutil/imgutils.h"
30 #include "libavutil/opt.h"
31
32 #include "avcodec.h"
33 #include "internal.h"
34 #include "bytestream.h"
35 #include "thread.h"
36 #include "cfhd.h"
37
38 #define SUBBAND_COUNT 10
39
40 static av_cold int cfhd_decode_init(AVCodecContext *avctx)
41 {
42     CFHDContext *s = avctx->priv_data;
43
44     avctx->bits_per_raw_sample = 10;
45     s->avctx                   = avctx;
46
47     return ff_cfhd_init_vlcs(s);
48 }
49
50 static void init_plane_defaults(CFHDContext *s)
51 {
52     s->subband_num        = 0;
53     s->level              = 0;
54     s->subband_num_actual = 0;
55 }
56
57 static void init_frame_defaults(CFHDContext *s)
58 {
59     s->coded_width       = 0;
60     s->coded_height      = 0;
61     s->bpc               = 10;
62     s->channel_cnt       = 4;
63     s->subband_cnt       = 10;
64     s->channel_num       = 0;
65     s->lowpass_precision = 16;
66     s->quantisation      = 1;
67     s->wavelet_depth     = 3;
68     s->pshift            = 1;
69     s->codebook          = 0;
70     init_plane_defaults(s);
71 }
72
73 /* TODO: merge with VLC tables or use LUT */
74 static inline int dequant_and_decompand(int level, int quantisation)
75 {
76     int64_t abslevel = abs(level);
77     return (abslevel + ((768 * abslevel * abslevel * abslevel) / (255 * 255 * 255))) * FFSIGN(level) * quantisation;
78 }
79
80 static inline void filter(int16_t *output, ptrdiff_t out_stride, int16_t *low, ptrdiff_t low_stride,
81                           int16_t *high, ptrdiff_t high_stride, int len, uint8_t clip)
82 {
83     int16_t tmp;
84
85     int i;
86     for (i = 0; i < len; i++) {
87         if (i == 0) {
88             tmp = (11*low[0*low_stride] - 4*low[1*low_stride] + low[2*low_stride] + 4) >> 3;
89             output[(2*i+0)*out_stride] = (tmp + high[0*high_stride]) >> 1;
90             if (clip)
91                 output[(2*i+0)*out_stride] = av_clip_uintp2_c(output[(2*i+0)*out_stride], clip);
92
93             tmp = ( 5*low[0*low_stride] + 4*low[1*low_stride] - low[2*low_stride] + 4) >> 3;
94             output[(2*i+1)*out_stride] = (tmp - high[0*high_stride]) >> 1;
95             if (clip)
96                 output[(2*i+1)*out_stride] = av_clip_uintp2_c(output[(2*i+1)*out_stride], clip);
97         } else if (i == len-1) {
98             tmp = ( 5*low[i*low_stride] + 4*low[(i-1)*low_stride] - low[(i-2)*low_stride] + 4) >> 3;
99             output[(2*i+0)*out_stride] = (tmp + high[i*high_stride]) >> 1;
100             if (clip)
101                 output[(2*i+0)*out_stride] = av_clip_uintp2_c(output[(2*i+0)*out_stride], clip);
102
103             tmp = (11*low[i*low_stride] - 4*low[(i-1)*low_stride] + low[(i-2)*low_stride] + 4) >> 3;
104             output[(2*i+1)*out_stride] = (tmp - high[i*high_stride]) >> 1;
105             if (clip)
106                 output[(2*i+1)*out_stride] = av_clip_uintp2_c(output[(2*i+1)*out_stride], clip);
107         } else {
108             tmp = (low[(i-1)*low_stride] - low[(i+1)*low_stride] + 4) >> 3;
109             output[(2*i+0)*out_stride] = (tmp + low[i*low_stride] + high[i*high_stride]) >> 1;
110             if (clip)
111                 output[(2*i+0)*out_stride] = av_clip_uintp2_c(output[(2*i+0)*out_stride], clip);
112
113             tmp = (low[(i+1)*low_stride] - low[(i-1)*low_stride] + 4) >> 3;
114             output[(2*i+1)*out_stride] = (tmp + low[i*low_stride] - high[i*high_stride]) >> 1;
115             if (clip)
116                 output[(2*i+1)*out_stride] = av_clip_uintp2_c(output[(2*i+1)*out_stride], clip);
117         }
118     }
119 }
120
121 static void horiz_filter(int16_t *output, int16_t *low, int16_t *high, int width)
122 {
123     filter(output, 1, low, 1, high, 1, width, 0);
124 }
125
126 static void horiz_filter_clip(int16_t *output, int16_t *low, int16_t *high, int width, uint8_t clip)
127 {
128     filter(output, 1, low, 1, high, 1, width, clip);
129 }
130
131 static void vert_filter(int16_t *output, int out_stride, int16_t *low, int low_stride,
132                         int16_t *high, int high_stride, int len)
133 {
134     filter(output, out_stride, low, low_stride, high, high_stride, len, 0);
135 }
136
137 static void free_buffers(AVCodecContext *avctx)
138 {
139     CFHDContext *s = avctx->priv_data;
140     int i, j;
141
142     for (i = 0; i < 4; i++) {
143         av_freep(&s->plane[i].idwt_buf);
144         av_freep(&s->plane[i].idwt_tmp);
145
146         for (j = 0; j < 9; j++)
147             s->plane[i].subband[j] = NULL;
148
149         for (j = 0; j < 8; j++)
150             s->plane[i].l_h[j] = NULL;
151     }
152     s->a_height = 0;
153     s->a_width  = 0;
154 }
155
156 static int alloc_buffers(AVCodecContext *avctx)
157 {
158     CFHDContext *s = avctx->priv_data;
159     int i, j, k, ret, planes;
160
161     if ((ret = ff_set_dimensions(avctx, s->coded_width, s->coded_height)) < 0)
162         return ret;
163     avctx->pix_fmt = s->coded_format;
164
165     avcodec_get_chroma_sub_sample(avctx->pix_fmt, &s->chroma_x_shift, &s->chroma_y_shift);
166     planes = av_pix_fmt_count_planes(avctx->pix_fmt);
167
168     for (i = 0; i < planes; i++) {
169         int width = i ? avctx->width >> s->chroma_x_shift : avctx->width;
170         int height = i ? avctx->height >> s->chroma_y_shift : avctx->height;
171         int stride = FFALIGN(width / 8, 8) * 8;
172         int w8, h8, w4, h4, w2, h2;
173         height = FFALIGN(height / 8, 2) * 8;
174         s->plane[i].width = width;
175         s->plane[i].height = height;
176         s->plane[i].stride = stride;
177
178         w8 = FFALIGN(s->plane[i].width / 8, 8);
179         h8 = FFALIGN(s->plane[i].height / 8, 2);
180         w4 = w8 * 2;
181         h4 = h8 * 2;
182         w2 = w4 * 2;
183         h2 = h4 * 2;
184
185         s->plane[i].idwt_buf = av_mallocz_array(height * stride, sizeof(*s->plane[i].idwt_buf));
186         s->plane[i].idwt_tmp = av_malloc_array(height * stride, sizeof(*s->plane[i].idwt_tmp));
187         if (!s->plane[i].idwt_buf || !s->plane[i].idwt_tmp) {
188             return AVERROR(ENOMEM);
189         }
190
191         s->plane[i].subband[0] = s->plane[i].idwt_buf;
192         s->plane[i].subband[1] = s->plane[i].idwt_buf + 2 * w8 * h8;
193         s->plane[i].subband[2] = s->plane[i].idwt_buf + 1 * w8 * h8;
194         s->plane[i].subband[3] = s->plane[i].idwt_buf + 3 * w8 * h8;
195         s->plane[i].subband[4] = s->plane[i].idwt_buf + 2 * w4 * h4;
196         s->plane[i].subband[5] = s->plane[i].idwt_buf + 1 * w4 * h4;
197         s->plane[i].subband[6] = s->plane[i].idwt_buf + 3 * w4 * h4;
198         s->plane[i].subband[7] = s->plane[i].idwt_buf + 2 * w2 * h2;
199         s->plane[i].subband[8] = s->plane[i].idwt_buf + 1 * w2 * h2;
200         s->plane[i].subband[9] = s->plane[i].idwt_buf + 3 * w2 * h2;
201
202         for (j = 0; j < DWT_LEVELS; j++) {
203             for(k = 0; k < 4; k++) {
204                 s->plane[i].band[j][k].a_width  = w8 << j;
205                 s->plane[i].band[j][k].a_height = h8 << j;
206             }
207         }
208
209         /* ll2 and ll1 commented out because they are done in-place */
210         s->plane[i].l_h[0] = s->plane[i].idwt_tmp;
211         s->plane[i].l_h[1] = s->plane[i].idwt_tmp + 2 * w8 * h8;
212         //s->plane[i].l_h[2] = ll2;
213         s->plane[i].l_h[3] = s->plane[i].idwt_tmp;
214         s->plane[i].l_h[4] = s->plane[i].idwt_tmp + 2 * w4 * h4;
215         //s->plane[i].l_h[5] = ll1;
216         s->plane[i].l_h[6] = s->plane[i].idwt_tmp;
217         s->plane[i].l_h[7] = s->plane[i].idwt_tmp + 2 * w2 * h2;
218     }
219
220     s->a_height = s->coded_height;
221     s->a_width  = s->coded_width;
222     s->a_format = s->coded_format;
223
224     return 0;
225 }
226
227 static int cfhd_decode(AVCodecContext *avctx, void *data, int *got_frame,
228                        AVPacket *avpkt)
229 {
230     CFHDContext *s = avctx->priv_data;
231     GetByteContext gb;
232     ThreadFrame frame = { .f = data };
233     AVFrame *pic = data;
234     int ret = 0, i, j, planes, plane, got_buffer = 0;
235     int16_t *coeff_data;
236
237     s->coded_format = AV_PIX_FMT_YUV422P10;
238     init_frame_defaults(s);
239     planes = av_pix_fmt_count_planes(s->coded_format);
240
241     bytestream2_init(&gb, avpkt->data, avpkt->size);
242
243     while (bytestream2_get_bytes_left(&gb) > 4) {
244         /* Bit weird but implement the tag parsing as the spec says */
245         uint16_t tagu   = bytestream2_get_be16(&gb);
246         int16_t tag     = (int16_t)tagu;
247         int8_t tag8     = (int8_t)(tagu >> 8);
248         uint16_t abstag = abs(tag);
249         int8_t abs_tag8 = abs(tag8);
250         uint16_t data   = bytestream2_get_be16(&gb);
251         if (abs_tag8 >= 0x60 && abs_tag8 <= 0x6f) {
252             av_log(avctx, AV_LOG_DEBUG, "large len %x\n", ((tagu & 0xff) << 16) | data);
253         } else if (tag == 20) {
254             av_log(avctx, AV_LOG_DEBUG, "Width %"PRIu16"\n", data);
255             s->coded_width = data;
256         } else if (tag == 21) {
257             av_log(avctx, AV_LOG_DEBUG, "Height %"PRIu16"\n", data);
258             s->coded_height = data;
259         } else if (tag == 101) {
260             av_log(avctx, AV_LOG_DEBUG, "Bits per component: %"PRIu16"\n", data);
261             if (data < 1 || data > 31) {
262                 av_log(avctx, AV_LOG_ERROR, "Bits per component %d is invalid\n", data);
263                 ret = AVERROR(EINVAL);
264                 break;
265             }
266             s->bpc = data;
267         } else if (tag == 12) {
268             av_log(avctx, AV_LOG_DEBUG, "Channel Count: %"PRIu16"\n", data);
269             s->channel_cnt = data;
270             if (data > 4) {
271                 av_log(avctx, AV_LOG_ERROR, "Channel Count of %"PRIu16" is unsupported\n", data);
272                 ret = AVERROR_PATCHWELCOME;
273                 break;
274             }
275         } else if (tag == 14) {
276             av_log(avctx, AV_LOG_DEBUG, "Subband Count: %"PRIu16"\n", data);
277             if (data != SUBBAND_COUNT) {
278                 av_log(avctx, AV_LOG_ERROR, "Subband Count of %"PRIu16" is unsupported\n", data);
279                 ret = AVERROR_PATCHWELCOME;
280                 break;
281             }
282         } else if (tag == 62) {
283             s->channel_num = data;
284             av_log(avctx, AV_LOG_DEBUG, "Channel number %"PRIu16"\n", data);
285             if (s->channel_num >= planes) {
286                 av_log(avctx, AV_LOG_ERROR, "Invalid channel number\n");
287                 ret = AVERROR(EINVAL);
288                 break;
289             }
290             init_plane_defaults(s);
291         } else if (tag == 48) {
292             if (s->subband_num != 0 && data == 1)  // hack
293                 s->level++;
294             av_log(avctx, AV_LOG_DEBUG, "Subband number %"PRIu16"\n", data);
295             s->subband_num = data;
296             if (s->level >= DWT_LEVELS) {
297                 av_log(avctx, AV_LOG_ERROR, "Invalid level\n");
298                 ret = AVERROR(EINVAL);
299                 break;
300             }
301             if (s->subband_num > 3) {
302                 av_log(avctx, AV_LOG_ERROR, "Invalid subband number\n");
303                 ret = AVERROR(EINVAL);
304                 break;
305             }
306         } else if (tag == 51) {
307             av_log(avctx, AV_LOG_DEBUG, "Subband number actual %"PRIu16"\n", data);
308             s->subband_num_actual = data;
309             if (s->subband_num_actual >= 10) {
310                 av_log(avctx, AV_LOG_ERROR, "Invalid subband number actual\n");
311                 ret = AVERROR(EINVAL);
312                 break;
313             }
314         } else if (tag == 35)
315             av_log(avctx, AV_LOG_DEBUG, "Lowpass precision bits: %"PRIu16"\n", data);
316         else if (tag == 53) {
317             s->quantisation = data;
318             av_log(avctx, AV_LOG_DEBUG, "Quantisation: %"PRIu16"\n", data);
319         } else if (tag == 109) {
320             s->prescale_shift[0] = (data >> 0) & 0x7;
321             s->prescale_shift[1] = (data >> 3) & 0x7;
322             s->prescale_shift[2] = (data >> 6) & 0x7;
323             av_log(avctx, AV_LOG_DEBUG, "Prescale shift (VC-5): %x\n", data);
324         } else if (tag == 27) {
325             av_log(avctx, AV_LOG_DEBUG, "Lowpass width %"PRIu16"\n", data);
326             if (data < 3 || data > s->plane[s->channel_num].band[0][0].a_width) {
327                 av_log(avctx, AV_LOG_ERROR, "Invalid lowpass width\n");
328                 ret = AVERROR(EINVAL);
329                 break;
330             }
331             s->plane[s->channel_num].band[0][0].width  = data;
332             s->plane[s->channel_num].band[0][0].stride = data;
333         } else if (tag == 28) {
334             av_log(avctx, AV_LOG_DEBUG, "Lowpass height %"PRIu16"\n", data);
335             if (data < 3 || data > s->plane[s->channel_num].band[0][0].a_height) {
336                 av_log(avctx, AV_LOG_ERROR, "Invalid lowpass height\n");
337                 ret = AVERROR(EINVAL);
338                 break;
339             }
340             s->plane[s->channel_num].band[0][0].height = data;
341         } else if (tag == 1)
342             av_log(avctx, AV_LOG_DEBUG, "Sample type? %"PRIu16"\n", data);
343         else if (tag == 10) {
344             if (data != 0) {
345                 avpriv_report_missing_feature(avctx, "Transform type of %"PRIu16, data);
346                 ret = AVERROR_PATCHWELCOME;
347                 break;
348             }
349             av_log(avctx, AV_LOG_DEBUG, "Transform-type? %"PRIu16"\n", data);
350         } else if (abstag >= 0x4000 && abstag <= 0x40ff) {
351             av_log(avctx, AV_LOG_DEBUG, "Small chunk length %d %s\n", data * 4, tag < 0 ? "optional" : "required");
352             bytestream2_skipu(&gb, data * 4);
353         } else if (tag == 23) {
354             av_log(avctx, AV_LOG_DEBUG, "Skip frame\n");
355             avpriv_report_missing_feature(avctx, "Skip frame");
356             ret = AVERROR_PATCHWELCOME;
357             break;
358         } else if (tag == 2) {
359             av_log(avctx, AV_LOG_DEBUG, "tag=2 header - skipping %i tag/value pairs\n", data);
360             if (data > bytestream2_get_bytes_left(&gb) / 4) {
361                 av_log(avctx, AV_LOG_ERROR, "too many tag/value pairs (%d)\n", data);
362                 ret = AVERROR_INVALIDDATA;
363                 break;
364             }
365             for (i = 0; i < data; i++) {
366                 uint16_t tag2 = bytestream2_get_be16(&gb);
367                 uint16_t val2 = bytestream2_get_be16(&gb);
368                 av_log(avctx, AV_LOG_DEBUG, "Tag/Value = %x %x\n", tag2, val2);
369             }
370         } else if (tag == 41) {
371             av_log(avctx, AV_LOG_DEBUG, "Highpass width %i channel %i level %i subband %i\n", data, s->channel_num, s->level, s->subband_num);
372             if (data < 3) {
373                 av_log(avctx, AV_LOG_ERROR, "Invalid highpass width\n");
374                 ret = AVERROR(EINVAL);
375                 break;
376             }
377             s->plane[s->channel_num].band[s->level][s->subband_num].width  = data;
378             s->plane[s->channel_num].band[s->level][s->subband_num].stride = FFALIGN(data, 8);
379         } else if (tag == 42) {
380             av_log(avctx, AV_LOG_DEBUG, "Highpass height %i\n", data);
381             if (data < 3) {
382                 av_log(avctx, AV_LOG_ERROR, "Invalid highpass height\n");
383                 ret = AVERROR(EINVAL);
384                 break;
385             }
386             s->plane[s->channel_num].band[s->level][s->subband_num].height = data;
387         } else if (tag == 49) {
388             av_log(avctx, AV_LOG_DEBUG, "Highpass width2 %i\n", data);
389             if (data < 3) {
390                 av_log(avctx, AV_LOG_ERROR, "Invalid highpass width2\n");
391                 ret = AVERROR(EINVAL);
392                 break;
393             }
394             s->plane[s->channel_num].band[s->level][s->subband_num].width  = data;
395             s->plane[s->channel_num].band[s->level][s->subband_num].stride = FFALIGN(data, 8);
396         } else if (tag == 50) {
397             av_log(avctx, AV_LOG_DEBUG, "Highpass height2 %i\n", data);
398             if (data < 3) {
399                 av_log(avctx, AV_LOG_ERROR, "Invalid highpass height2\n");
400                 ret = AVERROR(EINVAL);
401                 break;
402             }
403             s->plane[s->channel_num].band[s->level][s->subband_num].height = data;
404         } else if (tag == 71) {
405             s->codebook = data;
406             av_log(avctx, AV_LOG_DEBUG, "Codebook %i\n", s->codebook);
407         } else if (tag == 72) {
408             s->codebook = data;
409             av_log(avctx, AV_LOG_DEBUG, "Other codebook? %i\n", s->codebook);
410         } else if (tag == 70) {
411             av_log(avctx, AV_LOG_DEBUG, "Subsampling or bit-depth flag? %i\n", data);
412             if (!(data == 10 || data == 12)) {
413                 av_log(avctx, AV_LOG_ERROR, "Invalid bits per channel\n");
414                 ret = AVERROR(EINVAL);
415                 break;
416             }
417             s->bpc = data;
418         } else if (tag == 84) {
419             av_log(avctx, AV_LOG_DEBUG, "Sample format? %i\n", data);
420             if (data == 1)
421                 s->coded_format = AV_PIX_FMT_YUV422P10;
422             else if (data == 3)
423                 s->coded_format = AV_PIX_FMT_GBRP12;
424             else if (data == 4)
425                 s->coded_format = AV_PIX_FMT_GBRAP12;
426             else {
427                 avpriv_report_missing_feature(avctx, "Sample format of %"PRIu16, data);
428                 ret = AVERROR_PATCHWELCOME;
429                 break;
430             }
431             planes = av_pix_fmt_count_planes(s->coded_format);
432         } else
433             av_log(avctx, AV_LOG_DEBUG,  "Unknown tag %i data %x\n", tag, data);
434
435         /* Some kind of end of header tag */
436         if (tag == 4 && data == 0x1a4a && s->coded_width && s->coded_height &&
437             s->coded_format != AV_PIX_FMT_NONE) {
438             if (s->a_width != s->coded_width || s->a_height != s->coded_height ||
439                 s->a_format != s->coded_format) {
440                 free_buffers(avctx);
441                 if ((ret = alloc_buffers(avctx)) < 0) {
442                     free_buffers(avctx);
443                     return ret;
444                 }
445             }
446             ret = ff_set_dimensions(avctx, s->coded_width, s->coded_height);
447             if (ret < 0)
448                 return ret;
449             frame.f->width =
450             frame.f->height = 0;
451
452             if ((ret = ff_thread_get_buffer(avctx, &frame, 0)) < 0)
453                 return ret;
454
455             s->coded_width = 0;
456             s->coded_height = 0;
457             s->coded_format = AV_PIX_FMT_NONE;
458             got_buffer = 1;
459         }
460         coeff_data = s->plane[s->channel_num].subband[s->subband_num_actual];
461
462         /* Lowpass coefficients */
463         if (tag == 4 && data == 0xf0f && s->a_width && s->a_height) {
464             int lowpass_height = s->plane[s->channel_num].band[0][0].height;
465             int lowpass_width  = s->plane[s->channel_num].band[0][0].width;
466             int lowpass_a_height = s->plane[s->channel_num].band[0][0].a_height;
467             int lowpass_a_width  = s->plane[s->channel_num].band[0][0].a_width;
468
469             if (!got_buffer) {
470                 av_log(avctx, AV_LOG_ERROR, "No end of header tag found\n");
471                 ret = AVERROR(EINVAL);
472                 goto end;
473             }
474
475             if (lowpass_height > lowpass_a_height || lowpass_width > lowpass_a_width ||
476                 lowpass_a_width * lowpass_a_height * sizeof(int16_t) > bytestream2_get_bytes_left(&gb)) {
477                 av_log(avctx, AV_LOG_ERROR, "Too many lowpass coefficients\n");
478                 ret = AVERROR(EINVAL);
479                 goto end;
480             }
481
482             av_log(avctx, AV_LOG_DEBUG, "Start of lowpass coeffs component %d height:%d, width:%d\n", s->channel_num, lowpass_height, lowpass_width);
483             for (i = 0; i < lowpass_height; i++) {
484                 for (j = 0; j < lowpass_width; j++)
485                     coeff_data[j] = bytestream2_get_be16u(&gb);
486
487                 coeff_data += lowpass_width;
488             }
489
490             /* Align to mod-4 position to continue reading tags */
491             bytestream2_seek(&gb, bytestream2_tell(&gb) & 3, SEEK_CUR);
492
493             /* Copy last line of coefficients if odd height */
494             if (lowpass_height & 1) {
495                 memcpy(&coeff_data[lowpass_height * lowpass_width],
496                        &coeff_data[(lowpass_height - 1) * lowpass_width],
497                        lowpass_width * sizeof(*coeff_data));
498             }
499
500             av_log(avctx, AV_LOG_DEBUG, "Lowpass coefficients %d\n", lowpass_width * lowpass_height);
501         }
502
503         if (tag == 55 && s->subband_num_actual != 255 && s->a_width && s->a_height) {
504             int highpass_height = s->plane[s->channel_num].band[s->level][s->subband_num].height;
505             int highpass_width  = s->plane[s->channel_num].band[s->level][s->subband_num].width;
506             int highpass_a_width = s->plane[s->channel_num].band[s->level][s->subband_num].a_width;
507             int highpass_a_height = s->plane[s->channel_num].band[s->level][s->subband_num].a_height;
508             int highpass_stride = s->plane[s->channel_num].band[s->level][s->subband_num].stride;
509             int expected;
510             int a_expected = highpass_a_height * highpass_a_width;
511             int level, run, coeff;
512             int count = 0, bytes;
513
514             if (!got_buffer) {
515                 av_log(avctx, AV_LOG_ERROR, "No end of header tag found\n");
516                 ret = AVERROR(EINVAL);
517                 goto end;
518             }
519
520             if (highpass_height > highpass_a_height || highpass_width > highpass_a_width || a_expected < highpass_height * (uint64_t)highpass_stride) {
521                 av_log(avctx, AV_LOG_ERROR, "Too many highpass coefficients\n");
522                 ret = AVERROR(EINVAL);
523                 goto end;
524             }
525             expected = highpass_height * highpass_stride;
526
527             av_log(avctx, AV_LOG_DEBUG, "Start subband coeffs plane %i level %i codebook %i expected %i\n", s->channel_num, s->level, s->codebook, expected);
528
529             init_get_bits(&s->gb, gb.buffer, bytestream2_get_bytes_left(&gb) * 8);
530             {
531                 OPEN_READER(re, &s->gb);
532                 if (!s->codebook) {
533                     while (1) {
534                         UPDATE_CACHE(re, &s->gb);
535                         GET_RL_VLC(level, run, re, &s->gb, s->table_9_rl_vlc,
536                                    VLC_BITS, 3, 1);
537
538                         /* escape */
539                         if (level == 64)
540                             break;
541
542                         count += run;
543
544                         if (count > expected)
545                             break;
546
547                         coeff = dequant_and_decompand(level, s->quantisation);
548                         for (i = 0; i < run; i++)
549                             *coeff_data++ = coeff;
550                     }
551                 } else {
552                     while (1) {
553                         UPDATE_CACHE(re, &s->gb);
554                         GET_RL_VLC(level, run, re, &s->gb, s->table_18_rl_vlc,
555                                    VLC_BITS, 3, 1);
556
557                         /* escape */
558                         if (level == 255 && run == 2)
559                             break;
560
561                         count += run;
562
563                         if (count > expected)
564                             break;
565
566                         coeff = dequant_and_decompand(level, s->quantisation);
567                         for (i = 0; i < run; i++)
568                             *coeff_data++ = coeff;
569                     }
570                 }
571                 CLOSE_READER(re, &s->gb);
572             }
573
574             if (count > expected) {
575                 av_log(avctx, AV_LOG_ERROR, "Escape codeword not found, probably corrupt data\n");
576                 ret = AVERROR(EINVAL);
577                 goto end;
578             }
579
580             bytes = FFALIGN(FF_CEIL_RSHIFT(get_bits_count(&s->gb), 3), 4);
581             if (bytes > bytestream2_get_bytes_left(&gb)) {
582                 av_log(avctx, AV_LOG_ERROR, "Bitstream overread error\n");
583                 ret = AVERROR(EINVAL);
584                 goto end;
585             } else
586                 bytestream2_seek(&gb, bytes, SEEK_CUR);
587
588             av_log(avctx, AV_LOG_DEBUG, "End subband coeffs %i extra %i\n", count, count - expected);
589             s->codebook = 0;
590
591             /* Copy last line of coefficients if odd height */
592             if (highpass_height & 1) {
593                 memcpy(&coeff_data[highpass_height * highpass_stride],
594                        &coeff_data[(highpass_height - 1) * highpass_stride],
595                        highpass_stride * sizeof(*coeff_data));
596             }
597         }
598     }
599
600     if (!s->a_width || !s->a_height || s->a_format == AV_PIX_FMT_NONE ||
601         s->coded_width || s->coded_height || s->coded_format != AV_PIX_FMT_NONE) {
602         av_log(avctx, AV_LOG_ERROR, "Invalid dimensions\n");
603         ret = AVERROR(EINVAL);
604         goto end;
605     }
606
607     if (!got_buffer) {
608         av_log(avctx, AV_LOG_ERROR, "No end of header tag found\n");
609         ret = AVERROR(EINVAL);
610         goto end;
611     }
612
613     planes = av_pix_fmt_count_planes(avctx->pix_fmt);
614     for (plane = 0; plane < planes && !ret; plane++) {
615         /* level 1 */
616         int lowpass_height  = s->plane[plane].band[0][0].height;
617         int lowpass_width   = s->plane[plane].band[0][0].width;
618         int highpass_stride = s->plane[plane].band[0][1].stride;
619         int act_plane = plane == 1 ? 2 : plane == 2 ? 1 : plane;
620         int16_t *low, *high, *output, *dst;
621
622         if (lowpass_height > s->plane[plane].band[0][0].a_height || lowpass_width > s->plane[plane].band[0][0].a_width ||
623             !highpass_stride || s->plane[plane].band[0][1].width > s->plane[plane].band[0][1].a_width) {
624             av_log(avctx, AV_LOG_ERROR, "Invalid plane dimensions\n");
625             ret = AVERROR(EINVAL);
626             goto end;
627         }
628
629         av_log(avctx, AV_LOG_DEBUG, "Decoding level 1 plane %i %i %i %i\n", plane, lowpass_height, lowpass_width, highpass_stride);
630
631         low    = s->plane[plane].subband[0];
632         high   = s->plane[plane].subband[2];
633         output = s->plane[plane].l_h[0];
634         for (i = 0; i < lowpass_width; i++) {
635             vert_filter(output, lowpass_width, low, lowpass_width, high, highpass_stride, lowpass_height);
636             low++;
637             high++;
638             output++;
639         }
640
641         low    = s->plane[plane].subband[1];
642         high   = s->plane[plane].subband[3];
643         output = s->plane[plane].l_h[1];
644
645         for (i = 0; i < lowpass_width; i++) {
646             // note the stride of "low" is highpass_stride
647             vert_filter(output, lowpass_width, low, highpass_stride, high, highpass_stride, lowpass_height);
648             low++;
649             high++;
650             output++;
651         }
652
653         low    = s->plane[plane].l_h[0];
654         high   = s->plane[plane].l_h[1];
655         output = s->plane[plane].subband[0];
656         for (i = 0; i < lowpass_height * 2; i++) {
657             horiz_filter(output, low, high, lowpass_width);
658             low    += lowpass_width;
659             high   += lowpass_width;
660             output += lowpass_width * 2;
661         }
662         if (s->bpc == 12) {
663             output = s->plane[plane].subband[0];
664             for (i = 0; i < lowpass_height * 2; i++) {
665                 for (j = 0; j < lowpass_width * 2; j++)
666                     output[j] *= 4;
667
668                 output += lowpass_width * 2;
669             }
670         }
671
672         /* level 2 */
673         lowpass_height  = s->plane[plane].band[1][1].height;
674         lowpass_width   = s->plane[plane].band[1][1].width;
675         highpass_stride = s->plane[plane].band[1][1].stride;
676
677         if (lowpass_height > s->plane[plane].band[1][1].a_height || lowpass_width > s->plane[plane].band[1][1].a_width ||
678             !highpass_stride || s->plane[plane].band[1][1].width > s->plane[plane].band[1][1].a_width) {
679             av_log(avctx, AV_LOG_ERROR, "Invalid plane dimensions\n");
680             ret = AVERROR(EINVAL);
681             goto end;
682         }
683
684         av_log(avctx, AV_LOG_DEBUG, "Level 2 plane %i %i %i %i\n", plane, lowpass_height, lowpass_width, highpass_stride);
685
686         low    = s->plane[plane].subband[0];
687         high   = s->plane[plane].subband[5];
688         output = s->plane[plane].l_h[3];
689         for (i = 0; i < lowpass_width; i++) {
690             vert_filter(output, lowpass_width, low, lowpass_width, high, highpass_stride, lowpass_height);
691             low++;
692             high++;
693             output++;
694         }
695
696         low    = s->plane[plane].subband[4];
697         high   = s->plane[plane].subband[6];
698         output = s->plane[plane].l_h[4];
699         for (i = 0; i < lowpass_width; i++) {
700             vert_filter(output, lowpass_width, low, highpass_stride, high, highpass_stride, lowpass_height);
701             low++;
702             high++;
703             output++;
704         }
705
706         low    = s->plane[plane].l_h[3];
707         high   = s->plane[plane].l_h[4];
708         output = s->plane[plane].subband[0];
709         for (i = 0; i < lowpass_height * 2; i++) {
710             horiz_filter(output, low, high, lowpass_width);
711             low    += lowpass_width;
712             high   += lowpass_width;
713             output += lowpass_width * 2;
714         }
715
716         output = s->plane[plane].subband[0];
717         for (i = 0; i < lowpass_height * 2; i++) {
718             for (j = 0; j < lowpass_width * 2; j++)
719                 output[j] *= 4;
720
721             output += lowpass_width * 2;
722         }
723
724         /* level 3 */
725         lowpass_height  = s->plane[plane].band[2][1].height;
726         lowpass_width   = s->plane[plane].band[2][1].width;
727         highpass_stride = s->plane[plane].band[2][1].stride;
728
729         if (lowpass_height > s->plane[plane].band[2][1].a_height || lowpass_width > s->plane[plane].band[2][1].a_width ||
730             !highpass_stride || s->plane[plane].band[2][1].width > s->plane[plane].band[2][1].a_width) {
731             av_log(avctx, AV_LOG_ERROR, "Invalid plane dimensions\n");
732             ret = AVERROR(EINVAL);
733             goto end;
734         }
735
736         av_log(avctx, AV_LOG_DEBUG, "Level 3 plane %i %i %i %i\n", plane, lowpass_height, lowpass_width, highpass_stride);
737
738         low    = s->plane[plane].subband[0];
739         high   = s->plane[plane].subband[8];
740         output = s->plane[plane].l_h[6];
741         for (i = 0; i < lowpass_width; i++) {
742             vert_filter(output, lowpass_width, low, lowpass_width, high, highpass_stride, lowpass_height);
743             low++;
744             high++;
745             output++;
746         }
747
748         low    = s->plane[plane].subband[7];
749         high   = s->plane[plane].subband[9];
750         output = s->plane[plane].l_h[7];
751         for (i = 0; i < lowpass_width; i++) {
752             vert_filter(output, lowpass_width, low, highpass_stride, high, highpass_stride, lowpass_height);
753             low++;
754             high++;
755             output++;
756         }
757
758         dst = (int16_t *)pic->data[act_plane];
759         low  = s->plane[plane].l_h[6];
760         high = s->plane[plane].l_h[7];
761         for (i = 0; i < lowpass_height * 2; i++) {
762             horiz_filter_clip(dst, low, high, lowpass_width, s->bpc);
763             low  += lowpass_width;
764             high += lowpass_width;
765             dst  += pic->linesize[act_plane] / 2;
766         }
767     }
768
769
770 end:
771     if (ret < 0)
772         return ret;
773
774     *got_frame = 1;
775     return avpkt->size;
776 }
777
778 static av_cold int cfhd_close_decoder(AVCodecContext *avctx)
779 {
780     CFHDContext *s = avctx->priv_data;
781
782     free_buffers(avctx);
783
784     if (!avctx->internal->is_copy) {
785         ff_free_vlc(&s->vlc_9);
786         ff_free_vlc(&s->vlc_18);
787     }
788
789     return 0;
790 }
791
792 AVCodec ff_cfhd_decoder = {
793     .name           = "cfhd",
794     .long_name      = NULL_IF_CONFIG_SMALL("Cineform HD"),
795     .type           = AVMEDIA_TYPE_VIDEO,
796     .id             = AV_CODEC_ID_CFHD,
797     .priv_data_size = sizeof(CFHDContext),
798     .init           = cfhd_decode_init,
799     .close          = cfhd_close_decoder,
800     .decode         = cfhd_decode,
801     .capabilities   = AV_CODEC_CAP_DR1 | AV_CODEC_CAP_FRAME_THREADS,
802     .caps_internal  = FF_CODEC_CAP_INIT_THREADSAFE | FF_CODEC_CAP_INIT_CLEANUP,
803 };