2 * Error resilience / concealment
4 * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
6 * This file is part of Libav.
8 * Libav is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
13 * Libav is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with Libav; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
25 * Error resilience / concealment.
32 #include "mpegvideo.h"
34 #include "rectangle.h"
38 * H264 redefines mb_intra so it is not mistakely used (its uninitialized in h264)
39 * but error concealment must support both h264 and h263 thus we must undo this
43 static void decode_mb(MpegEncContext *s, int ref){
44 s->dest[0] = s->current_picture.f.data[0] + (s->mb_y * 16 * s->linesize) + s->mb_x * 16;
45 s->dest[1] = s->current_picture.f.data[1] + (s->mb_y * (16 >> s->chroma_y_shift) * s->uvlinesize) + s->mb_x * (16 >> s->chroma_x_shift);
46 s->dest[2] = s->current_picture.f.data[2] + (s->mb_y * (16 >> s->chroma_y_shift) * s->uvlinesize) + s->mb_x * (16 >> s->chroma_x_shift);
48 if(CONFIG_H264_DECODER && s->codec_id == CODEC_ID_H264){
49 H264Context *h= (void*)s;
50 h->mb_xy= s->mb_x + s->mb_y*s->mb_stride;
51 memset(h->non_zero_count_cache, 0, sizeof(h->non_zero_count_cache));
53 if(ref >= h->ref_count[0]) //FIXME it is posible albeit uncommon that slice references differ between slices, we take the easy approuch and ignore it for now. If this turns out to have any relevance in practice then correct remapping should be added
55 fill_rectangle(&s->current_picture.f.ref_index[0][4*h->mb_xy], 2, 2, 2, ref, 1);
56 fill_rectangle(&h->ref_cache[0][scan8[0]], 4, 4, 8, ref, 1);
57 fill_rectangle(h->mv_cache[0][ scan8[0] ], 4, 4, 8, pack16to32(s->mv[0][0][0],s->mv[0][0][1]), 4);
59 ff_h264_hl_decode_mb(h);
62 MPV_decode_mb(s, s->block);
67 * @param stride the number of MVs to get to the next row
68 * @param mv_step the number of MVs per row or column in a macroblock
70 static void set_mv_strides(MpegEncContext *s, int *mv_step, int *stride){
71 if(s->codec_id == CODEC_ID_H264){
72 H264Context *h= (void*)s;
73 assert(s->quarter_sample);
78 *stride= s->b8_stride;
83 * replaces the current MB with a flat dc only version.
85 static void put_dc(MpegEncContext *s, uint8_t *dest_y, uint8_t *dest_cb, uint8_t *dest_cr, int mb_x, int mb_y)
87 int dc, dcu, dcv, y, i;
89 dc= s->dc_val[0][mb_x*2 + (i&1) + (mb_y*2 + (i>>1))*s->b8_stride];
91 else if(dc>2040) dc=2040;
95 dest_y[x + (i&1)*8 + (y + (i>>1)*8)*s->linesize]= dc/8;
99 dcu = s->dc_val[1][mb_x + mb_y*s->mb_stride];
100 dcv = s->dc_val[2][mb_x + mb_y*s->mb_stride];
102 else if(dcu>2040) dcu=2040;
104 else if(dcv>2040) dcv=2040;
108 dest_cb[x + y*(s->uvlinesize)]= dcu/8;
109 dest_cr[x + y*(s->uvlinesize)]= dcv/8;
114 static void filter181(int16_t *data, int width, int height, int stride){
117 /* horizontal filter */
118 for(y=1; y<height-1; y++){
119 int prev_dc= data[0 + y*stride];
121 for(x=1; x<width-1; x++){
125 + data[x + y*stride]*8
126 - data[x + 1 + y*stride];
127 dc= (dc*10923 + 32768)>>16;
128 prev_dc= data[x + y*stride];
129 data[x + y*stride]= dc;
133 /* vertical filter */
134 for(x=1; x<width-1; x++){
135 int prev_dc= data[x];
137 for(y=1; y<height-1; y++){
141 + data[x + y *stride]*8
142 - data[x + (y+1)*stride];
143 dc= (dc*10923 + 32768)>>16;
144 prev_dc= data[x + y*stride];
145 data[x + y*stride]= dc;
151 * guess the dc of blocks which do not have an undamaged dc
152 * @param w width in 8 pixel blocks
153 * @param h height in 8 pixel blocks
155 static void guess_dc(MpegEncContext *s, int16_t *dc, int w, int h, int stride, int is_luma){
158 for(b_y=0; b_y<h; b_y++){
159 for(b_x=0; b_x<w; b_x++){
160 int color[4]={1024,1024,1024,1024};
161 int distance[4]={9999,9999,9999,9999};
162 int mb_index, error, j;
163 int64_t guess, weight_sum;
165 mb_index= (b_x>>is_luma) + (b_y>>is_luma)*s->mb_stride;
167 error= s->error_status_table[mb_index];
169 if(IS_INTER(s->current_picture.f.mb_type[mb_index])) continue; //inter
170 if(!(error&DC_ERROR)) continue; //dc-ok
173 for(j=b_x+1; j<w; j++){
174 int mb_index_j= (j>>is_luma) + (b_y>>is_luma)*s->mb_stride;
175 int error_j= s->error_status_table[mb_index_j];
176 int intra_j = IS_INTRA(s->current_picture.f.mb_type[mb_index_j]);
177 if(intra_j==0 || !(error_j&DC_ERROR)){
178 color[0]= dc[j + b_y*stride];
185 for(j=b_x-1; j>=0; j--){
186 int mb_index_j= (j>>is_luma) + (b_y>>is_luma)*s->mb_stride;
187 int error_j= s->error_status_table[mb_index_j];
188 int intra_j = IS_INTRA(s->current_picture.f.mb_type[mb_index_j]);
189 if(intra_j==0 || !(error_j&DC_ERROR)){
190 color[1]= dc[j + b_y*stride];
197 for(j=b_y+1; j<h; j++){
198 int mb_index_j= (b_x>>is_luma) + (j>>is_luma)*s->mb_stride;
199 int error_j= s->error_status_table[mb_index_j];
200 int intra_j = IS_INTRA(s->current_picture.f.mb_type[mb_index_j]);
201 if(intra_j==0 || !(error_j&DC_ERROR)){
202 color[2]= dc[b_x + j*stride];
209 for(j=b_y-1; j>=0; j--){
210 int mb_index_j= (b_x>>is_luma) + (j>>is_luma)*s->mb_stride;
211 int error_j= s->error_status_table[mb_index_j];
212 int intra_j = IS_INTRA(s->current_picture.f.mb_type[mb_index_j]);
213 if(intra_j==0 || !(error_j&DC_ERROR)){
214 color[3]= dc[b_x + j*stride];
223 int64_t weight= 256*256*256*16/distance[j];
224 guess+= weight*(int64_t)color[j];
227 guess= (guess + weight_sum/2) / weight_sum;
229 dc[b_x + b_y*stride]= guess;
235 * simple horizontal deblocking filter used for error resilience
236 * @param w width in 8 pixel blocks
237 * @param h height in 8 pixel blocks
239 static void h_block_filter(MpegEncContext *s, uint8_t *dst, int w, int h, int stride, int is_luma){
240 int b_x, b_y, mvx_stride, mvy_stride;
241 uint8_t *cm = ff_cropTbl + MAX_NEG_CROP;
242 set_mv_strides(s, &mvx_stride, &mvy_stride);
243 mvx_stride >>= is_luma;
244 mvy_stride *= mvx_stride;
246 for(b_y=0; b_y<h; b_y++){
247 for(b_x=0; b_x<w-1; b_x++){
249 int left_status = s->error_status_table[( b_x >>is_luma) + (b_y>>is_luma)*s->mb_stride];
250 int right_status= s->error_status_table[((b_x+1)>>is_luma) + (b_y>>is_luma)*s->mb_stride];
251 int left_intra = IS_INTRA(s->current_picture.f.mb_type[( b_x >> is_luma) + (b_y >> is_luma) * s->mb_stride]);
252 int right_intra = IS_INTRA(s->current_picture.f.mb_type[((b_x + 1) >> is_luma) + (b_y >> is_luma) * s->mb_stride]);
253 int left_damage = left_status&(DC_ERROR|AC_ERROR|MV_ERROR);
254 int right_damage= right_status&(DC_ERROR|AC_ERROR|MV_ERROR);
255 int offset= b_x*8 + b_y*stride*8;
256 int16_t *left_mv= s->current_picture.f.motion_val[0][mvy_stride*b_y + mvx_stride* b_x ];
257 int16_t *right_mv= s->current_picture.f.motion_val[0][mvy_stride*b_y + mvx_stride*(b_x+1)];
259 if(!(left_damage||right_damage)) continue; // both undamaged
261 if( (!left_intra) && (!right_intra)
262 && FFABS(left_mv[0]-right_mv[0]) + FFABS(left_mv[1]+right_mv[1]) < 2) continue;
267 a= dst[offset + 7 + y*stride] - dst[offset + 6 + y*stride];
268 b= dst[offset + 8 + y*stride] - dst[offset + 7 + y*stride];
269 c= dst[offset + 9 + y*stride] - dst[offset + 8 + y*stride];
271 d= FFABS(b) - ((FFABS(a) + FFABS(c) + 1)>>1);
277 if(!(left_damage && right_damage))
281 dst[offset + 7 + y*stride] = cm[dst[offset + 7 + y*stride] + ((d*7)>>4)];
282 dst[offset + 6 + y*stride] = cm[dst[offset + 6 + y*stride] + ((d*5)>>4)];
283 dst[offset + 5 + y*stride] = cm[dst[offset + 5 + y*stride] + ((d*3)>>4)];
284 dst[offset + 4 + y*stride] = cm[dst[offset + 4 + y*stride] + ((d*1)>>4)];
287 dst[offset + 8 + y*stride] = cm[dst[offset + 8 + y*stride] - ((d*7)>>4)];
288 dst[offset + 9 + y*stride] = cm[dst[offset + 9 + y*stride] - ((d*5)>>4)];
289 dst[offset + 10+ y*stride] = cm[dst[offset +10 + y*stride] - ((d*3)>>4)];
290 dst[offset + 11+ y*stride] = cm[dst[offset +11 + y*stride] - ((d*1)>>4)];
298 * simple vertical deblocking filter used for error resilience
299 * @param w width in 8 pixel blocks
300 * @param h height in 8 pixel blocks
302 static void v_block_filter(MpegEncContext *s, uint8_t *dst, int w, int h, int stride, int is_luma){
303 int b_x, b_y, mvx_stride, mvy_stride;
304 uint8_t *cm = ff_cropTbl + MAX_NEG_CROP;
305 set_mv_strides(s, &mvx_stride, &mvy_stride);
306 mvx_stride >>= is_luma;
307 mvy_stride *= mvx_stride;
309 for(b_y=0; b_y<h-1; b_y++){
310 for(b_x=0; b_x<w; b_x++){
312 int top_status = s->error_status_table[(b_x>>is_luma) + ( b_y >>is_luma)*s->mb_stride];
313 int bottom_status= s->error_status_table[(b_x>>is_luma) + ((b_y+1)>>is_luma)*s->mb_stride];
314 int top_intra = IS_INTRA(s->current_picture.f.mb_type[(b_x >> is_luma) + ( b_y >> is_luma) * s->mb_stride]);
315 int bottom_intra = IS_INTRA(s->current_picture.f.mb_type[(b_x >> is_luma) + ((b_y + 1) >> is_luma) * s->mb_stride]);
316 int top_damage = top_status&(DC_ERROR|AC_ERROR|MV_ERROR);
317 int bottom_damage= bottom_status&(DC_ERROR|AC_ERROR|MV_ERROR);
318 int offset= b_x*8 + b_y*stride*8;
319 int16_t *top_mv = s->current_picture.f.motion_val[0][mvy_stride * b_y + mvx_stride * b_x];
320 int16_t *bottom_mv = s->current_picture.f.motion_val[0][mvy_stride * (b_y + 1) + mvx_stride * b_x];
322 if(!(top_damage||bottom_damage)) continue; // both undamaged
324 if( (!top_intra) && (!bottom_intra)
325 && FFABS(top_mv[0]-bottom_mv[0]) + FFABS(top_mv[1]+bottom_mv[1]) < 2) continue;
330 a= dst[offset + x + 7*stride] - dst[offset + x + 6*stride];
331 b= dst[offset + x + 8*stride] - dst[offset + x + 7*stride];
332 c= dst[offset + x + 9*stride] - dst[offset + x + 8*stride];
334 d= FFABS(b) - ((FFABS(a) + FFABS(c)+1)>>1);
340 if(!(top_damage && bottom_damage))
344 dst[offset + x + 7*stride] = cm[dst[offset + x + 7*stride] + ((d*7)>>4)];
345 dst[offset + x + 6*stride] = cm[dst[offset + x + 6*stride] + ((d*5)>>4)];
346 dst[offset + x + 5*stride] = cm[dst[offset + x + 5*stride] + ((d*3)>>4)];
347 dst[offset + x + 4*stride] = cm[dst[offset + x + 4*stride] + ((d*1)>>4)];
350 dst[offset + x + 8*stride] = cm[dst[offset + x + 8*stride] - ((d*7)>>4)];
351 dst[offset + x + 9*stride] = cm[dst[offset + x + 9*stride] - ((d*5)>>4)];
352 dst[offset + x + 10*stride] = cm[dst[offset + x + 10*stride] - ((d*3)>>4)];
353 dst[offset + x + 11*stride] = cm[dst[offset + x + 11*stride] - ((d*1)>>4)];
360 static void guess_mv(MpegEncContext *s){
361 uint8_t fixed[s->mb_stride * s->mb_height];
364 #define MV_UNCHANGED 1
365 const int mb_stride = s->mb_stride;
366 const int mb_width = s->mb_width;
367 const int mb_height= s->mb_height;
368 int i, depth, num_avail;
369 int mb_x, mb_y, mot_step, mot_stride;
371 set_mv_strides(s, &mot_step, &mot_stride);
374 for(i=0; i<s->mb_num; i++){
375 const int mb_xy= s->mb_index2xy[ i ];
377 int error= s->error_status_table[mb_xy];
379 if(IS_INTRA(s->current_picture.f.mb_type[mb_xy])) f=MV_FROZEN; //intra //FIXME check
380 if(!(error&MV_ERROR)) f=MV_FROZEN; //inter with undamaged MV
387 if((!(s->avctx->error_concealment&FF_EC_GUESS_MVS)) || num_avail <= mb_width/2){
388 for(mb_y=0; mb_y<s->mb_height; mb_y++){
389 for(mb_x=0; mb_x<s->mb_width; mb_x++){
390 const int mb_xy= mb_x + mb_y*s->mb_stride;
392 if(IS_INTRA(s->current_picture.f.mb_type[mb_xy])) continue;
393 if(!(s->error_status_table[mb_xy]&MV_ERROR)) continue;
395 s->mv_dir = s->last_picture.f.data[0] ? MV_DIR_FORWARD : MV_DIR_BACKWARD;
397 s->mv_type = MV_TYPE_16X16;
400 s->dsp.clear_blocks(s->block[0]);
412 for(depth=0;; depth++){
413 int changed, pass, none_left;
417 for(pass=0; (changed || pass<2) && pass<10; pass++){
422 for(mb_y=0; mb_y<s->mb_height; mb_y++){
423 for(mb_x=0; mb_x<s->mb_width; mb_x++){
424 const int mb_xy= mb_x + mb_y*s->mb_stride;
425 int mv_predictor[8][2]={{0}};
429 int best_score=256*256*256*64;
431 const int mot_index= (mb_x + mb_y*mot_stride) * mot_step;
432 int prev_x, prev_y, prev_ref;
434 if((mb_x^mb_y^pass)&1) continue;
436 if(fixed[mb_xy]==MV_FROZEN) continue;
437 assert(!IS_INTRA(s->current_picture.f.mb_type[mb_xy]));
438 assert(s->last_picture_ptr && s->last_picture_ptr->f.data[0]);
441 if(mb_x>0 && fixed[mb_xy-1 ]==MV_FROZEN) j=1;
442 if(mb_x+1<mb_width && fixed[mb_xy+1 ]==MV_FROZEN) j=1;
443 if(mb_y>0 && fixed[mb_xy-mb_stride]==MV_FROZEN) j=1;
444 if(mb_y+1<mb_height && fixed[mb_xy+mb_stride]==MV_FROZEN) j=1;
448 if(mb_x>0 && fixed[mb_xy-1 ]==MV_CHANGED) j=1;
449 if(mb_x+1<mb_width && fixed[mb_xy+1 ]==MV_CHANGED) j=1;
450 if(mb_y>0 && fixed[mb_xy-mb_stride]==MV_CHANGED) j=1;
451 if(mb_y+1<mb_height && fixed[mb_xy+mb_stride]==MV_CHANGED) j=1;
452 if(j==0 && pass>1) continue;
456 if(mb_x>0 && fixed[mb_xy-1]){
457 mv_predictor[pred_count][0]= s->current_picture.f.motion_val[0][mot_index - mot_step][0];
458 mv_predictor[pred_count][1]= s->current_picture.f.motion_val[0][mot_index - mot_step][1];
459 ref [pred_count] = s->current_picture.f.ref_index[0][4*(mb_xy-1)];
462 if(mb_x+1<mb_width && fixed[mb_xy+1]){
463 mv_predictor[pred_count][0]= s->current_picture.f.motion_val[0][mot_index + mot_step][0];
464 mv_predictor[pred_count][1]= s->current_picture.f.motion_val[0][mot_index + mot_step][1];
465 ref [pred_count] = s->current_picture.f.ref_index[0][4*(mb_xy+1)];
468 if(mb_y>0 && fixed[mb_xy-mb_stride]){
469 mv_predictor[pred_count][0]= s->current_picture.f.motion_val[0][mot_index - mot_stride*mot_step][0];
470 mv_predictor[pred_count][1]= s->current_picture.f.motion_val[0][mot_index - mot_stride*mot_step][1];
471 ref [pred_count] = s->current_picture.f.ref_index[0][4*(mb_xy-s->mb_stride)];
474 if(mb_y+1<mb_height && fixed[mb_xy+mb_stride]){
475 mv_predictor[pred_count][0]= s->current_picture.f.motion_val[0][mot_index + mot_stride*mot_step][0];
476 mv_predictor[pred_count][1]= s->current_picture.f.motion_val[0][mot_index + mot_stride*mot_step][1];
477 ref [pred_count] = s->current_picture.f.ref_index[0][4*(mb_xy+s->mb_stride)];
480 if(pred_count==0) continue;
483 int sum_x=0, sum_y=0, sum_r=0;
484 int max_x, max_y, min_x, min_y, max_r, min_r;
486 for(j=0; j<pred_count; j++){
487 sum_x+= mv_predictor[j][0];
488 sum_y+= mv_predictor[j][1];
490 if(j && ref[j] != ref[j-1])
491 goto skip_mean_and_median;
495 mv_predictor[pred_count][0] = sum_x/j;
496 mv_predictor[pred_count][1] = sum_y/j;
497 ref [pred_count] = sum_r/j;
501 min_y= min_x= min_r= 99999;
502 max_y= max_x= max_r=-99999;
504 min_x=min_y=max_x=max_y=min_r=max_r=0;
506 for(j=0; j<pred_count; j++){
507 max_x= FFMAX(max_x, mv_predictor[j][0]);
508 max_y= FFMAX(max_y, mv_predictor[j][1]);
509 max_r= FFMAX(max_r, ref[j]);
510 min_x= FFMIN(min_x, mv_predictor[j][0]);
511 min_y= FFMIN(min_y, mv_predictor[j][1]);
512 min_r= FFMIN(min_r, ref[j]);
514 mv_predictor[pred_count+1][0] = sum_x - max_x - min_x;
515 mv_predictor[pred_count+1][1] = sum_y - max_y - min_y;
516 ref [pred_count+1] = sum_r - max_r - min_r;
519 mv_predictor[pred_count+1][0] /= 2;
520 mv_predictor[pred_count+1][1] /= 2;
521 ref [pred_count+1] /= 2;
525 skip_mean_and_median:
531 if (s->avctx->codec_id == CODEC_ID_H264) {
534 ff_thread_await_progress((AVFrame *) s->last_picture_ptr,
537 if (!s->last_picture.f.motion_val[0] ||
538 !s->last_picture.f.ref_index[0])
540 prev_x = s->last_picture.f.motion_val[0][mot_index][0];
541 prev_y = s->last_picture.f.motion_val[0][mot_index][1];
542 prev_ref = s->last_picture.f.ref_index[0][4*mb_xy];
544 prev_x = s->current_picture.f.motion_val[0][mot_index][0];
545 prev_y = s->current_picture.f.motion_val[0][mot_index][1];
546 prev_ref = s->current_picture.f.ref_index[0][4*mb_xy];
550 mv_predictor[pred_count][0]= prev_x;
551 mv_predictor[pred_count][1]= prev_y;
552 ref [pred_count] = prev_ref;
556 s->mv_dir = MV_DIR_FORWARD;
558 s->mv_type = MV_TYPE_16X16;
561 s->dsp.clear_blocks(s->block[0]);
566 for(j=0; j<pred_count; j++){
568 uint8_t *src = s->current_picture.f.data[0] + mb_x*16 + mb_y*16*s->linesize;
570 s->current_picture.f.motion_val[0][mot_index][0] = s->mv[0][0][0] = mv_predictor[j][0];
571 s->current_picture.f.motion_val[0][mot_index][1] = s->mv[0][0][1] = mv_predictor[j][1];
573 if(ref[j]<0) //predictor intra or otherwise not available
576 decode_mb(s, ref[j]);
578 if(mb_x>0 && fixed[mb_xy-1]){
581 score += FFABS(src[k*s->linesize-1 ]-src[k*s->linesize ]);
583 if(mb_x+1<mb_width && fixed[mb_xy+1]){
586 score += FFABS(src[k*s->linesize+15]-src[k*s->linesize+16]);
588 if(mb_y>0 && fixed[mb_xy-mb_stride]){
591 score += FFABS(src[k-s->linesize ]-src[k ]);
593 if(mb_y+1<mb_height && fixed[mb_xy+mb_stride]){
596 score += FFABS(src[k+s->linesize*15]-src[k+s->linesize*16]);
599 if(score <= best_score){ // <= will favor the last MV
604 score_sum+= best_score;
605 s->mv[0][0][0]= mv_predictor[best_pred][0];
606 s->mv[0][0][1]= mv_predictor[best_pred][1];
608 for(i=0; i<mot_step; i++)
609 for(j=0; j<mot_step; j++){
610 s->current_picture.f.motion_val[0][mot_index + i + j * mot_stride][0] = s->mv[0][0][0];
611 s->current_picture.f.motion_val[0][mot_index + i + j * mot_stride][1] = s->mv[0][0][1];
614 decode_mb(s, ref[best_pred]);
617 if(s->mv[0][0][0] != prev_x || s->mv[0][0][1] != prev_y){
618 fixed[mb_xy]=MV_CHANGED;
621 fixed[mb_xy]=MV_UNCHANGED;
625 // printf(".%d/%d", changed, score_sum); fflush(stdout);
631 for(i=0; i<s->mb_num; i++){
632 int mb_xy= s->mb_index2xy[i];
634 fixed[mb_xy]=MV_FROZEN;
636 // printf(":"); fflush(stdout);
640 static int is_intra_more_likely(MpegEncContext *s){
641 int is_intra_likely, i, j, undamaged_count, skip_amount, mb_x, mb_y;
643 if (!s->last_picture_ptr || !s->last_picture_ptr->f.data[0]) return 1; //no previous frame available -> use spatial prediction
646 for(i=0; i<s->mb_num; i++){
647 const int mb_xy= s->mb_index2xy[i];
648 const int error= s->error_status_table[mb_xy];
649 if(!((error&DC_ERROR) && (error&MV_ERROR)))
653 if(s->codec_id == CODEC_ID_H264){
654 H264Context *h= (void*)s;
655 if (h->ref_count[0] <= 0 || !h->ref_list[0][0].f.data[0])
659 if(undamaged_count < 5) return 0; //almost all MBs damaged -> use temporal prediction
661 //prevent dsp.sad() check, that requires access to the image
662 if(CONFIG_MPEG_XVMC_DECODER && s->avctx->xvmc_acceleration && s->pict_type == AV_PICTURE_TYPE_I)
665 skip_amount= FFMAX(undamaged_count/50, 1); //check only upto 50 MBs
669 for(mb_y= 0; mb_y<s->mb_height-1; mb_y++){
670 for(mb_x= 0; mb_x<s->mb_width; mb_x++){
672 const int mb_xy= mb_x + mb_y*s->mb_stride;
674 error= s->error_status_table[mb_xy];
675 if((error&DC_ERROR) && (error&MV_ERROR))
676 continue; //skip damaged
679 if((j%skip_amount) != 0) continue; //skip a few to speed things up
681 if(s->pict_type==AV_PICTURE_TYPE_I){
682 uint8_t *mb_ptr = s->current_picture.f.data[0] + mb_x*16 + mb_y*16*s->linesize;
683 uint8_t *last_mb_ptr= s->last_picture.f.data [0] + mb_x*16 + mb_y*16*s->linesize;
685 if (s->avctx->codec_id == CODEC_ID_H264) {
688 ff_thread_await_progress((AVFrame *) s->last_picture_ptr,
691 is_intra_likely += s->dsp.sad[0](NULL, last_mb_ptr, mb_ptr , s->linesize, 16);
692 is_intra_likely -= s->dsp.sad[0](NULL, last_mb_ptr, last_mb_ptr+s->linesize*16, s->linesize, 16);
694 if (IS_INTRA(s->current_picture.f.mb_type[mb_xy]))
701 //printf("is_intra_likely: %d type:%d\n", is_intra_likely, s->pict_type);
702 return is_intra_likely > 0;
705 void ff_er_frame_start(MpegEncContext *s){
706 if(!s->error_recognition) return;
708 memset(s->error_status_table, MV_ERROR|AC_ERROR|DC_ERROR|VP_START|AC_END|DC_END|MV_END, s->mb_stride*s->mb_height*sizeof(uint8_t));
709 s->error_count= 3*s->mb_num;
710 s->error_occurred = 0;
715 * @param endx x component of the last macroblock, can be -1 for the last of the previous line
716 * @param status the status at the end (MV_END, AC_ERROR, ...), it is assumed that no earlier end or
717 * error of the same type occurred
719 void ff_er_add_slice(MpegEncContext *s, int startx, int starty, int endx, int endy, int status){
720 const int start_i= av_clip(startx + starty * s->mb_width , 0, s->mb_num-1);
721 const int end_i = av_clip(endx + endy * s->mb_width , 0, s->mb_num);
722 const int start_xy= s->mb_index2xy[start_i];
723 const int end_xy = s->mb_index2xy[end_i];
726 if(s->avctx->hwaccel)
729 if(start_i > end_i || start_xy > end_xy){
730 av_log(s->avctx, AV_LOG_ERROR, "internal error, slice end before start\n");
734 if(!s->error_recognition) return;
737 if(status & (AC_ERROR|AC_END)){
738 mask &= ~(AC_ERROR|AC_END);
739 s->error_count -= end_i - start_i + 1;
741 if(status & (DC_ERROR|DC_END)){
742 mask &= ~(DC_ERROR|DC_END);
743 s->error_count -= end_i - start_i + 1;
745 if(status & (MV_ERROR|MV_END)){
746 mask &= ~(MV_ERROR|MV_END);
747 s->error_count -= end_i - start_i + 1;
750 if(status & (AC_ERROR|DC_ERROR|MV_ERROR)) {
751 s->error_occurred = 1;
752 s->error_count= INT_MAX;
756 memset(&s->error_status_table[start_xy], 0, (end_xy - start_xy) * sizeof(uint8_t));
759 for(i=start_xy; i<end_xy; i++){
760 s->error_status_table[ i ] &= mask;
764 if(end_i == s->mb_num)
765 s->error_count= INT_MAX;
767 s->error_status_table[end_xy] &= mask;
768 s->error_status_table[end_xy] |= status;
771 s->error_status_table[start_xy] |= VP_START;
773 if(start_xy > 0 && s->avctx->thread_count <= 1 && s->avctx->skip_top*s->mb_width < start_i){
774 int prev_status= s->error_status_table[ s->mb_index2xy[start_i - 1] ];
776 prev_status &= ~ VP_START;
777 if(prev_status != (MV_END|DC_END|AC_END)) s->error_count= INT_MAX;
781 void ff_er_frame_end(MpegEncContext *s){
782 int i, mb_x, mb_y, error, error_type, dc_error, mv_error, ac_error;
784 int threshold_part[4]= {100,100,100};
787 int size = s->b8_stride * 2 * s->mb_height;
788 Picture *pic= s->current_picture_ptr;
790 if(!s->error_recognition || s->error_count==0 || s->avctx->lowres ||
792 s->avctx->codec->capabilities&CODEC_CAP_HWACCEL_VDPAU ||
793 s->picture_structure != PICT_FRAME || // we dont support ER of field pictures yet, though it should not crash if enabled
794 s->error_count==3*s->mb_width*(s->avctx->skip_top + s->avctx->skip_bottom)) return;
796 if (s->current_picture.f.motion_val[0] == NULL) {
797 av_log(s->avctx, AV_LOG_ERROR, "Warning MVs not available\n");
800 pic->f.ref_index[i] = av_mallocz(s->mb_stride * s->mb_height * 4 * sizeof(uint8_t));
801 pic->motion_val_base[i]= av_mallocz((size+4) * 2 * sizeof(uint16_t));
802 pic->f.motion_val[i] = pic->motion_val_base[i] + 4;
804 pic->f.motion_subsample_log2 = 3;
805 s->current_picture= *s->current_picture_ptr;
808 if(s->avctx->debug&FF_DEBUG_ER){
809 for(mb_y=0; mb_y<s->mb_height; mb_y++){
810 for(mb_x=0; mb_x<s->mb_width; mb_x++){
811 int status= s->error_status_table[mb_x + mb_y*s->mb_stride];
813 av_log(s->avctx, AV_LOG_DEBUG, "%2X ", status);
815 av_log(s->avctx, AV_LOG_DEBUG, "\n");
819 /* handle overlapping slices */
820 for(error_type=1; error_type<=3; error_type++){
823 for(i=s->mb_num-1; i>=0; i--){
824 const int mb_xy= s->mb_index2xy[i];
825 int error= s->error_status_table[mb_xy];
827 if(error&(1<<error_type))
829 if(error&(8<<error_type))
833 s->error_status_table[mb_xy]|= 1<<error_type;
840 /* handle slices with partitions of different length */
841 if(s->partitioned_frame){
844 for(i=s->mb_num-1; i>=0; i--){
845 const int mb_xy= s->mb_index2xy[i];
846 int error= s->error_status_table[mb_xy];
850 if((error&MV_END) || (error&DC_END) || (error&AC_ERROR))
854 s->error_status_table[mb_xy]|= AC_ERROR;
861 /* handle missing slices */
862 if(s->error_recognition>=4){
865 for(i=s->mb_num-2; i>=s->mb_width+100; i--){ //FIXME +100 hack
866 const int mb_xy= s->mb_index2xy[i];
867 int error1= s->error_status_table[mb_xy ];
868 int error2= s->error_status_table[s->mb_index2xy[i+1]];
873 if( error2==(VP_START|DC_ERROR|AC_ERROR|MV_ERROR|AC_END|DC_END|MV_END)
874 && error1!=(VP_START|DC_ERROR|AC_ERROR|MV_ERROR|AC_END|DC_END|MV_END)
875 && ((error1&AC_END) || (error1&DC_END) || (error1&MV_END))){ //end & uninit
880 s->error_status_table[mb_xy]|= DC_ERROR|AC_ERROR|MV_ERROR;
884 /* backward mark errors */
886 for(error_type=1; error_type<=3; error_type++){
887 for(i=s->mb_num-1; i>=0; i--){
888 const int mb_xy= s->mb_index2xy[i];
889 int error= s->error_status_table[mb_xy];
891 if(!s->mbskip_table[mb_xy]) //FIXME partition specific
893 if(error&(1<<error_type))
896 if(s->partitioned_frame){
897 if(distance < threshold_part[error_type-1])
898 s->error_status_table[mb_xy]|= 1<<error_type;
900 if(distance < threshold)
901 s->error_status_table[mb_xy]|= 1<<error_type;
909 /* forward mark errors */
911 for(i=0; i<s->mb_num; i++){
912 const int mb_xy= s->mb_index2xy[i];
913 int old_error= s->error_status_table[mb_xy];
915 if(old_error&VP_START)
916 error= old_error& (DC_ERROR|AC_ERROR|MV_ERROR);
918 error|= old_error& (DC_ERROR|AC_ERROR|MV_ERROR);
919 s->error_status_table[mb_xy]|= error;
923 /* handle not partitioned case */
924 if(!s->partitioned_frame){
925 for(i=0; i<s->mb_num; i++){
926 const int mb_xy= s->mb_index2xy[i];
927 error= s->error_status_table[mb_xy];
928 if(error&(AC_ERROR|DC_ERROR|MV_ERROR))
929 error|= AC_ERROR|DC_ERROR|MV_ERROR;
930 s->error_status_table[mb_xy]= error;
934 dc_error= ac_error= mv_error=0;
935 for(i=0; i<s->mb_num; i++){
936 const int mb_xy= s->mb_index2xy[i];
937 error= s->error_status_table[mb_xy];
938 if(error&DC_ERROR) dc_error ++;
939 if(error&AC_ERROR) ac_error ++;
940 if(error&MV_ERROR) mv_error ++;
942 av_log(s->avctx, AV_LOG_INFO, "concealing %d DC, %d AC, %d MV errors\n", dc_error, ac_error, mv_error);
944 is_intra_likely= is_intra_more_likely(s);
946 /* set unknown mb-type to most likely */
947 for(i=0; i<s->mb_num; i++){
948 const int mb_xy= s->mb_index2xy[i];
949 error= s->error_status_table[mb_xy];
950 if(!((error&DC_ERROR) && (error&MV_ERROR)))
954 s->current_picture.f.mb_type[mb_xy] = MB_TYPE_INTRA4x4;
956 s->current_picture.f.mb_type[mb_xy] = MB_TYPE_16x16 | MB_TYPE_L0;
959 // change inter to intra blocks if no reference frames are available
960 if (!s->last_picture.f.data[0] && !s->next_picture.f.data[0])
961 for(i=0; i<s->mb_num; i++){
962 const int mb_xy= s->mb_index2xy[i];
963 if (!IS_INTRA(s->current_picture.f.mb_type[mb_xy]))
964 s->current_picture.f.mb_type[mb_xy] = MB_TYPE_INTRA4x4;
967 /* handle inter blocks with damaged AC */
968 for(mb_y=0; mb_y<s->mb_height; mb_y++){
969 for(mb_x=0; mb_x<s->mb_width; mb_x++){
970 const int mb_xy= mb_x + mb_y * s->mb_stride;
971 const int mb_type= s->current_picture.f.mb_type[mb_xy];
972 int dir = !s->last_picture.f.data[0];
973 error= s->error_status_table[mb_xy];
975 if(IS_INTRA(mb_type)) continue; //intra
976 if(error&MV_ERROR) continue; //inter with damaged MV
977 if(!(error&AC_ERROR)) continue; //undamaged inter
979 s->mv_dir = dir ? MV_DIR_BACKWARD : MV_DIR_FORWARD;
983 int mb_index= mb_x*2 + mb_y*2*s->b8_stride;
985 s->mv_type = MV_TYPE_8X8;
987 s->mv[0][j][0] = s->current_picture.f.motion_val[dir][mb_index + (j & 1) + (j >> 1) * s->b8_stride][0];
988 s->mv[0][j][1] = s->current_picture.f.motion_val[dir][mb_index + (j & 1) + (j >> 1) * s->b8_stride][1];
991 s->mv_type = MV_TYPE_16X16;
992 s->mv[0][0][0] = s->current_picture.f.motion_val[dir][ mb_x*2 + mb_y*2*s->b8_stride ][0];
993 s->mv[0][0][1] = s->current_picture.f.motion_val[dir][ mb_x*2 + mb_y*2*s->b8_stride ][1];
996 s->dsp.clear_blocks(s->block[0]);
1000 decode_mb(s, 0/*FIXME h264 partitioned slices need this set*/);
1005 if(s->pict_type==AV_PICTURE_TYPE_B){
1006 for(mb_y=0; mb_y<s->mb_height; mb_y++){
1007 for(mb_x=0; mb_x<s->mb_width; mb_x++){
1008 int xy= mb_x*2 + mb_y*2*s->b8_stride;
1009 const int mb_xy= mb_x + mb_y * s->mb_stride;
1010 const int mb_type= s->current_picture.f.mb_type[mb_xy];
1011 error= s->error_status_table[mb_xy];
1013 if(IS_INTRA(mb_type)) continue;
1014 if(!(error&MV_ERROR)) continue; //inter with undamaged MV
1015 if(!(error&AC_ERROR)) continue; //undamaged inter
1017 s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD;
1018 if(!s->last_picture.f.data[0]) s->mv_dir &= ~MV_DIR_FORWARD;
1019 if(!s->next_picture.f.data[0]) s->mv_dir &= ~MV_DIR_BACKWARD;
1021 s->mv_type = MV_TYPE_16X16;
1025 int time_pp= s->pp_time;
1026 int time_pb= s->pb_time;
1028 if (s->avctx->codec_id == CODEC_ID_H264) {
1031 ff_thread_await_progress((AVFrame *) s->next_picture_ptr,
1034 s->mv[0][0][0] = s->next_picture.f.motion_val[0][xy][0] * time_pb / time_pp;
1035 s->mv[0][0][1] = s->next_picture.f.motion_val[0][xy][1] * time_pb / time_pp;
1036 s->mv[1][0][0] = s->next_picture.f.motion_val[0][xy][0] * (time_pb - time_pp) / time_pp;
1037 s->mv[1][0][1] = s->next_picture.f.motion_val[0][xy][1] * (time_pb - time_pp) / time_pp;
1045 s->dsp.clear_blocks(s->block[0]);
1054 /* the filters below are not XvMC compatible, skip them */
1055 if(CONFIG_MPEG_XVMC_DECODER && s->avctx->xvmc_acceleration)
1057 /* fill DC for inter blocks */
1058 for(mb_y=0; mb_y<s->mb_height; mb_y++){
1059 for(mb_x=0; mb_x<s->mb_width; mb_x++){
1060 int dc, dcu, dcv, y, n;
1062 uint8_t *dest_y, *dest_cb, *dest_cr;
1063 const int mb_xy= mb_x + mb_y * s->mb_stride;
1064 const int mb_type = s->current_picture.f.mb_type[mb_xy];
1066 error= s->error_status_table[mb_xy];
1068 if(IS_INTRA(mb_type) && s->partitioned_frame) continue;
1069 // if(error&MV_ERROR) continue; //inter data damaged FIXME is this good?
1071 dest_y = s->current_picture.f.data[0] + mb_x * 16 + mb_y * 16 * s->linesize;
1072 dest_cb = s->current_picture.f.data[1] + mb_x * 8 + mb_y * 8 * s->uvlinesize;
1073 dest_cr = s->current_picture.f.data[2] + mb_x * 8 + mb_y * 8 * s->uvlinesize;
1075 dc_ptr= &s->dc_val[0][mb_x*2 + mb_y*2*s->b8_stride];
1081 dc+= dest_y[x + (n&1)*8 + (y + (n>>1)*8)*s->linesize];
1084 dc_ptr[(n&1) + (n>>1)*s->b8_stride]= (dc+4)>>3;
1091 dcu+=dest_cb[x + y*(s->uvlinesize)];
1092 dcv+=dest_cr[x + y*(s->uvlinesize)];
1095 s->dc_val[1][mb_x + mb_y*s->mb_stride]= (dcu+4)>>3;
1096 s->dc_val[2][mb_x + mb_y*s->mb_stride]= (dcv+4)>>3;
1100 /* guess DC for damaged blocks */
1101 guess_dc(s, s->dc_val[0], s->mb_width*2, s->mb_height*2, s->b8_stride, 1);
1102 guess_dc(s, s->dc_val[1], s->mb_width , s->mb_height , s->mb_stride, 0);
1103 guess_dc(s, s->dc_val[2], s->mb_width , s->mb_height , s->mb_stride, 0);
1105 /* filter luma DC */
1106 filter181(s->dc_val[0], s->mb_width*2, s->mb_height*2, s->b8_stride);
1108 /* render DC only intra */
1109 for(mb_y=0; mb_y<s->mb_height; mb_y++){
1110 for(mb_x=0; mb_x<s->mb_width; mb_x++){
1111 uint8_t *dest_y, *dest_cb, *dest_cr;
1112 const int mb_xy= mb_x + mb_y * s->mb_stride;
1113 const int mb_type = s->current_picture.f.mb_type[mb_xy];
1115 error= s->error_status_table[mb_xy];
1117 if(IS_INTER(mb_type)) continue;
1118 if(!(error&AC_ERROR)) continue; //undamaged
1120 dest_y = s->current_picture.f.data[0] + mb_x * 16 + mb_y * 16 * s->linesize;
1121 dest_cb = s->current_picture.f.data[1] + mb_x * 8 + mb_y * 8 * s->uvlinesize;
1122 dest_cr = s->current_picture.f.data[2] + mb_x * 8 + mb_y * 8 * s->uvlinesize;
1124 put_dc(s, dest_y, dest_cb, dest_cr, mb_x, mb_y);
1128 if(s->avctx->error_concealment&FF_EC_DEBLOCK){
1129 /* filter horizontal block boundaries */
1130 h_block_filter(s, s->current_picture.f.data[0], s->mb_width*2, s->mb_height*2, s->linesize , 1);
1131 h_block_filter(s, s->current_picture.f.data[1], s->mb_width , s->mb_height , s->uvlinesize, 0);
1132 h_block_filter(s, s->current_picture.f.data[2], s->mb_width , s->mb_height , s->uvlinesize, 0);
1134 /* filter vertical block boundaries */
1135 v_block_filter(s, s->current_picture.f.data[0], s->mb_width*2, s->mb_height*2, s->linesize , 1);
1136 v_block_filter(s, s->current_picture.f.data[1], s->mb_width , s->mb_height , s->uvlinesize, 0);
1137 v_block_filter(s, s->current_picture.f.data[2], s->mb_width , s->mb_height , s->uvlinesize, 0);
1141 /* clean a few tables */
1142 for(i=0; i<s->mb_num; i++){
1143 const int mb_xy= s->mb_index2xy[i];
1144 int error= s->error_status_table[mb_xy];
1146 if(s->pict_type!=AV_PICTURE_TYPE_B && (error&(DC_ERROR|MV_ERROR|AC_ERROR))){
1147 s->mbskip_table[mb_xy]=0;
1149 s->mbintra_table[mb_xy]=1;