git.sesse.net Git - ffmpeg/blob - libavcodec/mpegvideo_enc.c

   1 /*
   2  * The simplest mpeg encoder (well, it was the simplest!)
   3  * Copyright (c) 2000,2001 Fabrice Bellard
   4  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
   5  *
   6  * 4MV & hq & B-frame encoding stuff by Michael Niedermayer <michaelni@gmx.at>
   7  *
   8  * This file is part of FFmpeg.
   9  *
  10  * FFmpeg is free software; you can redistribute it and/or
  11  * modify it under the terms of the GNU Lesser General Public
  12  * License as published by the Free Software Foundation; either
  13  * version 2.1 of the License, or (at your option) any later version.
  14  *
  15  * FFmpeg is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18  * Lesser General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU Lesser General Public
  21  * License along with FFmpeg; if not, write to the Free Software
  22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  23  */
  24
  25 /*
  26  * non linear quantizers with large QPs and VBV with restrictive qmin fixes sponsored by NOA GmbH
  27  */
  28
  29 /**
  30  * @file
  31  * The simplest mpeg encoder (well, it was the simplest!).
  32  */
  33
  34 #include <stdint.h>
  35
  36 #include "libavutil/internal.h"
  37 #include "libavutil/intmath.h"
  38 #include "libavutil/mathematics.h"
  39 #include "libavutil/pixdesc.h"
  40 #include "libavutil/opt.h"
  41 #include "libavutil/timer.h"
  42 #include "avcodec.h"
  43 #include "dct.h"
  44 #include "idctdsp.h"
  45 #include "mpeg12.h"
  46 #include "mpegvideo.h"
  47 #include "mpegvideodata.h"
  48 #include "h261.h"
  49 #include "h263.h"
  50 #include "h263data.h"
  51 #include "mjpegenc_common.h"
  52 #include "mathops.h"
  53 #include "mpegutils.h"
  54 #include "mjpegenc.h"
  55 #include "msmpeg4.h"
  56 #include "pixblockdsp.h"
  57 #include "qpeldsp.h"
  58 #include "faandct.h"
  59 #include "thread.h"
  60 #include "aandcttab.h"
  61 #include "flv.h"
  62 #include "mpeg4video.h"
  63 #include "internal.h"
  64 #include "bytestream.h"
  65 #include "wmv2.h"
  66 #include "rv10.h"
  67 #include "libxvid.h"
  68 #include <limits.h>
  69 #include "sp5x.h"
  70
  71 #define QUANT_BIAS_SHIFT 8
  72
  73 #define QMAT_SHIFT_MMX 16
  74 #define QMAT_SHIFT 21
  75
  76 static int encode_picture(MpegEncContext *s, int picture_number);
  77 static int dct_quantize_refine(MpegEncContext *s, int16_t *block, int16_t *weight, int16_t *orig, int n, int qscale);
  78 static int sse_mb(MpegEncContext *s);
  79 static void denoise_dct_c(MpegEncContext *s, int16_t *block);
  80 static int dct_quantize_trellis_c(MpegEncContext *s, int16_t *block, int n, int qscale, int *overflow);
  81
  82 static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_DMV * 2 + 1];
  83 static uint8_t default_fcode_tab[MAX_MV * 2 + 1];
  84
  85 const AVOption ff_mpv_generic_options[] = {
  86     FF_MPV_COMMON_OPTS
  87     { NULL },
  88 };
  89
  90 void ff_convert_matrix(MpegEncContext *s, int (*qmat)[64],
  91                        uint16_t (*qmat16)[2][64],
  92                        const uint16_t *quant_matrix,
  93                        int bias, int qmin, int qmax, int intra)
  94 {
  95     FDCTDSPContext *fdsp = &s->fdsp;
  96     int qscale;
  97     int shift = 0;
  98
  99     for (qscale = qmin; qscale <= qmax; qscale++) {
 100         int i;
 101         int qscale2;
 102
 103         if (s->q_scale_type) qscale2 = ff_mpeg2_non_linear_qscale[qscale];
 104         else                 qscale2 = qscale << 1;
 105
 106         if (fdsp->fdct == ff_jpeg_fdct_islow_8  ||
 107 #if CONFIG_FAANDCT
 108             fdsp->fdct == ff_faandct            ||
 109 #endif /* CONFIG_FAANDCT */
 110             fdsp->fdct == ff_jpeg_fdct_islow_10) {
 111             for (i = 0; i < 64; i++) {
 112                 const int j = s->idsp.idct_permutation[i];
 113                 int64_t den = (int64_t) qscale2 * quant_matrix[j];
 114                 /* 16 <= qscale * quant_matrix[i] <= 7905
 115                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
 116                  *             19952 <=              x  <= 249205026
 117                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
 118                  *           3444240 >= (1 << 36) / (x) >= 275 */
 119
 120                 qmat[qscale][i] = (int)((UINT64_C(2) << QMAT_SHIFT) / den);
 121             }
 122         } else if (fdsp->fdct == ff_fdct_ifast) {
 123             for (i = 0; i < 64; i++) {
 124                 const int j = s->idsp.idct_permutation[i];
 125                 int64_t den = ff_aanscales[i] * (int64_t) qscale2 * quant_matrix[j];
 126                 /* 16 <= qscale * quant_matrix[i] <= 7905
 127                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
 128                  *             19952 <=              x  <= 249205026
 129                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
 130                  *           3444240 >= (1 << 36) / (x) >= 275 */
 131
 132                 qmat[qscale][i] = (int)((UINT64_C(2) << (QMAT_SHIFT + 14)) / den);
 133             }
 134         } else {
 135             for (i = 0; i < 64; i++) {
 136                 const int j = s->idsp.idct_permutation[i];
 137                 int64_t den = (int64_t) qscale2 * quant_matrix[j];
 138                 /* We can safely suppose that 16 <= quant_matrix[i] <= 255
 139                  * Assume x = qscale * quant_matrix[i]
 140                  * So             16 <=              x  <= 7905
 141                  * so (1 << 19) / 16 >= (1 << 19) / (x) >= (1 << 19) / 7905
 142                  * so          32768 >= (1 << 19) / (x) >= 67 */
 143                 qmat[qscale][i] = (int)((UINT64_C(2) << QMAT_SHIFT) / den);
 144                 //qmat  [qscale][i] = (1 << QMAT_SHIFT_MMX) /
 145                 //                    (qscale * quant_matrix[i]);
 146                 qmat16[qscale][0][i] = (2 << QMAT_SHIFT_MMX) / den;
 147
 148                 if (qmat16[qscale][0][i] == 0 ||
 149                     qmat16[qscale][0][i] == 128 * 256)
 150                     qmat16[qscale][0][i] = 128 * 256 - 1;
 151                 qmat16[qscale][1][i] =
 152                     ROUNDED_DIV(bias * (1<<(16 - QUANT_BIAS_SHIFT)),
 153                                 qmat16[qscale][0][i]);
 154             }
 155         }
 156
 157         for (i = intra; i < 64; i++) {
 158             int64_t max = 8191;
 159             if (fdsp->fdct == ff_fdct_ifast) {
 160                 max = (8191LL * ff_aanscales[i]) >> 14;
 161             }
 162             while (((max * qmat[qscale][i]) >> shift) > INT_MAX) {
 163                 shift++;
 164             }
 165         }
 166     }
 167     if (shift) {
 168         av_log(NULL, AV_LOG_INFO,
 169                "Warning, QMAT_SHIFT is larger than %d, overflows possible\n",
 170                QMAT_SHIFT - shift);
 171     }
 172 }
 173
 174 static inline void update_qscale(MpegEncContext *s)
 175 {
 176     if (s->q_scale_type == 1 && 0) {
 177         int i;
 178         int bestdiff=INT_MAX;
 179         int best = 1;
 180
 181         for (i = 0 ; i<FF_ARRAY_ELEMS(ff_mpeg2_non_linear_qscale); i++) {
 182             int diff = FFABS((ff_mpeg2_non_linear_qscale[i]<<(FF_LAMBDA_SHIFT + 6)) - (int)s->lambda * 139);
 183             if (ff_mpeg2_non_linear_qscale[i] < s->avctx->qmin ||
 184                 (ff_mpeg2_non_linear_qscale[i] > s->avctx->qmax && !s->vbv_ignore_qmax))
 185                 continue;
 186             if (diff < bestdiff) {
 187                 bestdiff = diff;
 188                 best = i;
 189             }
 190         }
 191         s->qscale = best;
 192     } else {
 193         s->qscale = (s->lambda * 139 + FF_LAMBDA_SCALE * 64) >>
 194                     (FF_LAMBDA_SHIFT + 7);
 195         s->qscale = av_clip(s->qscale, s->avctx->qmin, s->vbv_ignore_qmax ? 31 : s->avctx->qmax);
 196     }
 197
 198     s->lambda2 = (s->lambda * s->lambda + FF_LAMBDA_SCALE / 2) >>
 199                  FF_LAMBDA_SHIFT;
 200 }
 201
 202 void ff_write_quant_matrix(PutBitContext *pb, uint16_t *matrix)
 203 {
 204     int i;
 205
 206     if (matrix) {
 207         put_bits(pb, 1, 1);
 208         for (i = 0; i < 64; i++) {
 209             put_bits(pb, 8, matrix[ff_zigzag_direct[i]]);
 210         }
 211     } else
 212         put_bits(pb, 1, 0);
 213 }
 214
 215 /**
 216  * init s->current_picture.qscale_table from s->lambda_table
 217  */
 218 void ff_init_qscale_tab(MpegEncContext *s)
 219 {
 220     int8_t * const qscale_table = s->current_picture.qscale_table;
 221     int i;
 222
 223     for (i = 0; i < s->mb_num; i++) {
 224         unsigned int lam = s->lambda_table[s->mb_index2xy[i]];
 225         int qp = (lam * 139 + FF_LAMBDA_SCALE * 64) >> (FF_LAMBDA_SHIFT + 7);
 226         qscale_table[s->mb_index2xy[i]] = av_clip(qp, s->avctx->qmin,
 227                                                   s->avctx->qmax);
 228     }
 229 }
 230
 231 static void update_duplicate_context_after_me(MpegEncContext *dst,
 232                                               MpegEncContext *src)
 233 {
 234 #define COPY(a) dst->a= src->a
 235     COPY(pict_type);
 236     COPY(current_picture);
 237     COPY(f_code);
 238     COPY(b_code);
 239     COPY(qscale);
 240     COPY(lambda);
 241     COPY(lambda2);
 242     COPY(picture_in_gop_number);
 243     COPY(gop_picture_number);
 244     COPY(frame_pred_frame_dct); // FIXME don't set in encode_header
 245     COPY(progressive_frame);    // FIXME don't set in encode_header
 246     COPY(partitioned_frame);    // FIXME don't set in encode_header
 247 #undef COPY
 248 }
 249
 250 /**
 251  * Set the given MpegEncContext to defaults for encoding.
 252  * the changed fields will not depend upon the prior state of the MpegEncContext.
 253  */
 254 static void mpv_encode_defaults(MpegEncContext *s)
 255 {
 256     int i;
 257     ff_mpv_common_defaults(s);
 258
 259     for (i = -16; i < 16; i++) {
 260         default_fcode_tab[i + MAX_MV] = 1;
 261     }
 262     s->me.mv_penalty = default_mv_penalty;
 263     s->fcode_tab     = default_fcode_tab;
 264
 265     s->input_picture_number  = 0;
 266     s->picture_in_gop_number = 0;
 267 }
 268
 269 av_cold int ff_dct_encode_init(MpegEncContext *s)
 270 {
 271     if (ARCH_X86)
 272         ff_dct_encode_init_x86(s);
 273
 274     if (CONFIG_H263_ENCODER)
 275         ff_h263dsp_init(&s->h263dsp);
 276     if (!s->dct_quantize)
 277         s->dct_quantize = ff_dct_quantize_c;
 278     if (!s->denoise_dct)
 279         s->denoise_dct  = denoise_dct_c;
 280     s->fast_dct_quantize = s->dct_quantize;
 281     if (s->avctx->trellis)
 282         s->dct_quantize  = dct_quantize_trellis_c;
 283
 284     return 0;
 285 }
 286
 287 /* init video encoder */
 288 av_cold int ff_mpv_encode_init(AVCodecContext *avctx)
 289 {
 290     MpegEncContext *s = avctx->priv_data;
 291     AVCPBProperties *cpb_props;
 292     int i, ret, format_supported;
 293
 294     mpv_encode_defaults(s);
 295
 296     switch (avctx->codec_id) {
 297     case AV_CODEC_ID_MPEG2VIDEO:
 298         if (avctx->pix_fmt != AV_PIX_FMT_YUV420P &&
 299             avctx->pix_fmt != AV_PIX_FMT_YUV422P) {
 300             av_log(avctx, AV_LOG_ERROR,
 301                    "only YUV420 and YUV422 are supported\n");
 302             return -1;
 303         }
 304         break;
 305     case AV_CODEC_ID_MJPEG:
 306     case AV_CODEC_ID_AMV:
 307         format_supported = 0;
 308         /* JPEG color space */
 309         if (avctx->pix_fmt == AV_PIX_FMT_YUVJ420P ||
 310             avctx->pix_fmt == AV_PIX_FMT_YUVJ422P ||
 311             avctx->pix_fmt == AV_PIX_FMT_YUVJ444P ||
 312             (avctx->color_range == AVCOL_RANGE_JPEG &&
 313              (avctx->pix_fmt == AV_PIX_FMT_YUV420P ||
 314               avctx->pix_fmt == AV_PIX_FMT_YUV422P ||
 315               avctx->pix_fmt == AV_PIX_FMT_YUV444P)))
 316             format_supported = 1;
 317         /* MPEG color space */
 318         else if (avctx->strict_std_compliance <= FF_COMPLIANCE_UNOFFICIAL &&
 319                  (avctx->pix_fmt == AV_PIX_FMT_YUV420P ||
 320                   avctx->pix_fmt == AV_PIX_FMT_YUV422P ||
 321                   avctx->pix_fmt == AV_PIX_FMT_YUV444P))
 322             format_supported = 1;
 323
 324         if (!format_supported) {
 325             av_log(avctx, AV_LOG_ERROR, "colorspace not supported in jpeg\n");
 326             return -1;
 327         }
 328         break;
 329     default:
 330         if (avctx->pix_fmt != AV_PIX_FMT_YUV420P) {
 331             av_log(avctx, AV_LOG_ERROR, "only YUV420 is supported\n");
 332             return -1;
 333         }
 334     }
 335
 336     switch (avctx->pix_fmt) {
 337     case AV_PIX_FMT_YUVJ444P:
 338     case AV_PIX_FMT_YUV444P:
 339         s->chroma_format = CHROMA_444;
 340         break;
 341     case AV_PIX_FMT_YUVJ422P:
 342     case AV_PIX_FMT_YUV422P:
 343         s->chroma_format = CHROMA_422;
 344         break;
 345     case AV_PIX_FMT_YUVJ420P:
 346     case AV_PIX_FMT_YUV420P:
 347     default:
 348         s->chroma_format = CHROMA_420;
 349         break;
 350     }
 351
 352     avctx->bits_per_raw_sample = av_clip(avctx->bits_per_raw_sample, 0, 8);
 353
 354 #if FF_API_PRIVATE_OPT
 355 FF_DISABLE_DEPRECATION_WARNINGS
 356     if (avctx->rtp_payload_size)
 357         s->rtp_payload_size = avctx->rtp_payload_size;
 358     if (avctx->me_penalty_compensation)
 359         s->me_penalty_compensation = avctx->me_penalty_compensation;
 360     if (avctx->pre_me)
 361         s->me_pre = avctx->pre_me;
 362 FF_ENABLE_DEPRECATION_WARNINGS
 363 #endif
 364
 365     s->bit_rate = avctx->bit_rate;
 366     s->width    = avctx->width;
 367     s->height   = avctx->height;
 368     if (avctx->gop_size > 600 &&
 369         avctx->strict_std_compliance > FF_COMPLIANCE_EXPERIMENTAL) {
 370         av_log(avctx, AV_LOG_WARNING,
 371                "keyframe interval too large!, reducing it from %d to %d\n",
 372                avctx->gop_size, 600);
 373         avctx->gop_size = 600;
 374     }
 375     s->gop_size     = avctx->gop_size;
 376     s->avctx        = avctx;
 377     if (avctx->max_b_frames > MAX_B_FRAMES) {
 378         av_log(avctx, AV_LOG_ERROR, "Too many B-frames requested, maximum "
 379                "is %d.\n", MAX_B_FRAMES);
 380         avctx->max_b_frames = MAX_B_FRAMES;
 381     }
 382     s->max_b_frames = avctx->max_b_frames;
 383     s->codec_id     = avctx->codec->id;
 384     s->strict_std_compliance = avctx->strict_std_compliance;
 385     s->quarter_sample     = (avctx->flags & AV_CODEC_FLAG_QPEL) != 0;
 386     s->rtp_mode           = !!s->rtp_payload_size;
 387     s->intra_dc_precision = avctx->intra_dc_precision;
 388
 389     // workaround some differences between how applications specify dc precision
 390     if (s->intra_dc_precision < 0) {
 391         s->intra_dc_precision += 8;
 392     } else if (s->intra_dc_precision >= 8)
 393         s->intra_dc_precision -= 8;
 394
 395     if (s->intra_dc_precision < 0) {
 396         av_log(avctx, AV_LOG_ERROR,
 397                 "intra dc precision must be positive, note some applications use"
 398                 " 0 and some 8 as base meaning 8bit, the value must not be smaller than that\n");
 399         return AVERROR(EINVAL);
 400     }
 401
 402     if (s->intra_dc_precision > (avctx->codec_id == AV_CODEC_ID_MPEG2VIDEO ? 3 : 0)) {
 403         av_log(avctx, AV_LOG_ERROR, "intra dc precision too large\n");
 404         return AVERROR(EINVAL);
 405     }
 406     s->user_specified_pts = AV_NOPTS_VALUE;
 407
 408     if (s->gop_size <= 1) {
 409         s->intra_only = 1;
 410         s->gop_size   = 12;
 411     } else {
 412         s->intra_only = 0;
 413     }
 414
 415 #if FF_API_MOTION_EST
 416 FF_DISABLE_DEPRECATION_WARNINGS
 417     s->me_method = avctx->me_method;
 418 FF_ENABLE_DEPRECATION_WARNINGS
 419 #endif
 420
 421     /* Fixed QSCALE */
 422     s->fixed_qscale = !!(avctx->flags & AV_CODEC_FLAG_QSCALE);
 423
 424 #if FF_API_MPV_OPT
 425     FF_DISABLE_DEPRECATION_WARNINGS
 426     if (avctx->border_masking != 0.0)
 427         s->border_masking = avctx->border_masking;
 428     FF_ENABLE_DEPRECATION_WARNINGS
 429 #endif
 430
 431     s->adaptive_quant = (s->avctx->lumi_masking ||
 432                          s->avctx->dark_masking ||
 433                          s->avctx->temporal_cplx_masking ||
 434                          s->avctx->spatial_cplx_masking  ||
 435                          s->avctx->p_masking      ||
 436                          s->border_masking ||
 437                          (s->mpv_flags & FF_MPV_FLAG_QP_RD)) &&
 438                         !s->fixed_qscale;
 439
 440     s->loop_filter = !!(s->avctx->flags & AV_CODEC_FLAG_LOOP_FILTER);
 441
 442     if (avctx->rc_max_rate && !avctx->rc_buffer_size) {
 443         switch(avctx->codec_id) {
 444         case AV_CODEC_ID_MPEG1VIDEO:
 445         case AV_CODEC_ID_MPEG2VIDEO:
 446             avctx->rc_buffer_size = FFMAX(avctx->rc_max_rate, 15000000) * 112LL / 15000000 * 16384;
 447             break;
 448         case AV_CODEC_ID_MPEG4:
 449         case AV_CODEC_ID_MSMPEG4V1:
 450         case AV_CODEC_ID_MSMPEG4V2:
 451         case AV_CODEC_ID_MSMPEG4V3:
 452             if       (avctx->rc_max_rate >= 15000000) {
 453                 avctx->rc_buffer_size = 320 + (avctx->rc_max_rate - 15000000LL) * (760-320) / (38400000 - 15000000);
 454             } else if(avctx->rc_max_rate >=  2000000) {
 455                 avctx->rc_buffer_size =  80 + (avctx->rc_max_rate -  2000000LL) * (320- 80) / (15000000 -  2000000);
 456             } else if(avctx->rc_max_rate >=   384000) {
 457                 avctx->rc_buffer_size =  40 + (avctx->rc_max_rate -   384000LL) * ( 80- 40) / ( 2000000 -   384000);
 458             } else
 459                 avctx->rc_buffer_size = 40;
 460             avctx->rc_buffer_size *= 16384;
 461             break;
 462         }
 463         if (avctx->rc_buffer_size) {
 464             av_log(avctx, AV_LOG_INFO, "Automatically choosing VBV buffer size of %d kbyte\n", avctx->rc_buffer_size/8192);
 465         }
 466     }
 467
 468     if ((!avctx->rc_max_rate) != (!avctx->rc_buffer_size)) {
 469         av_log(avctx, AV_LOG_ERROR, "Either both buffer size and max rate or neither must be specified\n");
 470         return -1;
 471     }
 472
 473     if (avctx->rc_min_rate && avctx->rc_max_rate != avctx->rc_min_rate) {
 474         av_log(avctx, AV_LOG_INFO,
 475                "Warning min_rate > 0 but min_rate != max_rate isn't recommended!\n");
 476     }
 477
 478     if (avctx->rc_min_rate && avctx->rc_min_rate > avctx->bit_rate) {
 479         av_log(avctx, AV_LOG_ERROR, "bitrate below min bitrate\n");
 480         return -1;
 481     }
 482
 483     if (avctx->rc_max_rate && avctx->rc_max_rate < avctx->bit_rate) {
 484         av_log(avctx, AV_LOG_ERROR, "bitrate above max bitrate\n");
 485         return -1;
 486     }
 487
 488     if (avctx->rc_max_rate &&
 489         avctx->rc_max_rate == avctx->bit_rate &&
 490         avctx->rc_max_rate != avctx->rc_min_rate) {
 491         av_log(avctx, AV_LOG_INFO,
 492                "impossible bitrate constraints, this will fail\n");
 493     }
 494
 495     if (avctx->rc_buffer_size &&
 496         avctx->bit_rate * (int64_t)avctx->time_base.num >
 497             avctx->rc_buffer_size * (int64_t)avctx->time_base.den) {
 498         av_log(avctx, AV_LOG_ERROR, "VBV buffer too small for bitrate\n");
 499         return -1;
 500     }
 501
 502     if (!s->fixed_qscale &&
 503         avctx->bit_rate * av_q2d(avctx->time_base) >
 504             avctx->bit_rate_tolerance) {
 505         av_log(avctx, AV_LOG_WARNING,
 506                "bitrate tolerance %d too small for bitrate %"PRId64", overriding\n", avctx->bit_rate_tolerance, (int64_t)avctx->bit_rate);
 507         avctx->bit_rate_tolerance = 5 * avctx->bit_rate * av_q2d(avctx->time_base);
 508     }
 509
 510     if (s->avctx->rc_max_rate &&
 511         s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
 512         (s->codec_id == AV_CODEC_ID_MPEG1VIDEO ||
 513          s->codec_id == AV_CODEC_ID_MPEG2VIDEO) &&
 514         90000LL * (avctx->rc_buffer_size - 1) >
 515             s->avctx->rc_max_rate * 0xFFFFLL) {
 516         av_log(avctx, AV_LOG_INFO,
 517                "Warning vbv_delay will be set to 0xFFFF (=VBR) as the "
 518                "specified vbv buffer is too large for the given bitrate!\n");
 519     }
 520
 521     if ((s->avctx->flags & AV_CODEC_FLAG_4MV) && s->codec_id != AV_CODEC_ID_MPEG4 &&
 522         s->codec_id != AV_CODEC_ID_H263 && s->codec_id != AV_CODEC_ID_H263P &&
 523         s->codec_id != AV_CODEC_ID_FLV1) {
 524         av_log(avctx, AV_LOG_ERROR, "4MV not supported by codec\n");
 525         return -1;
 526     }
 527
 528     if (s->obmc && s->avctx->mb_decision != FF_MB_DECISION_SIMPLE) {
 529         av_log(avctx, AV_LOG_ERROR,
 530                "OBMC is only supported with simple mb decision\n");
 531         return -1;
 532     }
 533
 534     if (s->quarter_sample && s->codec_id != AV_CODEC_ID_MPEG4) {
 535         av_log(avctx, AV_LOG_ERROR, "qpel not supported by codec\n");
 536         return -1;
 537     }
 538
 539     if (s->max_b_frames                    &&
 540         s->codec_id != AV_CODEC_ID_MPEG4      &&
 541         s->codec_id != AV_CODEC_ID_MPEG1VIDEO &&
 542         s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
 543         av_log(avctx, AV_LOG_ERROR, "B-frames not supported by codec\n");
 544         return -1;
 545     }
 546     if (s->max_b_frames < 0) {
 547         av_log(avctx, AV_LOG_ERROR,
 548                "max b frames must be 0 or positive for mpegvideo based encoders\n");
 549         return -1;
 550     }
 551
 552     if ((s->codec_id == AV_CODEC_ID_MPEG4 ||
 553          s->codec_id == AV_CODEC_ID_H263  ||
 554          s->codec_id == AV_CODEC_ID_H263P) &&
 555         (avctx->sample_aspect_ratio.num > 255 ||
 556          avctx->sample_aspect_ratio.den > 255)) {
 557         av_log(avctx, AV_LOG_WARNING,
 558                "Invalid pixel aspect ratio %i/%i, limit is 255/255 reducing\n",
 559                avctx->sample_aspect_ratio.num, avctx->sample_aspect_ratio.den);
 560         av_reduce(&avctx->sample_aspect_ratio.num, &avctx->sample_aspect_ratio.den,
 561                    avctx->sample_aspect_ratio.num,  avctx->sample_aspect_ratio.den, 255);
 562     }
 563
 564     if ((s->codec_id == AV_CODEC_ID_H263  ||
 565          s->codec_id == AV_CODEC_ID_H263P) &&
 566         (avctx->width  > 2048 ||
 567          avctx->height > 1152 )) {
 568         av_log(avctx, AV_LOG_ERROR, "H.263 does not support resolutions above 2048x1152\n");
 569         return -1;
 570     }
 571     if ((s->codec_id == AV_CODEC_ID_H263  ||
 572          s->codec_id == AV_CODEC_ID_H263P) &&
 573         ((avctx->width &3) ||
 574          (avctx->height&3) )) {
 575         av_log(avctx, AV_LOG_ERROR, "w/h must be a multiple of 4\n");
 576         return -1;
 577     }
 578
 579     if (s->codec_id == AV_CODEC_ID_MPEG1VIDEO &&
 580         (avctx->width  > 4095 ||
 581          avctx->height > 4095 )) {
 582         av_log(avctx, AV_LOG_ERROR, "MPEG-1 does not support resolutions above 4095x4095\n");
 583         return -1;
 584     }
 585
 586     if (s->codec_id == AV_CODEC_ID_MPEG2VIDEO &&
 587         (avctx->width  > 16383 ||
 588          avctx->height > 16383 )) {
 589         av_log(avctx, AV_LOG_ERROR, "MPEG-2 does not support resolutions above 16383x16383\n");
 590         return -1;
 591     }
 592
 593     if (s->codec_id == AV_CODEC_ID_RV10 &&
 594         (avctx->width &15 ||
 595          avctx->height&15 )) {
 596         av_log(avctx, AV_LOG_ERROR, "width and height must be a multiple of 16\n");
 597         return AVERROR(EINVAL);
 598     }
 599
 600     if (s->codec_id == AV_CODEC_ID_RV20 &&
 601         (avctx->width &3 ||
 602          avctx->height&3 )) {
 603         av_log(avctx, AV_LOG_ERROR, "width and height must be a multiple of 4\n");
 604         return AVERROR(EINVAL);
 605     }
 606
 607     if ((s->codec_id == AV_CODEC_ID_WMV1 ||
 608          s->codec_id == AV_CODEC_ID_WMV2) &&
 609          avctx->width & 1) {
 610          av_log(avctx, AV_LOG_ERROR, "width must be multiple of 2\n");
 611          return -1;
 612     }
 613
 614     if ((s->avctx->flags & (AV_CODEC_FLAG_INTERLACED_DCT | AV_CODEC_FLAG_INTERLACED_ME)) &&
 615         s->codec_id != AV_CODEC_ID_MPEG4 && s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
 616         av_log(avctx, AV_LOG_ERROR, "interlacing not supported by codec\n");
 617         return -1;
 618     }
 619
 620 #if FF_API_PRIVATE_OPT
 621     FF_DISABLE_DEPRECATION_WARNINGS
 622     if (avctx->mpeg_quant)
 623         s->mpeg_quant = avctx->mpeg_quant;
 624     FF_ENABLE_DEPRECATION_WARNINGS
 625 #endif
 626
 627     // FIXME mpeg2 uses that too
 628     if (s->mpeg_quant && (   s->codec_id != AV_CODEC_ID_MPEG4
 629                           && s->codec_id != AV_CODEC_ID_MPEG2VIDEO)) {
 630         av_log(avctx, AV_LOG_ERROR,
 631                "mpeg2 style quantization not supported by codec\n");
 632         return -1;
 633     }
 634
 635     if ((s->mpv_flags & FF_MPV_FLAG_CBP_RD) && !avctx->trellis) {
 636         av_log(avctx, AV_LOG_ERROR, "CBP RD needs trellis quant\n");
 637         return -1;
 638     }
 639
 640     if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) &&
 641         s->avctx->mb_decision != FF_MB_DECISION_RD) {
 642         av_log(avctx, AV_LOG_ERROR, "QP RD needs mbd=2\n");
 643         return -1;
 644     }
 645
 646     if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) &&
 647             (s->codec_id == AV_CODEC_ID_AMV ||
 648              s->codec_id == AV_CODEC_ID_MJPEG)) {
 649         // Used to produce garbage with MJPEG.
 650         av_log(avctx, AV_LOG_ERROR,
 651                "QP RD is no longer compatible with MJPEG or AMV\n");
 652         return -1;
 653     }
 654
 655 #if FF_API_PRIVATE_OPT
 656 FF_DISABLE_DEPRECATION_WARNINGS
 657     if (avctx->scenechange_threshold)
 658         s->scenechange_threshold = avctx->scenechange_threshold;
 659 FF_ENABLE_DEPRECATION_WARNINGS
 660 #endif
 661
 662     if (s->scenechange_threshold < 1000000000 &&
 663         (s->avctx->flags & AV_CODEC_FLAG_CLOSED_GOP)) {
 664         av_log(avctx, AV_LOG_ERROR,
 665                "closed gop with scene change detection are not supported yet, "
 666                "set threshold to 1000000000\n");
 667         return -1;
 668     }
 669
 670     if (s->avctx->flags & AV_CODEC_FLAG_LOW_DELAY) {
 671         if (s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
 672             av_log(avctx, AV_LOG_ERROR,
 673                   "low delay forcing is only available for mpeg2\n");
 674             return -1;
 675         }
 676         if (s->max_b_frames != 0) {
 677             av_log(avctx, AV_LOG_ERROR,
 678                    "B-frames cannot be used with low delay\n");
 679             return -1;
 680         }
 681     }
 682
 683     if (s->q_scale_type == 1) {
 684         if (avctx->qmax > 28) {
 685             av_log(avctx, AV_LOG_ERROR,
 686                    "non linear quant only supports qmax <= 28 currently\n");
 687             return -1;
 688         }
 689     }
 690
 691     if (avctx->slices > 1 &&
 692         (avctx->codec_id == AV_CODEC_ID_FLV1 || avctx->codec_id == AV_CODEC_ID_H261)) {
 693         av_log(avctx, AV_LOG_ERROR, "Multiple slices are not supported by this codec\n");
 694         return AVERROR(EINVAL);
 695     }
 696
 697     if (s->avctx->thread_count > 1         &&
 698         s->codec_id != AV_CODEC_ID_MPEG4      &&
 699         s->codec_id != AV_CODEC_ID_MPEG1VIDEO &&
 700         s->codec_id != AV_CODEC_ID_MPEG2VIDEO &&
 701         s->codec_id != AV_CODEC_ID_MJPEG      &&
 702         (s->codec_id != AV_CODEC_ID_H263P)) {
 703         av_log(avctx, AV_LOG_ERROR,
 704                "multi threaded encoding not supported by codec\n");
 705         return -1;
 706     }
 707
 708     if (s->avctx->thread_count < 1) {
 709         av_log(avctx, AV_LOG_ERROR,
 710                "automatic thread number detection not supported by codec, "
 711                "patch welcome\n");
 712         return -1;
 713     }
 714
 715     if (!avctx->time_base.den || !avctx->time_base.num) {
 716         av_log(avctx, AV_LOG_ERROR, "framerate not set\n");
 717         return -1;
 718     }
 719
 720 #if FF_API_PRIVATE_OPT
 721 FF_DISABLE_DEPRECATION_WARNINGS
 722     if (avctx->b_frame_strategy)
 723         s->b_frame_strategy = avctx->b_frame_strategy;
 724     if (avctx->b_sensitivity != 40)
 725         s->b_sensitivity = avctx->b_sensitivity;
 726 FF_ENABLE_DEPRECATION_WARNINGS
 727 #endif
 728
 729     if (s->b_frame_strategy && (avctx->flags & AV_CODEC_FLAG_PASS2)) {
 730         av_log(avctx, AV_LOG_INFO,
 731                "notice: b_frame_strategy only affects the first pass\n");
 732         s->b_frame_strategy = 0;
 733     }
 734
 735     i = av_gcd(avctx->time_base.den, avctx->time_base.num);
 736     if (i > 1) {
 737         av_log(avctx, AV_LOG_INFO, "removing common factors from framerate\n");
 738         avctx->time_base.den /= i;
 739         avctx->time_base.num /= i;
 740         //return -1;
 741     }
 742
 743     if (s->mpeg_quant || s->codec_id == AV_CODEC_ID_MPEG1VIDEO || s->codec_id == AV_CODEC_ID_MPEG2VIDEO || s->codec_id == AV_CODEC_ID_MJPEG || s->codec_id==AV_CODEC_ID_AMV) {
 744         // (a + x * 3 / 8) / x
 745         s->intra_quant_bias = 3 << (QUANT_BIAS_SHIFT - 3);
 746         s->inter_quant_bias = 0;
 747     } else {
 748         s->intra_quant_bias = 0;
 749         // (a - x / 4) / x
 750         s->inter_quant_bias = -(1 << (QUANT_BIAS_SHIFT - 2));
 751     }
 752
 753     if (avctx->qmin > avctx->qmax || avctx->qmin <= 0) {
 754         av_log(avctx, AV_LOG_ERROR, "qmin and or qmax are invalid, they must be 0 < min <= max\n");
 755         return AVERROR(EINVAL);
 756     }
 757
 758 #if FF_API_QUANT_BIAS
 759 FF_DISABLE_DEPRECATION_WARNINGS
 760     if (avctx->intra_quant_bias != FF_DEFAULT_QUANT_BIAS)
 761         s->intra_quant_bias = avctx->intra_quant_bias;
 762     if (avctx->inter_quant_bias != FF_DEFAULT_QUANT_BIAS)
 763         s->inter_quant_bias = avctx->inter_quant_bias;
 764 FF_ENABLE_DEPRECATION_WARNINGS
 765 #endif
 766
 767     av_log(avctx, AV_LOG_DEBUG, "intra_quant_bias = %d inter_quant_bias = %d\n",s->intra_quant_bias,s->inter_quant_bias);
 768
 769     if (avctx->codec_id == AV_CODEC_ID_MPEG4 &&
 770         s->avctx->time_base.den > (1 << 16) - 1) {
 771         av_log(avctx, AV_LOG_ERROR,
 772                "timebase %d/%d not supported by MPEG 4 standard, "
 773                "the maximum admitted value for the timebase denominator "
 774                "is %d\n", s->avctx->time_base.num, s->avctx->time_base.den,
 775                (1 << 16) - 1);
 776         return -1;
 777     }
 778     s->time_increment_bits = av_log2(s->avctx->time_base.den - 1) + 1;
 779
 780     switch (avctx->codec->id) {
 781     case AV_CODEC_ID_MPEG1VIDEO:
 782         s->out_format = FMT_MPEG1;
 783         s->low_delay  = !!(s->avctx->flags & AV_CODEC_FLAG_LOW_DELAY);
 784         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
 785         break;
 786     case AV_CODEC_ID_MPEG2VIDEO:
 787         s->out_format = FMT_MPEG1;
 788         s->low_delay  = !!(s->avctx->flags & AV_CODEC_FLAG_LOW_DELAY);
 789         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
 790         s->rtp_mode   = 1;
 791         break;
 792     case AV_CODEC_ID_MJPEG:
 793     case AV_CODEC_ID_AMV:
 794         s->out_format = FMT_MJPEG;
 795         s->intra_only = 1; /* force intra only for jpeg */
 796         if (!CONFIG_MJPEG_ENCODER ||
 797             ff_mjpeg_encode_init(s) < 0)
 798             return -1;
 799         avctx->delay = 0;
 800         s->low_delay = 1;
 801         break;
 802     case AV_CODEC_ID_H261:
 803         if (!CONFIG_H261_ENCODER)
 804             return -1;
 805         if (ff_h261_get_picture_format(s->width, s->height) < 0) {
 806             av_log(avctx, AV_LOG_ERROR,
 807                    "The specified picture size of %dx%d is not valid for the "
 808                    "H.261 codec.\nValid sizes are 176x144, 352x288\n",
 809                     s->width, s->height);
 810             return -1;
 811         }
 812         s->out_format = FMT_H261;
 813         avctx->delay  = 0;
 814         s->low_delay  = 1;
 815         s->rtp_mode   = 0; /* Sliced encoding not supported */
 816         break;
 817     case AV_CODEC_ID_H263:
 818         if (!CONFIG_H263_ENCODER)
 819             return -1;
 820         if (ff_match_2uint16(ff_h263_format, FF_ARRAY_ELEMS(ff_h263_format),
 821                              s->width, s->height) == 8) {
 822             av_log(avctx, AV_LOG_ERROR,
 823                    "The specified picture size of %dx%d is not valid for "
 824                    "the H.263 codec.\nValid sizes are 128x96, 176x144, "
 825                    "352x288, 704x576, and 1408x1152. "
 826                    "Try H.263+.\n", s->width, s->height);
 827             return -1;
 828         }
 829         s->out_format = FMT_H263;
 830         avctx->delay  = 0;
 831         s->low_delay  = 1;
 832         break;
 833     case AV_CODEC_ID_H263P:
 834         s->out_format = FMT_H263;
 835         s->h263_plus  = 1;
 836         /* Fx */
 837         s->h263_aic        = (avctx->flags & AV_CODEC_FLAG_AC_PRED) ? 1 : 0;
 838         s->modified_quant  = s->h263_aic;
 839         s->loop_filter     = (avctx->flags & AV_CODEC_FLAG_LOOP_FILTER) ? 1 : 0;
 840         s->unrestricted_mv = s->obmc || s->loop_filter || s->umvplus;
 841
 842         /* /Fx */
 843         /* These are just to be sure */
 844         avctx->delay = 0;
 845         s->low_delay = 1;
 846         break;
 847     case AV_CODEC_ID_FLV1:
 848         s->out_format      = FMT_H263;
 849         s->h263_flv        = 2; /* format = 1; 11-bit codes */
 850         s->unrestricted_mv = 1;
 851         s->rtp_mode  = 0; /* don't allow GOB */
 852         avctx->delay = 0;
 853         s->low_delay = 1;
 854         break;
 855     case AV_CODEC_ID_RV10:
 856         s->out_format = FMT_H263;
 857         avctx->delay  = 0;
 858         s->low_delay  = 1;
 859         break;
 860     case AV_CODEC_ID_RV20:
 861         s->out_format      = FMT_H263;
 862         avctx->delay       = 0;
 863         s->low_delay       = 1;
 864         s->modified_quant  = 1;
 865         s->h263_aic        = 1;
 866         s->h263_plus       = 1;
 867         s->loop_filter     = 1;
 868         s->unrestricted_mv = 0;
 869         break;
 870     case AV_CODEC_ID_MPEG4:
 871         s->out_format      = FMT_H263;
 872         s->h263_pred       = 1;
 873         s->unrestricted_mv = 1;
 874         s->low_delay       = s->max_b_frames ? 0 : 1;
 875         avctx->delay       = s->low_delay ? 0 : (s->max_b_frames + 1);
 876         break;
 877     case AV_CODEC_ID_MSMPEG4V2:
 878         s->out_format      = FMT_H263;
 879         s->h263_pred       = 1;
 880         s->unrestricted_mv = 1;
 881         s->msmpeg4_version = 2;
 882         avctx->delay       = 0;
 883         s->low_delay       = 1;
 884         break;
 885     case AV_CODEC_ID_MSMPEG4V3:
 886         s->out_format        = FMT_H263;
 887         s->h263_pred         = 1;
 888         s->unrestricted_mv   = 1;
 889         s->msmpeg4_version   = 3;
 890         s->flipflop_rounding = 1;
 891         avctx->delay         = 0;
 892         s->low_delay         = 1;
 893         break;
 894     case AV_CODEC_ID_WMV1:
 895         s->out_format        = FMT_H263;
 896         s->h263_pred         = 1;
 897         s->unrestricted_mv   = 1;
 898         s->msmpeg4_version   = 4;
 899         s->flipflop_rounding = 1;
 900         avctx->delay         = 0;
 901         s->low_delay         = 1;
 902         break;
 903     case AV_CODEC_ID_WMV2:
 904         s->out_format        = FMT_H263;
 905         s->h263_pred         = 1;
 906         s->unrestricted_mv   = 1;
 907         s->msmpeg4_version   = 5;
 908         s->flipflop_rounding = 1;
 909         avctx->delay         = 0;
 910         s->low_delay         = 1;
 911         break;
 912     default:
 913         return -1;
 914     }
 915
 916 #if FF_API_PRIVATE_OPT
 917     FF_DISABLE_DEPRECATION_WARNINGS
 918     if (avctx->noise_reduction)
 919         s->noise_reduction = avctx->noise_reduction;
 920     FF_ENABLE_DEPRECATION_WARNINGS
 921 #endif
 922
 923     avctx->has_b_frames = !s->low_delay;
 924
 925     s->encoding = 1;
 926
 927     s->progressive_frame    =
 928     s->progressive_sequence = !(avctx->flags & (AV_CODEC_FLAG_INTERLACED_DCT |
 929                                                 AV_CODEC_FLAG_INTERLACED_ME) ||
 930                                 s->alternate_scan);
 931
 932     /* init */
 933     ff_mpv_idct_init(s);
 934     if (ff_mpv_common_init(s) < 0)
 935         return -1;
 936
 937     ff_fdctdsp_init(&s->fdsp, avctx);
 938     ff_me_cmp_init(&s->mecc, avctx);
 939     ff_mpegvideoencdsp_init(&s->mpvencdsp, avctx);
 940     ff_pixblockdsp_init(&s->pdsp, avctx);
 941     ff_qpeldsp_init(&s->qdsp);
 942
 943     if (s->msmpeg4_version) {
 944         FF_ALLOCZ_OR_GOTO(s->avctx, s->ac_stats,
 945                           2 * 2 * (MAX_LEVEL + 1) *
 946                           (MAX_RUN + 1) * 2 * sizeof(int), fail);
 947     }
 948     FF_ALLOCZ_OR_GOTO(s->avctx, s->avctx->stats_out, 256, fail);
 949
 950     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_intra_matrix,   64 * 32 * sizeof(int), fail);
 951     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_chroma_intra_matrix, 64 * 32 * sizeof(int), fail);
 952     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_inter_matrix,   64 * 32 * sizeof(int), fail);
 953     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_intra_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
 954     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_chroma_intra_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
 955     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_inter_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
 956     FF_ALLOCZ_OR_GOTO(s->avctx, s->input_picture,
 957                       MAX_PICTURE_COUNT * sizeof(Picture *), fail);
 958     FF_ALLOCZ_OR_GOTO(s->avctx, s->reordered_input_picture,
 959                       MAX_PICTURE_COUNT * sizeof(Picture *), fail);
 960
 961
 962     if (s->noise_reduction) {
 963         FF_ALLOCZ_OR_GOTO(s->avctx, s->dct_offset,
 964                           2 * 64 * sizeof(uint16_t), fail);
 965     }
 966
 967     ff_dct_encode_init(s);
 968
 969     if ((CONFIG_H263P_ENCODER || CONFIG_RV20_ENCODER) && s->modified_quant)
 970         s->chroma_qscale_table = ff_h263_chroma_qscale_table;
 971
 972     if (s->slice_context_count > 1) {
 973         s->rtp_mode = 1;
 974
 975         if (avctx->codec_id == AV_CODEC_ID_H263P)
 976             s->h263_slice_structured = 1;
 977     }
 978
 979     s->quant_precision = 5;
 980
 981 #if FF_API_PRIVATE_OPT
 982 FF_DISABLE_DEPRECATION_WARNINGS
 983     if (avctx->frame_skip_threshold)
 984         s->frame_skip_threshold = avctx->frame_skip_threshold;
 985     if (avctx->frame_skip_factor)
 986         s->frame_skip_factor = avctx->frame_skip_factor;
 987     if (avctx->frame_skip_exp)
 988         s->frame_skip_exp = avctx->frame_skip_exp;
 989     if (avctx->frame_skip_cmp != FF_CMP_DCTMAX)
 990         s->frame_skip_cmp = avctx->frame_skip_cmp;
 991 FF_ENABLE_DEPRECATION_WARNINGS
 992 #endif
 993
 994     ff_set_cmp(&s->mecc, s->mecc.ildct_cmp,      s->avctx->ildct_cmp);
 995     ff_set_cmp(&s->mecc, s->mecc.frame_skip_cmp, s->frame_skip_cmp);
 996
 997     if (CONFIG_H261_ENCODER && s->out_format == FMT_H261)
 998         ff_h261_encode_init(s);
 999     if (CONFIG_H263_ENCODER && s->out_format == FMT_H263)
1000         ff_h263_encode_init(s);
1001     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
1002         if ((ret = ff_msmpeg4_encode_init(s)) < 0)
1003             return ret;
1004     if ((CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
1005         && s->out_format == FMT_MPEG1)
1006         ff_mpeg1_encode_init(s);
1007
1008     /* init q matrix */
1009     for (i = 0; i < 64; i++) {
1010         int j = s->idsp.idct_permutation[i];
1011         if (CONFIG_MPEG4_ENCODER && s->codec_id == AV_CODEC_ID_MPEG4 &&
1012             s->mpeg_quant) {
1013             s->intra_matrix[j] = ff_mpeg4_default_intra_matrix[i];
1014             s->inter_matrix[j] = ff_mpeg4_default_non_intra_matrix[i];
1015         } else if (s->out_format == FMT_H263 || s->out_format == FMT_H261) {
1016             s->intra_matrix[j] =
1017             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
1018         } else {
1019             /* MPEG-1/2 */
1020             s->chroma_intra_matrix[j] =
1021             s->intra_matrix[j] = ff_mpeg1_default_intra_matrix[i];
1022             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
1023         }
1024         if (s->avctx->intra_matrix)
1025             s->intra_matrix[j] = s->avctx->intra_matrix[i];
1026         if (s->avctx->inter_matrix)
1027             s->inter_matrix[j] = s->avctx->inter_matrix[i];
1028     }
1029
1030     /* precompute matrix */
1031     /* for mjpeg, we do include qscale in the matrix */
1032     if (s->out_format != FMT_MJPEG) {
1033         ff_convert_matrix(s, s->q_intra_matrix, s->q_intra_matrix16,
1034                           s->intra_matrix, s->intra_quant_bias, avctx->qmin,
1035                           31, 1);
1036         ff_convert_matrix(s, s->q_inter_matrix, s->q_inter_matrix16,
1037                           s->inter_matrix, s->inter_quant_bias, avctx->qmin,
1038                           31, 0);
1039     }
1040
1041 #if FF_API_RC_STRATEGY
1042 FF_DISABLE_DEPRECATION_WARNINGS
1043     if (!s->rc_strategy)
1044         s->rc_strategy = s->avctx->rc_strategy;
1045 FF_ENABLE_DEPRECATION_WARNINGS
1046 #endif
1047
1048     if (ff_rate_control_init(s) < 0)
1049         return -1;
1050
1051 #if FF_API_RC_STRATEGY
1052     av_assert0(MPV_RC_STRATEGY_XVID == FF_RC_STRATEGY_XVID);
1053 #endif
1054
1055     if ((s->avctx->flags & AV_CODEC_FLAG_PASS2) && s->rc_strategy == MPV_RC_STRATEGY_XVID) {
1056 #if CONFIG_LIBXVID
1057         ret = ff_xvid_rate_control_init(s);
1058 #else
1059         ret = AVERROR(ENOSYS);
1060         av_log(s->avctx, AV_LOG_ERROR,
1061                "Xvid ratecontrol requires libavcodec compiled with Xvid support.\n");
1062 #endif
1063         if (ret < 0)
1064             return ret;
1065     }
1066
1067 #if FF_API_ERROR_RATE
1068     FF_DISABLE_DEPRECATION_WARNINGS
1069     if (avctx->error_rate)
1070         s->error_rate = avctx->error_rate;
1071     FF_ENABLE_DEPRECATION_WARNINGS;
1072 #endif
1073
1074 #if FF_API_NORMALIZE_AQP
1075     FF_DISABLE_DEPRECATION_WARNINGS
1076     if (avctx->flags & CODEC_FLAG_NORMALIZE_AQP)
1077         s->mpv_flags |= FF_MPV_FLAG_NAQ;
1078     FF_ENABLE_DEPRECATION_WARNINGS;
1079 #endif
1080
1081 #if FF_API_MV0
1082     FF_DISABLE_DEPRECATION_WARNINGS
1083     if (avctx->flags & CODEC_FLAG_MV0)
1084         s->mpv_flags |= FF_MPV_FLAG_MV0;
1085     FF_ENABLE_DEPRECATION_WARNINGS
1086 #endif
1087
1088 #if FF_API_MPV_OPT
1089     FF_DISABLE_DEPRECATION_WARNINGS
1090     if (avctx->rc_qsquish != 0.0)
1091         s->rc_qsquish = avctx->rc_qsquish;
1092     if (avctx->rc_qmod_amp != 0.0)
1093         s->rc_qmod_amp = avctx->rc_qmod_amp;
1094     if (avctx->rc_qmod_freq)
1095         s->rc_qmod_freq = avctx->rc_qmod_freq;
1096     if (avctx->rc_buffer_aggressivity != 1.0)
1097         s->rc_buffer_aggressivity = avctx->rc_buffer_aggressivity;
1098     if (avctx->rc_initial_cplx != 0.0)
1099         s->rc_initial_cplx = avctx->rc_initial_cplx;
1100     if (avctx->lmin)
1101         s->lmin = avctx->lmin;
1102     if (avctx->lmax)
1103         s->lmax = avctx->lmax;
1104
1105     if (avctx->rc_eq) {
1106         av_freep(&s->rc_eq);
1107         s->rc_eq = av_strdup(avctx->rc_eq);
1108         if (!s->rc_eq)
1109             return AVERROR(ENOMEM);
1110     }
1111     FF_ENABLE_DEPRECATION_WARNINGS
1112 #endif
1113
1114 #if FF_API_PRIVATE_OPT
1115     FF_DISABLE_DEPRECATION_WARNINGS
1116     if (avctx->brd_scale)
1117         s->brd_scale = avctx->brd_scale;
1118
1119     if (avctx->prediction_method)
1120         s->pred = avctx->prediction_method + 1;
1121     FF_ENABLE_DEPRECATION_WARNINGS
1122 #endif
1123
1124     if (s->b_frame_strategy == 2) {
1125         for (i = 0; i < s->max_b_frames + 2; i++) {
1126             s->tmp_frames[i] = av_frame_alloc();
1127             if (!s->tmp_frames[i])
1128                 return AVERROR(ENOMEM);
1129
1130             s->tmp_frames[i]->format = AV_PIX_FMT_YUV420P;
1131             s->tmp_frames[i]->width  = s->width  >> s->brd_scale;
1132             s->tmp_frames[i]->height = s->height >> s->brd_scale;
1133
1134             ret = av_frame_get_buffer(s->tmp_frames[i], 32);
1135             if (ret < 0)
1136                 return ret;
1137         }
1138     }
1139
1140     cpb_props = ff_add_cpb_side_data(avctx);
1141     if (!cpb_props)
1142         return AVERROR(ENOMEM);
1143     cpb_props->max_bitrate = avctx->rc_max_rate;
1144     cpb_props->min_bitrate = avctx->rc_min_rate;
1145     cpb_props->avg_bitrate = avctx->bit_rate;
1146     cpb_props->buffer_size = avctx->rc_buffer_size;
1147
1148     return 0;
1149 fail:
1150     ff_mpv_encode_end(avctx);
1151     return AVERROR_UNKNOWN;
1152 }
1153
1154 av_cold int ff_mpv_encode_end(AVCodecContext *avctx)
1155 {
1156     MpegEncContext *s = avctx->priv_data;
1157     int i;
1158
1159     ff_rate_control_uninit(s);
1160 #if CONFIG_LIBXVID
1161     if ((avctx->flags & AV_CODEC_FLAG_PASS2) && s->rc_strategy == MPV_RC_STRATEGY_XVID)
1162         ff_xvid_rate_control_uninit(s);
1163 #endif
1164
1165     ff_mpv_common_end(s);
1166     if (CONFIG_MJPEG_ENCODER &&
1167         s->out_format == FMT_MJPEG)
1168         ff_mjpeg_encode_close(s);
1169
1170     av_freep(&avctx->extradata);
1171
1172     for (i = 0; i < FF_ARRAY_ELEMS(s->tmp_frames); i++)
1173         av_frame_free(&s->tmp_frames[i]);
1174
1175     ff_free_picture_tables(&s->new_picture);
1176     ff_mpeg_unref_picture(s->avctx, &s->new_picture);
1177
1178     av_freep(&s->avctx->stats_out);
1179     av_freep(&s->ac_stats);
1180
1181     if(s->q_chroma_intra_matrix   != s->q_intra_matrix  ) av_freep(&s->q_chroma_intra_matrix);
1182     if(s->q_chroma_intra_matrix16 != s->q_intra_matrix16) av_freep(&s->q_chroma_intra_matrix16);
1183     s->q_chroma_intra_matrix=   NULL;
1184     s->q_chroma_intra_matrix16= NULL;
1185     av_freep(&s->q_intra_matrix);
1186     av_freep(&s->q_inter_matrix);
1187     av_freep(&s->q_intra_matrix16);
1188     av_freep(&s->q_inter_matrix16);
1189     av_freep(&s->input_picture);
1190     av_freep(&s->reordered_input_picture);
1191     av_freep(&s->dct_offset);
1192
1193     return 0;
1194 }
1195
1196 static int get_sae(uint8_t *src, int ref, int stride)
1197 {
1198     int x,y;
1199     int acc = 0;
1200
1201     for (y = 0; y < 16; y++) {
1202         for (x = 0; x < 16; x++) {
1203             acc += FFABS(src[x + y * stride] - ref);
1204         }
1205     }
1206
1207     return acc;
1208 }
1209
1210 static int get_intra_count(MpegEncContext *s, uint8_t *src,
1211                            uint8_t *ref, int stride)
1212 {
1213     int x, y, w, h;
1214     int acc = 0;
1215
1216     w = s->width  & ~15;
1217     h = s->height & ~15;
1218
1219     for (y = 0; y < h; y += 16) {
1220         for (x = 0; x < w; x += 16) {
1221             int offset = x + y * stride;
1222             int sad  = s->mecc.sad[0](NULL, src + offset, ref + offset,
1223                                       stride, 16);
1224             int mean = (s->mpvencdsp.pix_sum(src + offset, stride) + 128) >> 8;
1225             int sae  = get_sae(src + offset, mean, stride);
1226
1227             acc += sae + 500 < sad;
1228         }
1229     }
1230     return acc;
1231 }
1232
1233 static int alloc_picture(MpegEncContext *s, Picture *pic, int shared)
1234 {
1235     return ff_alloc_picture(s->avctx, pic, &s->me, &s->sc, shared, 1,
1236                             s->chroma_x_shift, s->chroma_y_shift, s->out_format,
1237                             s->mb_stride, s->mb_width, s->mb_height, s->b8_stride,
1238                             &s->linesize, &s->uvlinesize);
1239 }
1240
1241 static int load_input_picture(MpegEncContext *s, const AVFrame *pic_arg)
1242 {
1243     Picture *pic = NULL;
1244     int64_t pts;
1245     int i, display_picture_number = 0, ret;
1246     int encoding_delay = s->max_b_frames ? s->max_b_frames
1247                                          : (s->low_delay ? 0 : 1);
1248     int flush_offset = 1;
1249     int direct = 1;
1250
1251     if (pic_arg) {
1252         pts = pic_arg->pts;
1253         display_picture_number = s->input_picture_number++;
1254
1255         if (pts != AV_NOPTS_VALUE) {
1256             if (s->user_specified_pts != AV_NOPTS_VALUE) {
1257                 int64_t last = s->user_specified_pts;
1258
1259                 if (pts <= last) {
1260                     av_log(s->avctx, AV_LOG_ERROR,
1261                            "Invalid pts (%"PRId64") <= last (%"PRId64")\n",
1262                            pts, last);
1263                     return AVERROR(EINVAL);
1264                 }
1265
1266                 if (!s->low_delay && display_picture_number == 1)
1267                     s->dts_delta = pts - last;
1268             }
1269             s->user_specified_pts = pts;
1270         } else {
1271             if (s->user_specified_pts != AV_NOPTS_VALUE) {
1272                 s->user_specified_pts =
1273                 pts = s->user_specified_pts + 1;
1274                 av_log(s->avctx, AV_LOG_INFO,
1275                        "Warning: AVFrame.pts=? trying to guess (%"PRId64")\n",
1276                        pts);
1277             } else {
1278                 pts = display_picture_number;
1279             }
1280         }
1281
1282         if (!pic_arg->buf[0] ||
1283             pic_arg->linesize[0] != s->linesize ||
1284             pic_arg->linesize[1] != s->uvlinesize ||
1285             pic_arg->linesize[2] != s->uvlinesize)
1286             direct = 0;
1287         if ((s->width & 15) || (s->height & 15))
1288             direct = 0;
1289         if (((intptr_t)(pic_arg->data[0])) & (STRIDE_ALIGN-1))
1290             direct = 0;
1291         if (s->linesize & (STRIDE_ALIGN-1))
1292             direct = 0;
1293
1294         ff_dlog(s->avctx, "%d %d %"PTRDIFF_SPECIFIER" %"PTRDIFF_SPECIFIER"\n", pic_arg->linesize[0],
1295                 pic_arg->linesize[1], s->linesize, s->uvlinesize);
1296
1297         i = ff_find_unused_picture(s->avctx, s->picture, direct);
1298         if (i < 0)
1299             return i;
1300
1301         pic = &s->picture[i];
1302         pic->reference = 3;
1303
1304         if (direct) {
1305             if ((ret = av_frame_ref(pic->f, pic_arg)) < 0)
1306                 return ret;
1307         }
1308         ret = alloc_picture(s, pic, direct);
1309         if (ret < 0)
1310             return ret;
1311
1312         if (!direct) {
1313             if (pic->f->data[0] + INPLACE_OFFSET == pic_arg->data[0] &&
1314                 pic->f->data[1] + INPLACE_OFFSET == pic_arg->data[1] &&
1315                 pic->f->data[2] + INPLACE_OFFSET == pic_arg->data[2]) {
1316                 // empty
1317             } else {
1318                 int h_chroma_shift, v_chroma_shift;
1319                 av_pix_fmt_get_chroma_sub_sample(s->avctx->pix_fmt,
1320                                                  &h_chroma_shift,
1321                                                  &v_chroma_shift);
1322
1323                 for (i = 0; i < 3; i++) {
1324                     int src_stride = pic_arg->linesize[i];
1325                     int dst_stride = i ? s->uvlinesize : s->linesize;
1326                     int h_shift = i ? h_chroma_shift : 0;
1327                     int v_shift = i ? v_chroma_shift : 0;
1328                     int w = s->width  >> h_shift;
1329                     int h = s->height >> v_shift;
1330                     uint8_t *src = pic_arg->data[i];
1331                     uint8_t *dst = pic->f->data[i];
1332                     int vpad = 16;
1333
1334                     if (   s->codec_id == AV_CODEC_ID_MPEG2VIDEO
1335                         && !s->progressive_sequence
1336                         && FFALIGN(s->height, 32) - s->height > 16)
1337                         vpad = 32;
1338
1339                     if (!s->avctx->rc_buffer_size)
1340                         dst += INPLACE_OFFSET;
1341
1342                     if (src_stride == dst_stride)
1343                         memcpy(dst, src, src_stride * h);
1344                     else {
1345                         int h2 = h;
1346                         uint8_t *dst2 = dst;
1347                         while (h2--) {
1348                             memcpy(dst2, src, w);
1349                             dst2 += dst_stride;
1350                             src += src_stride;
1351                         }
1352                     }
1353                     if ((s->width & 15) || (s->height & (vpad-1))) {
1354                         s->mpvencdsp.draw_edges(dst, dst_stride,
1355                                                 w, h,
1356                                                 16 >> h_shift,
1357                                                 vpad >> v_shift,
1358                                                 EDGE_BOTTOM);
1359                     }
1360                 }
1361                 emms_c();
1362             }
1363         }
1364         ret = av_frame_copy_props(pic->f, pic_arg);
1365         if (ret < 0)
1366             return ret;
1367
1368         pic->f->display_picture_number = display_picture_number;
1369         pic->f->pts = pts; // we set this here to avoid modifying pic_arg
1370     } else {
1371         /* Flushing: When we have not received enough input frames,
1372          * ensure s->input_picture[0] contains the first picture */
1373         for (flush_offset = 0; flush_offset < encoding_delay + 1; flush_offset++)
1374             if (s->input_picture[flush_offset])
1375                 break;
1376
1377         if (flush_offset <= 1)
1378             flush_offset = 1;
1379         else
1380             encoding_delay = encoding_delay - flush_offset + 1;
1381     }
1382
1383     /* shift buffer entries */
1384     for (i = flush_offset; i < MAX_PICTURE_COUNT /*s->encoding_delay + 1*/; i++)
1385         s->input_picture[i - flush_offset] = s->input_picture[i];
1386
1387     s->input_picture[encoding_delay] = (Picture*) pic;
1388
1389     return 0;
1390 }
1391
1392 static int skip_check(MpegEncContext *s, Picture *p, Picture *ref)
1393 {
1394     int x, y, plane;
1395     int score = 0;
1396     int64_t score64 = 0;
1397
1398     for (plane = 0; plane < 3; plane++) {
1399         const int stride = p->f->linesize[plane];
1400         const int bw = plane ? 1 : 2;
1401         for (y = 0; y < s->mb_height * bw; y++) {
1402             for (x = 0; x < s->mb_width * bw; x++) {
1403                 int off = p->shared ? 0 : 16;
1404                 uint8_t *dptr = p->f->data[plane] + 8 * (x + y * stride) + off;
1405                 uint8_t *rptr = ref->f->data[plane] + 8 * (x + y * stride);
1406                 int v = s->mecc.frame_skip_cmp[1](s, dptr, rptr, stride, 8);
1407
1408                 switch (FFABS(s->frame_skip_exp)) {
1409                 case 0: score    =  FFMAX(score, v);          break;
1410                 case 1: score   += FFABS(v);                  break;
1411                 case 2: score64 += v * (int64_t)v;                       break;
1412                 case 3: score64 += FFABS(v * (int64_t)v * v);            break;
1413                 case 4: score64 += (v * (int64_t)v) * (v * (int64_t)v);  break;
1414                 }
1415             }
1416         }
1417     }
1418     emms_c();
1419
1420     if (score)
1421         score64 = score;
1422     if (s->frame_skip_exp < 0)
1423         score64 = pow(score64 / (double)(s->mb_width * s->mb_height),
1424                       -1.0/s->frame_skip_exp);
1425
1426     if (score64 < s->frame_skip_threshold)
1427         return 1;
1428     if (score64 < ((s->frame_skip_factor * (int64_t) s->lambda) >> 8))
1429         return 1;
1430     return 0;
1431 }
1432
1433 static int encode_frame(AVCodecContext *c, AVFrame *frame)
1434 {
1435     AVPacket pkt = { 0 };
1436     int ret;
1437     int size = 0;
1438
1439     av_init_packet(&pkt);
1440
1441     ret = avcodec_send_frame(c, frame);
1442     if (ret < 0)
1443         return ret;
1444
1445     do {
1446         ret = avcodec_receive_packet(c, &pkt);
1447         if (ret >= 0) {
1448             size += pkt.size;
1449             av_packet_unref(&pkt);
1450         } else if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF)
1451             return ret;
1452     } while (ret >= 0);
1453
1454     return size;
1455 }
1456
1457 static int estimate_best_b_count(MpegEncContext *s)
1458 {
1459     const AVCodec *codec = avcodec_find_encoder(s->avctx->codec_id);
1460     const int scale = s->brd_scale;
1461     int width  = s->width  >> scale;
1462     int height = s->height >> scale;
1463     int i, j, out_size, p_lambda, b_lambda, lambda2;
1464     int64_t best_rd  = INT64_MAX;
1465     int best_b_count = -1;
1466     int ret = 0;
1467
1468     av_assert0(scale >= 0 && scale <= 3);
1469
1470     //emms_c();
1471     //s->next_picture_ptr->quality;
1472     p_lambda = s->last_lambda_for[AV_PICTURE_TYPE_P];
1473     //p_lambda * FFABS(s->avctx->b_quant_factor) + s->avctx->b_quant_offset;
1474     b_lambda = s->last_lambda_for[AV_PICTURE_TYPE_B];
1475     if (!b_lambda) // FIXME we should do this somewhere else
1476         b_lambda = p_lambda;
1477     lambda2  = (b_lambda * b_lambda + (1 << FF_LAMBDA_SHIFT) / 2) >>
1478                FF_LAMBDA_SHIFT;
1479
1480     for (i = 0; i < s->max_b_frames + 2; i++) {
1481         Picture pre_input, *pre_input_ptr = i ? s->input_picture[i - 1] :
1482                                                 s->next_picture_ptr;
1483         uint8_t *data[4];
1484
1485         if (pre_input_ptr && (!i || s->input_picture[i - 1])) {
1486             pre_input = *pre_input_ptr;
1487             memcpy(data, pre_input_ptr->f->data, sizeof(data));
1488
1489             if (!pre_input.shared && i) {
1490                 data[0] += INPLACE_OFFSET;
1491                 data[1] += INPLACE_OFFSET;
1492                 data[2] += INPLACE_OFFSET;
1493             }
1494
1495             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[0],
1496                                        s->tmp_frames[i]->linesize[0],
1497                                        data[0],
1498                                        pre_input.f->linesize[0],
1499                                        width, height);
1500             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[1],
1501                                        s->tmp_frames[i]->linesize[1],
1502                                        data[1],
1503                                        pre_input.f->linesize[1],
1504                                        width >> 1, height >> 1);
1505             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[2],
1506                                        s->tmp_frames[i]->linesize[2],
1507                                        data[2],
1508                                        pre_input.f->linesize[2],
1509                                        width >> 1, height >> 1);
1510         }
1511     }
1512
1513     for (j = 0; j < s->max_b_frames + 1; j++) {
1514         AVCodecContext *c;
1515         int64_t rd = 0;
1516
1517         if (!s->input_picture[j])
1518             break;
1519
1520         c = avcodec_alloc_context3(NULL);
1521         if (!c)
1522             return AVERROR(ENOMEM);
1523
1524         c->width        = width;
1525         c->height       = height;
1526         c->flags        = AV_CODEC_FLAG_QSCALE | AV_CODEC_FLAG_PSNR;
1527         c->flags       |= s->avctx->flags & AV_CODEC_FLAG_QPEL;
1528         c->mb_decision  = s->avctx->mb_decision;
1529         c->me_cmp       = s->avctx->me_cmp;
1530         c->mb_cmp       = s->avctx->mb_cmp;
1531         c->me_sub_cmp   = s->avctx->me_sub_cmp;
1532         c->pix_fmt      = AV_PIX_FMT_YUV420P;
1533         c->time_base    = s->avctx->time_base;
1534         c->max_b_frames = s->max_b_frames;
1535
1536         ret = avcodec_open2(c, codec, NULL);
1537         if (ret < 0)
1538             goto fail;
1539
1540         s->tmp_frames[0]->pict_type = AV_PICTURE_TYPE_I;
1541         s->tmp_frames[0]->quality   = 1 * FF_QP2LAMBDA;
1542
1543         out_size = encode_frame(c, s->tmp_frames[0]);
1544         if (out_size < 0) {
1545             ret = out_size;
1546             goto fail;
1547         }
1548
1549         //rd += (out_size * lambda2) >> FF_LAMBDA_SHIFT;
1550
1551         for (i = 0; i < s->max_b_frames + 1; i++) {
1552             int is_p = i % (j + 1) == j || i == s->max_b_frames;
1553
1554             s->tmp_frames[i + 1]->pict_type = is_p ?
1555                                      AV_PICTURE_TYPE_P : AV_PICTURE_TYPE_B;
1556             s->tmp_frames[i + 1]->quality   = is_p ? p_lambda : b_lambda;
1557
1558             out_size = encode_frame(c, s->tmp_frames[i + 1]);
1559             if (out_size < 0) {
1560                 ret = out_size;
1561                 goto fail;
1562             }
1563
1564             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1565         }
1566
1567         /* get the delayed frames */
1568         out_size = encode_frame(c, NULL);
1569         if (out_size < 0) {
1570             ret = out_size;
1571             goto fail;
1572         }
1573         rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1574
1575         rd += c->error[0] + c->error[1] + c->error[2];
1576
1577         if (rd < best_rd) {
1578             best_rd = rd;
1579             best_b_count = j;
1580         }
1581
1582 fail:
1583         avcodec_free_context(&c);
1584         if (ret < 0)
1585             return ret;
1586     }
1587
1588     return best_b_count;
1589 }
1590
1591 static int select_input_picture(MpegEncContext *s)
1592 {
1593     int i, ret;
1594
1595     for (i = 1; i < MAX_PICTURE_COUNT; i++)
1596         s->reordered_input_picture[i - 1] = s->reordered_input_picture[i];
1597     s->reordered_input_picture[MAX_PICTURE_COUNT - 1] = NULL;
1598
1599     /* set next picture type & ordering */
1600     if (!s->reordered_input_picture[0] && s->input_picture[0]) {
1601         if (s->frame_skip_threshold || s->frame_skip_factor) {
1602             if (s->picture_in_gop_number < s->gop_size &&
1603                 s->next_picture_ptr &&
1604                 skip_check(s, s->input_picture[0], s->next_picture_ptr)) {
1605                 // FIXME check that the gop check above is +-1 correct
1606                 av_frame_unref(s->input_picture[0]->f);
1607
1608                 ff_vbv_update(s, 0);
1609
1610                 goto no_output_pic;
1611             }
1612         }
1613
1614         if (/*s->picture_in_gop_number >= s->gop_size ||*/
1615             !s->next_picture_ptr || s->intra_only) {
1616             s->reordered_input_picture[0] = s->input_picture[0];
1617             s->reordered_input_picture[0]->f->pict_type = AV_PICTURE_TYPE_I;
1618             s->reordered_input_picture[0]->f->coded_picture_number =
1619                 s->coded_picture_number++;
1620         } else {
1621             int b_frames = 0;
1622
1623             if (s->avctx->flags & AV_CODEC_FLAG_PASS2) {
1624                 for (i = 0; i < s->max_b_frames + 1; i++) {
1625                     int pict_num = s->input_picture[0]->f->display_picture_number + i;
1626
1627                     if (pict_num >= s->rc_context.num_entries)
1628                         break;
1629                     if (!s->input_picture[i]) {
1630                         s->rc_context.entry[pict_num - 1].new_pict_type = AV_PICTURE_TYPE_P;
1631                         break;
1632                     }
1633
1634                     s->input_picture[i]->f->pict_type =
1635                         s->rc_context.entry[pict_num].new_pict_type;
1636                 }
1637             }
1638
1639             if (s->b_frame_strategy == 0) {
1640                 b_frames = s->max_b_frames;
1641                 while (b_frames && !s->input_picture[b_frames])
1642                     b_frames--;
1643             } else if (s->b_frame_strategy == 1) {
1644                 for (i = 1; i < s->max_b_frames + 1; i++) {
1645                     if (s->input_picture[i] &&
1646                         s->input_picture[i]->b_frame_score == 0) {
1647                         s->input_picture[i]->b_frame_score =
1648                             get_intra_count(s,
1649                                             s->input_picture[i    ]->f->data[0],
1650                                             s->input_picture[i - 1]->f->data[0],
1651                                             s->linesize) + 1;
1652                     }
1653                 }
1654                 for (i = 0; i < s->max_b_frames + 1; i++) {
1655                     if (!s->input_picture[i] ||
1656                         s->input_picture[i]->b_frame_score - 1 >
1657                             s->mb_num / s->b_sensitivity)
1658                         break;
1659                 }
1660
1661                 b_frames = FFMAX(0, i - 1);
1662
1663                 /* reset scores */
1664                 for (i = 0; i < b_frames + 1; i++) {
1665                     s->input_picture[i]->b_frame_score = 0;
1666                 }
1667             } else if (s->b_frame_strategy == 2) {
1668                 b_frames = estimate_best_b_count(s);
1669                 if (b_frames < 0)
1670                     return b_frames;
1671             }
1672
1673             emms_c();
1674
1675             for (i = b_frames - 1; i >= 0; i--) {
1676                 int type = s->input_picture[i]->f->pict_type;
1677                 if (type && type != AV_PICTURE_TYPE_B)
1678                     b_frames = i;
1679             }
1680             if (s->input_picture[b_frames]->f->pict_type == AV_PICTURE_TYPE_B &&
1681                 b_frames == s->max_b_frames) {
1682                 av_log(s->avctx, AV_LOG_ERROR,
1683                        "warning, too many B-frames in a row\n");
1684             }
1685
1686             if (s->picture_in_gop_number + b_frames >= s->gop_size) {
1687                 if ((s->mpv_flags & FF_MPV_FLAG_STRICT_GOP) &&
1688                     s->gop_size > s->picture_in_gop_number) {
1689                     b_frames = s->gop_size - s->picture_in_gop_number - 1;
1690                 } else {
1691                     if (s->avctx->flags & AV_CODEC_FLAG_CLOSED_GOP)
1692                         b_frames = 0;
1693                     s->input_picture[b_frames]->f->pict_type = AV_PICTURE_TYPE_I;
1694                 }
1695             }
1696
1697             if ((s->avctx->flags & AV_CODEC_FLAG_CLOSED_GOP) && b_frames &&
1698                 s->input_picture[b_frames]->f->pict_type == AV_PICTURE_TYPE_I)
1699                 b_frames--;
1700
1701             s->reordered_input_picture[0] = s->input_picture[b_frames];
1702             if (s->reordered_input_picture[0]->f->pict_type != AV_PICTURE_TYPE_I)
1703                 s->reordered_input_picture[0]->f->pict_type = AV_PICTURE_TYPE_P;
1704             s->reordered_input_picture[0]->f->coded_picture_number =
1705                 s->coded_picture_number++;
1706             for (i = 0; i < b_frames; i++) {
1707                 s->reordered_input_picture[i + 1] = s->input_picture[i];
1708                 s->reordered_input_picture[i + 1]->f->pict_type =
1709                     AV_PICTURE_TYPE_B;
1710                 s->reordered_input_picture[i + 1]->f->coded_picture_number =
1711                     s->coded_picture_number++;
1712             }
1713         }
1714     }
1715 no_output_pic:
1716     ff_mpeg_unref_picture(s->avctx, &s->new_picture);
1717
1718     if (s->reordered_input_picture[0]) {
1719         s->reordered_input_picture[0]->reference =
1720            s->reordered_input_picture[0]->f->pict_type !=
1721                AV_PICTURE_TYPE_B ? 3 : 0;
1722
1723         if ((ret = ff_mpeg_ref_picture(s->avctx, &s->new_picture, s->reordered_input_picture[0])))
1724             return ret;
1725
1726         if (s->reordered_input_picture[0]->shared || s->avctx->rc_buffer_size) {
1727             // input is a shared pix, so we can't modify it -> allocate a new
1728             // one & ensure that the shared one is reuseable
1729
1730             Picture *pic;
1731             int i = ff_find_unused_picture(s->avctx, s->picture, 0);
1732             if (i < 0)
1733                 return i;
1734             pic = &s->picture[i];
1735
1736             pic->reference = s->reordered_input_picture[0]->reference;
1737             if (alloc_picture(s, pic, 0) < 0) {
1738                 return -1;
1739             }
1740
1741             ret = av_frame_copy_props(pic->f, s->reordered_input_picture[0]->f);
1742             if (ret < 0)
1743                 return ret;
1744
1745             /* mark us unused / free shared pic */
1746             av_frame_unref(s->reordered_input_picture[0]->f);
1747             s->reordered_input_picture[0]->shared = 0;
1748
1749             s->current_picture_ptr = pic;
1750         } else {
1751             // input is not a shared pix -> reuse buffer for current_pix
1752             s->current_picture_ptr = s->reordered_input_picture[0];
1753             for (i = 0; i < 4; i++) {
1754                 s->new_picture.f->data[i] += INPLACE_OFFSET;
1755             }
1756         }
1757         ff_mpeg_unref_picture(s->avctx, &s->current_picture);
1758         if ((ret = ff_mpeg_ref_picture(s->avctx, &s->current_picture,
1759                                        s->current_picture_ptr)) < 0)
1760             return ret;
1761
1762         s->picture_number = s->new_picture.f->display_picture_number;
1763     }
1764     return 0;
1765 }
1766
1767 static void frame_end(MpegEncContext *s)
1768 {
1769     if (s->unrestricted_mv &&
1770         s->current_picture.reference &&
1771         !s->intra_only) {
1772         const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(s->avctx->pix_fmt);
1773         int hshift = desc->log2_chroma_w;
1774         int vshift = desc->log2_chroma_h;
1775         s->mpvencdsp.draw_edges(s->current_picture.f->data[0],
1776                                 s->current_picture.f->linesize[0],
1777                                 s->h_edge_pos, s->v_edge_pos,
1778                                 EDGE_WIDTH, EDGE_WIDTH,
1779                                 EDGE_TOP | EDGE_BOTTOM);
1780         s->mpvencdsp.draw_edges(s->current_picture.f->data[1],
1781                                 s->current_picture.f->linesize[1],
1782                                 s->h_edge_pos >> hshift,
1783                                 s->v_edge_pos >> vshift,
1784                                 EDGE_WIDTH >> hshift,
1785                                 EDGE_WIDTH >> vshift,
1786                                 EDGE_TOP | EDGE_BOTTOM);
1787         s->mpvencdsp.draw_edges(s->current_picture.f->data[2],
1788                                 s->current_picture.f->linesize[2],
1789                                 s->h_edge_pos >> hshift,
1790                                 s->v_edge_pos >> vshift,
1791                                 EDGE_WIDTH >> hshift,
1792                                 EDGE_WIDTH >> vshift,
1793                                 EDGE_TOP | EDGE_BOTTOM);
1794     }
1795
1796     emms_c();
1797
1798     s->last_pict_type                 = s->pict_type;
1799     s->last_lambda_for [s->pict_type] = s->current_picture_ptr->f->quality;
1800     if (s->pict_type!= AV_PICTURE_TYPE_B)
1801         s->last_non_b_pict_type = s->pict_type;
1802
1803 #if FF_API_CODED_FRAME
1804 FF_DISABLE_DEPRECATION_WARNINGS
1805     av_frame_unref(s->avctx->coded_frame);
1806     av_frame_copy_props(s->avctx->coded_frame, s->current_picture.f);
1807 FF_ENABLE_DEPRECATION_WARNINGS
1808 #endif
1809 #if FF_API_ERROR_FRAME
1810 FF_DISABLE_DEPRECATION_WARNINGS
1811     memcpy(s->current_picture.f->error, s->current_picture.encoding_error,
1812            sizeof(s->current_picture.encoding_error));
1813 FF_ENABLE_DEPRECATION_WARNINGS
1814 #endif
1815 }
1816
1817 static void update_noise_reduction(MpegEncContext *s)
1818 {
1819     int intra, i;
1820
1821     for (intra = 0; intra < 2; intra++) {
1822         if (s->dct_count[intra] > (1 << 16)) {
1823             for (i = 0; i < 64; i++) {
1824                 s->dct_error_sum[intra][i] >>= 1;
1825             }
1826             s->dct_count[intra] >>= 1;
1827         }
1828
1829         for (i = 0; i < 64; i++) {
1830             s->dct_offset[intra][i] = (s->noise_reduction *
1831                                        s->dct_count[intra] +
1832                                        s->dct_error_sum[intra][i] / 2) /
1833                                       (s->dct_error_sum[intra][i] + 1);
1834         }
1835     }
1836 }
1837
1838 static int frame_start(MpegEncContext *s)
1839 {
1840     int ret;
1841
1842     /* mark & release old frames */
1843     if (s->pict_type != AV_PICTURE_TYPE_B && s->last_picture_ptr &&
1844         s->last_picture_ptr != s->next_picture_ptr &&
1845         s->last_picture_ptr->f->buf[0]) {
1846         ff_mpeg_unref_picture(s->avctx, s->last_picture_ptr);
1847     }
1848
1849     s->current_picture_ptr->f->pict_type = s->pict_type;
1850     s->current_picture_ptr->f->key_frame = s->pict_type == AV_PICTURE_TYPE_I;
1851
1852     ff_mpeg_unref_picture(s->avctx, &s->current_picture);
1853     if ((ret = ff_mpeg_ref_picture(s->avctx, &s->current_picture,
1854                                    s->current_picture_ptr)) < 0)
1855         return ret;
1856
1857     if (s->pict_type != AV_PICTURE_TYPE_B) {
1858         s->last_picture_ptr = s->next_picture_ptr;
1859         if (!s->droppable)
1860             s->next_picture_ptr = s->current_picture_ptr;
1861     }
1862
1863     if (s->last_picture_ptr) {
1864         ff_mpeg_unref_picture(s->avctx, &s->last_picture);
1865         if (s->last_picture_ptr->f->buf[0] &&
1866             (ret = ff_mpeg_ref_picture(s->avctx, &s->last_picture,
1867                                        s->last_picture_ptr)) < 0)
1868             return ret;
1869     }
1870     if (s->next_picture_ptr) {
1871         ff_mpeg_unref_picture(s->avctx, &s->next_picture);
1872         if (s->next_picture_ptr->f->buf[0] &&
1873             (ret = ff_mpeg_ref_picture(s->avctx, &s->next_picture,
1874                                        s->next_picture_ptr)) < 0)
1875             return ret;
1876     }
1877
1878     if (s->picture_structure!= PICT_FRAME) {
1879         int i;
1880         for (i = 0; i < 4; i++) {
1881             if (s->picture_structure == PICT_BOTTOM_FIELD) {
1882                 s->current_picture.f->data[i] +=
1883                     s->current_picture.f->linesize[i];
1884             }
1885             s->current_picture.f->linesize[i] *= 2;
1886             s->last_picture.f->linesize[i]    *= 2;
1887             s->next_picture.f->linesize[i]    *= 2;
1888         }
1889     }
1890
1891     if (s->mpeg_quant || s->codec_id == AV_CODEC_ID_MPEG2VIDEO) {
1892         s->dct_unquantize_intra = s->dct_unquantize_mpeg2_intra;
1893         s->dct_unquantize_inter = s->dct_unquantize_mpeg2_inter;
1894     } else if (s->out_format == FMT_H263 || s->out_format == FMT_H261) {
1895         s->dct_unquantize_intra = s->dct_unquantize_h263_intra;
1896         s->dct_unquantize_inter = s->dct_unquantize_h263_inter;
1897     } else {
1898         s->dct_unquantize_intra = s->dct_unquantize_mpeg1_intra;
1899         s->dct_unquantize_inter = s->dct_unquantize_mpeg1_inter;
1900     }
1901
1902     if (s->dct_error_sum) {
1903         av_assert2(s->noise_reduction && s->encoding);
1904         update_noise_reduction(s);
1905     }
1906
1907     return 0;
1908 }
1909
1910 int ff_mpv_encode_picture(AVCodecContext *avctx, AVPacket *pkt,
1911                           const AVFrame *pic_arg, int *got_packet)
1912 {
1913     MpegEncContext *s = avctx->priv_data;
1914     int i, stuffing_count, ret;
1915     int context_count = s->slice_context_count;
1916
1917     s->vbv_ignore_qmax = 0;
1918
1919     s->picture_in_gop_number++;
1920
1921     if (load_input_picture(s, pic_arg) < 0)
1922         return -1;
1923
1924     if (select_input_picture(s) < 0) {
1925         return -1;
1926     }
1927
1928     /* output? */
1929     if (s->new_picture.f->data[0]) {
1930         int growing_buffer = context_count == 1 && !pkt->data && !s->data_partitioning;
1931         int pkt_size = growing_buffer ? FFMAX(s->mb_width*s->mb_height*64+10000, avctx->internal->byte_buffer_size) - AV_INPUT_BUFFER_PADDING_SIZE
1932                                               :
1933                                               s->mb_width*s->mb_height*(MAX_MB_BYTES+100)+10000;
1934         if ((ret = ff_alloc_packet2(avctx, pkt, pkt_size, 0)) < 0)
1935             return ret;
1936         if (s->mb_info) {
1937             s->mb_info_ptr = av_packet_new_side_data(pkt,
1938                                  AV_PKT_DATA_H263_MB_INFO,
1939                                  s->mb_width*s->mb_height*12);
1940             s->prev_mb_info = s->last_mb_info = s->mb_info_size = 0;
1941         }
1942
1943         for (i = 0; i < context_count; i++) {
1944             int start_y = s->thread_context[i]->start_mb_y;
1945             int   end_y = s->thread_context[i]->  end_mb_y;
1946             int h       = s->mb_height;
1947             uint8_t *start = pkt->data + (size_t)(((int64_t) pkt->size) * start_y / h);
1948             uint8_t *end   = pkt->data + (size_t)(((int64_t) pkt->size) *   end_y / h);
1949
1950             init_put_bits(&s->thread_context[i]->pb, start, end - start);
1951         }
1952
1953         s->pict_type = s->new_picture.f->pict_type;
1954         //emms_c();
1955         ret = frame_start(s);
1956         if (ret < 0)
1957             return ret;
1958 vbv_retry:
1959         ret = encode_picture(s, s->picture_number);
1960         if (growing_buffer) {
1961             av_assert0(s->pb.buf == avctx->internal->byte_buffer);
1962             pkt->data = s->pb.buf;
1963             pkt->size = avctx->internal->byte_buffer_size;
1964         }
1965         if (ret < 0)
1966             return -1;
1967
1968 #if FF_API_STAT_BITS
1969 FF_DISABLE_DEPRECATION_WARNINGS
1970         avctx->header_bits = s->header_bits;
1971         avctx->mv_bits     = s->mv_bits;
1972         avctx->misc_bits   = s->misc_bits;
1973         avctx->i_tex_bits  = s->i_tex_bits;
1974         avctx->p_tex_bits  = s->p_tex_bits;
1975         avctx->i_count     = s->i_count;
1976         // FIXME f/b_count in avctx
1977         avctx->p_count     = s->mb_num - s->i_count - s->skip_count;
1978         avctx->skip_count  = s->skip_count;
1979 FF_ENABLE_DEPRECATION_WARNINGS
1980 #endif
1981
1982         frame_end(s);
1983
1984         if (CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG)
1985             ff_mjpeg_encode_picture_trailer(&s->pb, s->header_bits);
1986
1987         if (avctx->rc_buffer_size) {
1988             RateControlContext *rcc = &s->rc_context;
1989             int max_size = FFMAX(rcc->buffer_index * avctx->rc_max_available_vbv_use, rcc->buffer_index - 500);
1990             int hq = (s->avctx->mb_decision == FF_MB_DECISION_RD || s->avctx->trellis);
1991             int min_step = hq ? 1 : (1<<(FF_LAMBDA_SHIFT + 7))/139;
1992
1993             if (put_bits_count(&s->pb) > max_size &&
1994                 s->lambda < s->lmax) {
1995                 s->next_lambda = FFMAX(s->lambda + min_step, s->lambda *
1996                                        (s->qscale + 1) / s->qscale);
1997                 if (s->adaptive_quant) {
1998                     int i;
1999                     for (i = 0; i < s->mb_height * s->mb_stride; i++)
2000                         s->lambda_table[i] =
2001                             FFMAX(s->lambda_table[i] + min_step,
2002                                   s->lambda_table[i] * (s->qscale + 1) /
2003                                   s->qscale);
2004                 }
2005                 s->mb_skipped = 0;        // done in frame_start()
2006                 // done in encode_picture() so we must undo it
2007                 if (s->pict_type == AV_PICTURE_TYPE_P) {
2008                     if (s->flipflop_rounding          ||
2009                         s->codec_id == AV_CODEC_ID_H263P ||
2010                         s->codec_id == AV_CODEC_ID_MPEG4)
2011                         s->no_rounding ^= 1;
2012                 }
2013                 if (s->pict_type != AV_PICTURE_TYPE_B) {
2014                     s->time_base       = s->last_time_base;
2015                     s->last_non_b_time = s->time - s->pp_time;
2016                 }
2017                 for (i = 0; i < context_count; i++) {
2018                     PutBitContext *pb = &s->thread_context[i]->pb;
2019                     init_put_bits(pb, pb->buf, pb->buf_end - pb->buf);
2020                 }
2021                 s->vbv_ignore_qmax = 1;
2022                 av_log(s->avctx, AV_LOG_VERBOSE, "reencoding frame due to VBV\n");
2023                 goto vbv_retry;
2024             }
2025
2026             av_assert0(s->avctx->rc_max_rate);
2027         }
2028
2029         if (s->avctx->flags & AV_CODEC_FLAG_PASS1)
2030             ff_write_pass1_stats(s);
2031
2032         for (i = 0; i < 4; i++) {
2033             s->current_picture_ptr->encoding_error[i] = s->current_picture.encoding_error[i];
2034             avctx->error[i] += s->current_picture_ptr->encoding_error[i];
2035         }
2036         ff_side_data_set_encoder_stats(pkt, s->current_picture.f->quality,
2037                                        s->current_picture_ptr->encoding_error,
2038                                        (s->avctx->flags&AV_CODEC_FLAG_PSNR) ? 4 : 0,
2039                                        s->pict_type);
2040
2041         if (s->avctx->flags & AV_CODEC_FLAG_PASS1)
2042             assert(put_bits_count(&s->pb) == s->header_bits + s->mv_bits +
2043                                              s->misc_bits + s->i_tex_bits +
2044                                              s->p_tex_bits);
2045         flush_put_bits(&s->pb);
2046         s->frame_bits  = put_bits_count(&s->pb);
2047
2048         stuffing_count = ff_vbv_update(s, s->frame_bits);
2049         s->stuffing_bits = 8*stuffing_count;
2050         if (stuffing_count) {
2051             if (s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb) >> 3) <
2052                     stuffing_count + 50) {
2053                 av_log(s->avctx, AV_LOG_ERROR, "stuffing too large\n");
2054                 return -1;
2055             }
2056
2057             switch (s->codec_id) {
2058             case AV_CODEC_ID_MPEG1VIDEO:
2059             case AV_CODEC_ID_MPEG2VIDEO:
2060                 while (stuffing_count--) {
2061                     put_bits(&s->pb, 8, 0);
2062                 }
2063             break;
2064             case AV_CODEC_ID_MPEG4:
2065                 put_bits(&s->pb, 16, 0);
2066                 put_bits(&s->pb, 16, 0x1C3);
2067                 stuffing_count -= 4;
2068                 while (stuffing_count--) {
2069                     put_bits(&s->pb, 8, 0xFF);
2070                 }
2071             break;
2072             default:
2073                 av_log(s->avctx, AV_LOG_ERROR, "vbv buffer overflow\n");
2074             }
2075             flush_put_bits(&s->pb);
2076             s->frame_bits  = put_bits_count(&s->pb);
2077         }
2078
2079         /* update MPEG-1/2 vbv_delay for CBR */
2080         if (s->avctx->rc_max_rate                          &&
2081             s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
2082             s->out_format == FMT_MPEG1                     &&
2083             90000LL * (avctx->rc_buffer_size - 1) <=
2084                 s->avctx->rc_max_rate * 0xFFFFLL) {
2085             AVCPBProperties *props;
2086             size_t props_size;
2087
2088             int vbv_delay, min_delay;
2089             double inbits  = s->avctx->rc_max_rate *
2090                              av_q2d(s->avctx->time_base);
2091             int    minbits = s->frame_bits - 8 *
2092                              (s->vbv_delay_ptr - s->pb.buf - 1);
2093             double bits    = s->rc_context.buffer_index + minbits - inbits;
2094
2095             if (bits < 0)
2096                 av_log(s->avctx, AV_LOG_ERROR,
2097                        "Internal error, negative bits\n");
2098
2099             assert(s->repeat_first_field == 0);
2100
2101             vbv_delay = bits * 90000 / s->avctx->rc_max_rate;
2102             min_delay = (minbits * 90000LL + s->avctx->rc_max_rate - 1) /
2103                         s->avctx->rc_max_rate;
2104
2105             vbv_delay = FFMAX(vbv_delay, min_delay);
2106
2107             av_assert0(vbv_delay < 0xFFFF);
2108
2109             s->vbv_delay_ptr[0] &= 0xF8;
2110             s->vbv_delay_ptr[0] |= vbv_delay >> 13;
2111             s->vbv_delay_ptr[1]  = vbv_delay >> 5;
2112             s->vbv_delay_ptr[2] &= 0x07;
2113             s->vbv_delay_ptr[2] |= vbv_delay << 3;
2114
2115             props = av_cpb_properties_alloc(&props_size);
2116             if (!props)
2117                 return AVERROR(ENOMEM);
2118             props->vbv_delay = vbv_delay * 300;
2119
2120             ret = av_packet_add_side_data(pkt, AV_PKT_DATA_CPB_PROPERTIES,
2121                                           (uint8_t*)props, props_size);
2122             if (ret < 0) {
2123                 av_freep(&props);
2124                 return ret;
2125             }
2126
2127 #if FF_API_VBV_DELAY
2128 FF_DISABLE_DEPRECATION_WARNINGS
2129             avctx->vbv_delay     = vbv_delay * 300;
2130 FF_ENABLE_DEPRECATION_WARNINGS
2131 #endif
2132         }
2133         s->total_bits     += s->frame_bits;
2134 #if FF_API_STAT_BITS
2135 FF_DISABLE_DEPRECATION_WARNINGS
2136         avctx->frame_bits  = s->frame_bits;
2137 FF_ENABLE_DEPRECATION_WARNINGS
2138 #endif
2139
2140
2141         pkt->pts = s->current_picture.f->pts;
2142         if (!s->low_delay && s->pict_type != AV_PICTURE_TYPE_B) {
2143             if (!s->current_picture.f->coded_picture_number)
2144                 pkt->dts = pkt->pts - s->dts_delta;
2145             else
2146                 pkt->dts = s->reordered_pts;
2147             s->reordered_pts = pkt->pts;
2148         } else
2149             pkt->dts = pkt->pts;
2150         if (s->current_picture.f->key_frame)
2151             pkt->flags |= AV_PKT_FLAG_KEY;
2152         if (s->mb_info)
2153             av_packet_shrink_side_data(pkt, AV_PKT_DATA_H263_MB_INFO, s->mb_info_size);
2154     } else {
2155         s->frame_bits = 0;
2156     }
2157
2158     /* release non-reference frames */
2159     for (i = 0; i < MAX_PICTURE_COUNT; i++) {
2160         if (!s->picture[i].reference)
2161             ff_mpeg_unref_picture(s->avctx, &s->picture[i]);
2162     }
2163
2164     av_assert1((s->frame_bits & 7) == 0);
2165
2166     pkt->size = s->frame_bits / 8;
2167     *got_packet = !!pkt->size;
2168     return 0;
2169 }
2170
2171 static inline void dct_single_coeff_elimination(MpegEncContext *s,
2172                                                 int n, int threshold)
2173 {
2174     static const char tab[64] = {
2175         3, 2, 2, 1, 1, 1, 1, 1,
2176         1, 1, 1, 1, 1, 1, 1, 1,
2177         1, 1, 1, 1, 1, 1, 1, 1,
2178         0, 0, 0, 0, 0, 0, 0, 0,
2179         0, 0, 0, 0, 0, 0, 0, 0,
2180         0, 0, 0, 0, 0, 0, 0, 0,
2181         0, 0, 0, 0, 0, 0, 0, 0,
2182         0, 0, 0, 0, 0, 0, 0, 0
2183     };
2184     int score = 0;
2185     int run = 0;
2186     int i;
2187     int16_t *block = s->block[n];
2188     const int last_index = s->block_last_index[n];
2189     int skip_dc;
2190
2191     if (threshold < 0) {
2192         skip_dc = 0;
2193         threshold = -threshold;
2194     } else
2195         skip_dc = 1;
2196
2197     /* Are all we could set to zero already zero? */
2198     if (last_index <= skip_dc - 1)
2199         return;
2200
2201     for (i = 0; i <= last_index; i++) {
2202         const int j = s->intra_scantable.permutated[i];
2203         const int level = FFABS(block[j]);
2204         if (level == 1) {
2205             if (skip_dc && i == 0)
2206                 continue;
2207             score += tab[run];
2208             run = 0;
2209         } else if (level > 1) {
2210             return;
2211         } else {
2212             run++;
2213         }
2214     }
2215     if (score >= threshold)
2216         return;
2217     for (i = skip_dc; i <= last_index; i++) {
2218         const int j = s->intra_scantable.permutated[i];
2219         block[j] = 0;
2220     }
2221     if (block[0])
2222         s->block_last_index[n] = 0;
2223     else
2224         s->block_last_index[n] = -1;
2225 }
2226
2227 static inline void clip_coeffs(MpegEncContext *s, int16_t *block,
2228                                int last_index)
2229 {
2230     int i;
2231     const int maxlevel = s->max_qcoeff;
2232     const int minlevel = s->min_qcoeff;
2233     int overflow = 0;
2234
2235     if (s->mb_intra) {
2236         i = 1; // skip clipping of intra dc
2237     } else
2238         i = 0;
2239
2240     for (; i <= last_index; i++) {
2241         const int j = s->intra_scantable.permutated[i];
2242         int level = block[j];
2243
2244         if (level > maxlevel) {
2245             level = maxlevel;
2246             overflow++;
2247         } else if (level < minlevel) {
2248             level = minlevel;
2249             overflow++;
2250         }
2251
2252         block[j] = level;
2253     }
2254
2255     if (overflow && s->avctx->mb_decision == FF_MB_DECISION_SIMPLE)
2256         av_log(s->avctx, AV_LOG_INFO,
2257                "warning, clipping %d dct coefficients to %d..%d\n",
2258                overflow, minlevel, maxlevel);
2259 }
2260
2261 static void get_visual_weight(int16_t *weight, uint8_t *ptr, int stride)
2262 {
2263     int x, y;
2264     // FIXME optimize
2265     for (y = 0; y < 8; y++) {
2266         for (x = 0; x < 8; x++) {
2267             int x2, y2;
2268             int sum = 0;
2269             int sqr = 0;
2270             int count = 0;
2271
2272             for (y2 = FFMAX(y - 1, 0); y2 < FFMIN(8, y + 2); y2++) {
2273                 for (x2= FFMAX(x - 1, 0); x2 < FFMIN(8, x + 2); x2++) {
2274                     int v = ptr[x2 + y2 * stride];
2275                     sum += v;
2276                     sqr += v * v;
2277                     count++;
2278                 }
2279             }
2280             weight[x + 8 * y]= (36 * ff_sqrt(count * sqr - sum * sum)) / count;
2281         }
2282     }
2283 }
2284
2285 static av_always_inline void encode_mb_internal(MpegEncContext *s,
2286                                                 int motion_x, int motion_y,
2287                                                 int mb_block_height,
2288                                                 int mb_block_width,
2289                                                 int mb_block_count)
2290 {
2291     int16_t weight[12][64];
2292     int16_t orig[12][64];
2293     const int mb_x = s->mb_x;
2294     const int mb_y = s->mb_y;
2295     int i;
2296     int skip_dct[12];
2297     int dct_offset = s->linesize * 8; // default for progressive frames
2298     int uv_dct_offset = s->uvlinesize * 8;
2299     uint8_t *ptr_y, *ptr_cb, *ptr_cr;
2300     ptrdiff_t wrap_y, wrap_c;
2301
2302     for (i = 0; i < mb_block_count; i++)
2303         skip_dct[i] = s->skipdct;
2304
2305     if (s->adaptive_quant) {
2306         const int last_qp = s->qscale;
2307         const int mb_xy = mb_x + mb_y * s->mb_stride;
2308
2309         s->lambda = s->lambda_table[mb_xy];
2310         update_qscale(s);
2311
2312         if (!(s->mpv_flags & FF_MPV_FLAG_QP_RD)) {
2313             s->qscale = s->current_picture_ptr->qscale_table[mb_xy];
2314             s->dquant = s->qscale - last_qp;
2315
2316             if (s->out_format == FMT_H263) {
2317                 s->dquant = av_clip(s->dquant, -2, 2);
2318
2319                 if (s->codec_id == AV_CODEC_ID_MPEG4) {
2320                     if (!s->mb_intra) {
2321                         if (s->pict_type == AV_PICTURE_TYPE_B) {
2322                             if (s->dquant & 1 || s->mv_dir & MV_DIRECT)
2323                                 s->dquant = 0;
2324                         }
2325                         if (s->mv_type == MV_TYPE_8X8)
2326                             s->dquant = 0;
2327                     }
2328                 }
2329             }
2330         }
2331         ff_set_qscale(s, last_qp + s->dquant);
2332     } else if (s->mpv_flags & FF_MPV_FLAG_QP_RD)
2333         ff_set_qscale(s, s->qscale + s->dquant);
2334
2335     wrap_y = s->linesize;
2336     wrap_c = s->uvlinesize;
2337     ptr_y  = s->new_picture.f->data[0] +
2338              (mb_y * 16 * wrap_y)              + mb_x * 16;
2339     ptr_cb = s->new_picture.f->data[1] +
2340              (mb_y * mb_block_height * wrap_c) + mb_x * mb_block_width;
2341     ptr_cr = s->new_picture.f->data[2] +
2342              (mb_y * mb_block_height * wrap_c) + mb_x * mb_block_width;
2343
2344     if((mb_x * 16 + 16 > s->width || mb_y * 16 + 16 > s->height) && s->codec_id != AV_CODEC_ID_AMV){
2345         uint8_t *ebuf = s->sc.edge_emu_buffer + 38 * wrap_y;
2346         int cw = (s->width  + s->chroma_x_shift) >> s->chroma_x_shift;
2347         int ch = (s->height + s->chroma_y_shift) >> s->chroma_y_shift;
2348         s->vdsp.emulated_edge_mc(ebuf, ptr_y,
2349                                  wrap_y, wrap_y,
2350                                  16, 16, mb_x * 16, mb_y * 16,
2351                                  s->width, s->height);
2352         ptr_y = ebuf;
2353         s->vdsp.emulated_edge_mc(ebuf + 16 * wrap_y, ptr_cb,
2354                                  wrap_c, wrap_c,
2355                                  mb_block_width, mb_block_height,
2356                                  mb_x * mb_block_width, mb_y * mb_block_height,
2357                                  cw, ch);
2358         ptr_cb = ebuf + 16 * wrap_y;
2359         s->vdsp.emulated_edge_mc(ebuf + 16 * wrap_y + 16, ptr_cr,
2360                                  wrap_c, wrap_c,
2361                                  mb_block_width, mb_block_height,
2362                                  mb_x * mb_block_width, mb_y * mb_block_height,
2363                                  cw, ch);
2364         ptr_cr = ebuf + 16 * wrap_y + 16;
2365     }
2366
2367     if (s->mb_intra) {
2368         if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_DCT) {
2369             int progressive_score, interlaced_score;
2370
2371             s->interlaced_dct = 0;
2372             progressive_score = s->mecc.ildct_cmp[4](s, ptr_y, NULL, wrap_y, 8) +
2373                                 s->mecc.ildct_cmp[4](s, ptr_y + wrap_y * 8,
2374                                                      NULL, wrap_y, 8) - 400;
2375
2376             if (progressive_score > 0) {
2377                 interlaced_score = s->mecc.ildct_cmp[4](s, ptr_y,
2378                                                         NULL, wrap_y * 2, 8) +
2379                                    s->mecc.ildct_cmp[4](s, ptr_y + wrap_y,
2380                                                         NULL, wrap_y * 2, 8);
2381                 if (progressive_score > interlaced_score) {
2382                     s->interlaced_dct = 1;
2383
2384                     dct_offset = wrap_y;
2385                     uv_dct_offset = wrap_c;
2386                     wrap_y <<= 1;
2387                     if (s->chroma_format == CHROMA_422 ||
2388                         s->chroma_format == CHROMA_444)
2389                         wrap_c <<= 1;
2390                 }
2391             }
2392         }
2393
2394         s->pdsp.get_pixels(s->block[0], ptr_y,                  wrap_y);
2395         s->pdsp.get_pixels(s->block[1], ptr_y + 8,              wrap_y);
2396         s->pdsp.get_pixels(s->block[2], ptr_y + dct_offset,     wrap_y);
2397         s->pdsp.get_pixels(s->block[3], ptr_y + dct_offset + 8, wrap_y);
2398
2399         if (s->avctx->flags & AV_CODEC_FLAG_GRAY) {
2400             skip_dct[4] = 1;
2401             skip_dct[5] = 1;
2402         } else {
2403             s->pdsp.get_pixels(s->block[4], ptr_cb, wrap_c);
2404             s->pdsp.get_pixels(s->block[5], ptr_cr, wrap_c);
2405             if (!s->chroma_y_shift && s->chroma_x_shift) { /* 422 */
2406                 s->pdsp.get_pixels(s->block[6], ptr_cb + uv_dct_offset, wrap_c);
2407                 s->pdsp.get_pixels(s->block[7], ptr_cr + uv_dct_offset, wrap_c);
2408             } else if (!s->chroma_y_shift && !s->chroma_x_shift) { /* 444 */
2409                 s->pdsp.get_pixels(s->block[ 6], ptr_cb + 8, wrap_c);
2410                 s->pdsp.get_pixels(s->block[ 7], ptr_cr + 8, wrap_c);
2411                 s->pdsp.get_pixels(s->block[ 8], ptr_cb + uv_dct_offset, wrap_c);
2412                 s->pdsp.get_pixels(s->block[ 9], ptr_cr + uv_dct_offset, wrap_c);
2413                 s->pdsp.get_pixels(s->block[10], ptr_cb + uv_dct_offset + 8, wrap_c);
2414                 s->pdsp.get_pixels(s->block[11], ptr_cr + uv_dct_offset + 8, wrap_c);
2415             }
2416         }
2417     } else {
2418         op_pixels_func (*op_pix)[4];
2419         qpel_mc_func (*op_qpix)[16];
2420         uint8_t *dest_y, *dest_cb, *dest_cr;
2421
2422         dest_y  = s->dest[0];
2423         dest_cb = s->dest[1];
2424         dest_cr = s->dest[2];
2425
2426         if ((!s->no_rounding) || s->pict_type == AV_PICTURE_TYPE_B) {
2427             op_pix  = s->hdsp.put_pixels_tab;
2428             op_qpix = s->qdsp.put_qpel_pixels_tab;
2429         } else {
2430             op_pix  = s->hdsp.put_no_rnd_pixels_tab;
2431             op_qpix = s->qdsp.put_no_rnd_qpel_pixels_tab;
2432         }
2433
2434         if (s->mv_dir & MV_DIR_FORWARD) {
2435             ff_mpv_motion(s, dest_y, dest_cb, dest_cr, 0,
2436                           s->last_picture.f->data,
2437                           op_pix, op_qpix);
2438             op_pix  = s->hdsp.avg_pixels_tab;
2439             op_qpix = s->qdsp.avg_qpel_pixels_tab;
2440         }
2441         if (s->mv_dir & MV_DIR_BACKWARD) {
2442             ff_mpv_motion(s, dest_y, dest_cb, dest_cr, 1,
2443                           s->next_picture.f->data,
2444                           op_pix, op_qpix);
2445         }
2446
2447         if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_DCT) {
2448             int progressive_score, interlaced_score;
2449
2450             s->interlaced_dct = 0;
2451             progressive_score = s->mecc.ildct_cmp[0](s, dest_y, ptr_y, wrap_y, 8) +
2452                                 s->mecc.ildct_cmp[0](s, dest_y + wrap_y * 8,
2453                                                      ptr_y + wrap_y * 8,
2454                                                      wrap_y, 8) - 400;
2455
2456             if (s->avctx->ildct_cmp == FF_CMP_VSSE)
2457                 progressive_score -= 400;
2458
2459             if (progressive_score > 0) {
2460                 interlaced_score = s->mecc.ildct_cmp[0](s, dest_y, ptr_y,
2461                                                         wrap_y * 2, 8) +
2462                                    s->mecc.ildct_cmp[0](s, dest_y + wrap_y,
2463                                                         ptr_y + wrap_y,
2464                                                         wrap_y * 2, 8);
2465
2466                 if (progressive_score > interlaced_score) {
2467                     s->interlaced_dct = 1;
2468
2469                     dct_offset = wrap_y;
2470                     uv_dct_offset = wrap_c;
2471                     wrap_y <<= 1;
2472                     if (s->chroma_format == CHROMA_422)
2473                         wrap_c <<= 1;
2474                 }
2475             }
2476         }
2477
2478         s->pdsp.diff_pixels(s->block[0], ptr_y, dest_y, wrap_y);
2479         s->pdsp.diff_pixels(s->block[1], ptr_y + 8, dest_y + 8, wrap_y);
2480         s->pdsp.diff_pixels(s->block[2], ptr_y + dct_offset,
2481                             dest_y + dct_offset, wrap_y);
2482         s->pdsp.diff_pixels(s->block[3], ptr_y + dct_offset + 8,
2483                             dest_y + dct_offset + 8, wrap_y);
2484
2485         if (s->avctx->flags & AV_CODEC_FLAG_GRAY) {
2486             skip_dct[4] = 1;
2487             skip_dct[5] = 1;
2488         } else {
2489             s->pdsp.diff_pixels(s->block[4], ptr_cb, dest_cb, wrap_c);
2490             s->pdsp.diff_pixels(s->block[5], ptr_cr, dest_cr, wrap_c);
2491             if (!s->chroma_y_shift) { /* 422 */
2492                 s->pdsp.diff_pixels(s->block[6], ptr_cb + uv_dct_offset,
2493                                     dest_cb + uv_dct_offset, wrap_c);
2494                 s->pdsp.diff_pixels(s->block[7], ptr_cr + uv_dct_offset,
2495                                     dest_cr + uv_dct_offset, wrap_c);
2496             }
2497         }
2498         /* pre quantization */
2499         if (s->current_picture.mc_mb_var[s->mb_stride * mb_y + mb_x] <
2500                 2 * s->qscale * s->qscale) {
2501             // FIXME optimize
2502             if (s->mecc.sad[1](NULL, ptr_y, dest_y, wrap_y, 8) < 20 * s->qscale)
2503                 skip_dct[0] = 1;
2504             if (s->mecc.sad[1](NULL, ptr_y + 8, dest_y + 8, wrap_y, 8) < 20 * s->qscale)
2505                 skip_dct[1] = 1;
2506             if (s->mecc.sad[1](NULL, ptr_y + dct_offset, dest_y + dct_offset,
2507                                wrap_y, 8) < 20 * s->qscale)
2508                 skip_dct[2] = 1;
2509             if (s->mecc.sad[1](NULL, ptr_y + dct_offset + 8, dest_y + dct_offset + 8,
2510                                wrap_y, 8) < 20 * s->qscale)
2511                 skip_dct[3] = 1;
2512             if (s->mecc.sad[1](NULL, ptr_cb, dest_cb, wrap_c, 8) < 20 * s->qscale)
2513                 skip_dct[4] = 1;
2514             if (s->mecc.sad[1](NULL, ptr_cr, dest_cr, wrap_c, 8) < 20 * s->qscale)
2515                 skip_dct[5] = 1;
2516             if (!s->chroma_y_shift) { /* 422 */
2517                 if (s->mecc.sad[1](NULL, ptr_cb + uv_dct_offset,
2518                                    dest_cb + uv_dct_offset,
2519                                    wrap_c, 8) < 20 * s->qscale)
2520                     skip_dct[6] = 1;
2521                 if (s->mecc.sad[1](NULL, ptr_cr + uv_dct_offset,
2522                                    dest_cr + uv_dct_offset,
2523                                    wrap_c, 8) < 20 * s->qscale)
2524                     skip_dct[7] = 1;
2525             }
2526         }
2527     }
2528
2529     if (s->quantizer_noise_shaping) {
2530         if (!skip_dct[0])
2531             get_visual_weight(weight[0], ptr_y                 , wrap_y);
2532         if (!skip_dct[1])
2533             get_visual_weight(weight[1], ptr_y              + 8, wrap_y);
2534         if (!skip_dct[2])
2535             get_visual_weight(weight[2], ptr_y + dct_offset    , wrap_y);
2536         if (!skip_dct[3])
2537             get_visual_weight(weight[3], ptr_y + dct_offset + 8, wrap_y);
2538         if (!skip_dct[4])
2539             get_visual_weight(weight[4], ptr_cb                , wrap_c);
2540         if (!skip_dct[5])
2541             get_visual_weight(weight[5], ptr_cr                , wrap_c);
2542         if (!s->chroma_y_shift) { /* 422 */
2543             if (!skip_dct[6])
2544                 get_visual_weight(weight[6], ptr_cb + uv_dct_offset,
2545                                   wrap_c);
2546             if (!skip_dct[7])
2547                 get_visual_weight(weight[7], ptr_cr + uv_dct_offset,
2548                                   wrap_c);
2549         }
2550         memcpy(orig[0], s->block[0], sizeof(int16_t) * 64 * mb_block_count);
2551     }
2552
2553     /* DCT & quantize */
2554     av_assert2(s->out_format != FMT_MJPEG || s->qscale == 8);
2555     {
2556         for (i = 0; i < mb_block_count; i++) {
2557             if (!skip_dct[i]) {
2558                 int overflow;
2559                 s->block_last_index[i] = s->dct_quantize(s, s->block[i], i, s->qscale, &overflow);
2560                 // FIXME we could decide to change to quantizer instead of
2561                 // clipping
2562                 // JS: I don't think that would be a good idea it could lower
2563                 //     quality instead of improve it. Just INTRADC clipping
2564                 //     deserves changes in quantizer
2565                 if (overflow)
2566                     clip_coeffs(s, s->block[i], s->block_last_index[i]);
2567             } else
2568                 s->block_last_index[i] = -1;
2569         }
2570         if (s->quantizer_noise_shaping) {
2571             for (i = 0; i < mb_block_count; i++) {
2572                 if (!skip_dct[i]) {
2573                     s->block_last_index[i] =
2574                         dct_quantize_refine(s, s->block[i], weight[i],
2575                                             orig[i], i, s->qscale);
2576                 }
2577             }
2578         }
2579
2580         if (s->luma_elim_threshold && !s->mb_intra)
2581             for (i = 0; i < 4; i++)
2582                 dct_single_coeff_elimination(s, i, s->luma_elim_threshold);
2583         if (s->chroma_elim_threshold && !s->mb_intra)
2584             for (i = 4; i < mb_block_count; i++)
2585                 dct_single_coeff_elimination(s, i, s->chroma_elim_threshold);
2586
2587         if (s->mpv_flags & FF_MPV_FLAG_CBP_RD) {
2588             for (i = 0; i < mb_block_count; i++) {
2589                 if (s->block_last_index[i] == -1)
2590                     s->coded_score[i] = INT_MAX / 256;
2591             }
2592         }
2593     }
2594
2595     if ((s->avctx->flags & AV_CODEC_FLAG_GRAY) && s->mb_intra) {
2596         s->block_last_index[4] =
2597         s->block_last_index[5] = 0;
2598         s->block[4][0] =
2599         s->block[5][0] = (1024 + s->c_dc_scale / 2) / s->c_dc_scale;
2600         if (!s->chroma_y_shift) { /* 422 / 444 */
2601             for (i=6; i<12; i++) {
2602                 s->block_last_index[i] = 0;
2603                 s->block[i][0] = s->block[4][0];
2604             }
2605         }
2606     }
2607
2608     // non c quantize code returns incorrect block_last_index FIXME
2609     if (s->alternate_scan && s->dct_quantize != ff_dct_quantize_c) {
2610         for (i = 0; i < mb_block_count; i++) {
2611             int j;
2612             if (s->block_last_index[i] > 0) {
2613                 for (j = 63; j > 0; j--) {
2614                     if (s->block[i][s->intra_scantable.permutated[j]])
2615                         break;
2616                 }
2617                 s->block_last_index[i] = j;
2618             }
2619         }
2620     }
2621
2622     /* huffman encode */
2623     switch(s->codec_id){ //FIXME funct ptr could be slightly faster
2624     case AV_CODEC_ID_MPEG1VIDEO:
2625     case AV_CODEC_ID_MPEG2VIDEO:
2626         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
2627             ff_mpeg1_encode_mb(s, s->block, motion_x, motion_y);
2628         break;
2629     case AV_CODEC_ID_MPEG4:
2630         if (CONFIG_MPEG4_ENCODER)
2631             ff_mpeg4_encode_mb(s, s->block, motion_x, motion_y);
2632         break;
2633     case AV_CODEC_ID_MSMPEG4V2:
2634     case AV_CODEC_ID_MSMPEG4V3:
2635     case AV_CODEC_ID_WMV1:
2636         if (CONFIG_MSMPEG4_ENCODER)
2637             ff_msmpeg4_encode_mb(s, s->block, motion_x, motion_y);
2638         break;
2639     case AV_CODEC_ID_WMV2:
2640         if (CONFIG_WMV2_ENCODER)
2641             ff_wmv2_encode_mb(s, s->block, motion_x, motion_y);
2642         break;
2643     case AV_CODEC_ID_H261:
2644         if (CONFIG_H261_ENCODER)
2645             ff_h261_encode_mb(s, s->block, motion_x, motion_y);
2646         break;
2647     case AV_CODEC_ID_H263:
2648     case AV_CODEC_ID_H263P:
2649     case AV_CODEC_ID_FLV1:
2650     case AV_CODEC_ID_RV10:
2651     case AV_CODEC_ID_RV20:
2652         if (CONFIG_H263_ENCODER)
2653             ff_h263_encode_mb(s, s->block, motion_x, motion_y);
2654         break;
2655     case AV_CODEC_ID_MJPEG:
2656     case AV_CODEC_ID_AMV:
2657         if (CONFIG_MJPEG_ENCODER)
2658             ff_mjpeg_encode_mb(s, s->block);
2659         break;
2660     default:
2661         av_assert1(0);
2662     }
2663 }
2664
2665 static av_always_inline void encode_mb(MpegEncContext *s, int motion_x, int motion_y)
2666 {
2667     if (s->chroma_format == CHROMA_420) encode_mb_internal(s, motion_x, motion_y,  8, 8, 6);
2668     else if (s->chroma_format == CHROMA_422) encode_mb_internal(s, motion_x, motion_y, 16, 8, 8);
2669     else encode_mb_internal(s, motion_x, motion_y, 16, 16, 12);
2670 }
2671
2672 static inline void copy_context_before_encode(MpegEncContext *d, MpegEncContext *s, int type){
2673     int i;
2674
2675     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2676
2677     /* MPEG-1 */
2678     d->mb_skip_run= s->mb_skip_run;
2679     for(i=0; i<3; i++)
2680         d->last_dc[i] = s->last_dc[i];
2681
2682     /* statistics */
2683     d->mv_bits= s->mv_bits;
2684     d->i_tex_bits= s->i_tex_bits;
2685     d->p_tex_bits= s->p_tex_bits;
2686     d->i_count= s->i_count;
2687     d->f_count= s->f_count;
2688     d->b_count= s->b_count;
2689     d->skip_count= s->skip_count;
2690     d->misc_bits= s->misc_bits;
2691     d->last_bits= 0;
2692
2693     d->mb_skipped= 0;
2694     d->qscale= s->qscale;
2695     d->dquant= s->dquant;
2696
2697     d->esc3_level_length= s->esc3_level_length;
2698 }
2699
2700 static inline void copy_context_after_encode(MpegEncContext *d, MpegEncContext *s, int type){
2701     int i;
2702
2703     memcpy(d->mv, s->mv, 2*4*2*sizeof(int));
2704     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2705
2706     /* MPEG-1 */
2707     d->mb_skip_run= s->mb_skip_run;
2708     for(i=0; i<3; i++)
2709         d->last_dc[i] = s->last_dc[i];
2710
2711     /* statistics */
2712     d->mv_bits= s->mv_bits;
2713     d->i_tex_bits= s->i_tex_bits;
2714     d->p_tex_bits= s->p_tex_bits;
2715     d->i_count= s->i_count;
2716     d->f_count= s->f_count;
2717     d->b_count= s->b_count;
2718     d->skip_count= s->skip_count;
2719     d->misc_bits= s->misc_bits;
2720
2721     d->mb_intra= s->mb_intra;
2722     d->mb_skipped= s->mb_skipped;
2723     d->mv_type= s->mv_type;
2724     d->mv_dir= s->mv_dir;
2725     d->pb= s->pb;
2726     if(s->data_partitioning){
2727         d->pb2= s->pb2;
2728         d->tex_pb= s->tex_pb;
2729     }
2730     d->block= s->block;
2731     for(i=0; i<8; i++)
2732         d->block_last_index[i]= s->block_last_index[i];
2733     d->interlaced_dct= s->interlaced_dct;
2734     d->qscale= s->qscale;
2735
2736     d->esc3_level_length= s->esc3_level_length;
2737 }
2738
2739 static inline void encode_mb_hq(MpegEncContext *s, MpegEncContext *backup, MpegEncContext *best, int type,
2740                            PutBitContext pb[2], PutBitContext pb2[2], PutBitContext tex_pb[2],
2741                            int *dmin, int *next_block, int motion_x, int motion_y)
2742 {
2743     int score;
2744     uint8_t *dest_backup[3];
2745
2746     copy_context_before_encode(s, backup, type);
2747
2748     s->block= s->blocks[*next_block];
2749     s->pb= pb[*next_block];
2750     if(s->data_partitioning){
2751         s->pb2   = pb2   [*next_block];
2752         s->tex_pb= tex_pb[*next_block];
2753     }
2754
2755     if(*next_block){
2756         memcpy(dest_backup, s->dest, sizeof(s->dest));
2757         s->dest[0] = s->sc.rd_scratchpad;
2758         s->dest[1] = s->sc.rd_scratchpad + 16*s->linesize;
2759         s->dest[2] = s->sc.rd_scratchpad + 16*s->linesize + 8;
2760         av_assert0(s->linesize >= 32); //FIXME
2761     }
2762
2763     encode_mb(s, motion_x, motion_y);
2764
2765     score= put_bits_count(&s->pb);
2766     if(s->data_partitioning){
2767         score+= put_bits_count(&s->pb2);
2768         score+= put_bits_count(&s->tex_pb);
2769     }
2770
2771     if(s->avctx->mb_decision == FF_MB_DECISION_RD){
2772         ff_mpv_decode_mb(s, s->block);
2773
2774         score *= s->lambda2;
2775         score += sse_mb(s) << FF_LAMBDA_SHIFT;
2776     }
2777
2778     if(*next_block){
2779         memcpy(s->dest, dest_backup, sizeof(s->dest));
2780     }
2781
2782     if(score<*dmin){
2783         *dmin= score;
2784         *next_block^=1;
2785
2786         copy_context_after_encode(best, s, type);
2787     }
2788 }
2789
2790 static int sse(MpegEncContext *s, uint8_t *src1, uint8_t *src2, int w, int h, int stride){
2791     uint32_t *sq = ff_square_tab + 256;
2792     int acc=0;
2793     int x,y;
2794
2795     if(w==16 && h==16)
2796         return s->mecc.sse[0](NULL, src1, src2, stride, 16);
2797     else if(w==8 && h==8)
2798         return s->mecc.sse[1](NULL, src1, src2, stride, 8);
2799
2800     for(y=0; y<h; y++){
2801         for(x=0; x<w; x++){
2802             acc+= sq[src1[x + y*stride] - src2[x + y*stride]];
2803         }
2804     }
2805
2806     av_assert2(acc>=0);
2807
2808     return acc;
2809 }
2810
2811 static int sse_mb(MpegEncContext *s){
2812     int w= 16;
2813     int h= 16;
2814
2815     if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
2816     if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
2817
2818     if(w==16 && h==16)
2819       if(s->avctx->mb_cmp == FF_CMP_NSSE){
2820         return s->mecc.nsse[0](s, s->new_picture.f->data[0] + s->mb_x * 16 + s->mb_y * s->linesize   * 16, s->dest[0], s->linesize,   16) +
2821                s->mecc.nsse[1](s, s->new_picture.f->data[1] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[1], s->uvlinesize,  8) +
2822                s->mecc.nsse[1](s, s->new_picture.f->data[2] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[2], s->uvlinesize,  8);
2823       }else{
2824         return s->mecc.sse[0](NULL, s->new_picture.f->data[0] + s->mb_x * 16 + s->mb_y * s->linesize   * 16, s->dest[0], s->linesize,   16) +
2825                s->mecc.sse[1](NULL, s->new_picture.f->data[1] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[1], s->uvlinesize,  8) +
2826                s->mecc.sse[1](NULL, s->new_picture.f->data[2] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[2], s->uvlinesize,  8);
2827       }
2828     else
2829         return  sse(s, s->new_picture.f->data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], w, h, s->linesize)
2830                +sse(s, s->new_picture.f->data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], w>>1, h>>1, s->uvlinesize)
2831                +sse(s, s->new_picture.f->data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], w>>1, h>>1, s->uvlinesize);
2832 }
2833
2834 static int pre_estimate_motion_thread(AVCodecContext *c, void *arg){
2835     MpegEncContext *s= *(void**)arg;
2836
2837
2838     s->me.pre_pass=1;
2839     s->me.dia_size= s->avctx->pre_dia_size;
2840     s->first_slice_line=1;
2841     for(s->mb_y= s->end_mb_y-1; s->mb_y >= s->start_mb_y; s->mb_y--) {
2842         for(s->mb_x=s->mb_width-1; s->mb_x >=0 ;s->mb_x--) {
2843             ff_pre_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2844         }
2845         s->first_slice_line=0;
2846     }
2847
2848     s->me.pre_pass=0;
2849
2850     return 0;
2851 }
2852
2853 static int estimate_motion_thread(AVCodecContext *c, void *arg){
2854     MpegEncContext *s= *(void**)arg;
2855
2856     ff_check_alignment();
2857
2858     s->me.dia_size= s->avctx->dia_size;
2859     s->first_slice_line=1;
2860     for(s->mb_y= s->start_mb_y; s->mb_y < s->end_mb_y; s->mb_y++) {
2861         s->mb_x=0; //for block init below
2862         ff_init_block_index(s);
2863         for(s->mb_x=0; s->mb_x < s->mb_width; s->mb_x++) {
2864             s->block_index[0]+=2;
2865             s->block_index[1]+=2;
2866             s->block_index[2]+=2;
2867             s->block_index[3]+=2;
2868
2869             /* compute motion vector & mb_type and store in context */
2870             if(s->pict_type==AV_PICTURE_TYPE_B)
2871                 ff_estimate_b_frame_motion(s, s->mb_x, s->mb_y);
2872             else
2873                 ff_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2874         }
2875         s->first_slice_line=0;
2876     }
2877     return 0;
2878 }
2879
2880 static int mb_var_thread(AVCodecContext *c, void *arg){
2881     MpegEncContext *s= *(void**)arg;
2882     int mb_x, mb_y;
2883
2884     ff_check_alignment();
2885
2886     for(mb_y=s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2887         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2888             int xx = mb_x * 16;
2889             int yy = mb_y * 16;
2890             uint8_t *pix = s->new_picture.f->data[0] + (yy * s->linesize) + xx;
2891             int varc;
2892             int sum = s->mpvencdsp.pix_sum(pix, s->linesize);
2893
2894             varc = (s->mpvencdsp.pix_norm1(pix, s->linesize) -
2895                     (((unsigned) sum * sum) >> 8) + 500 + 128) >> 8;
2896
2897             s->current_picture.mb_var [s->mb_stride * mb_y + mb_x] = varc;
2898             s->current_picture.mb_mean[s->mb_stride * mb_y + mb_x] = (sum+128)>>8;
2899             s->me.mb_var_sum_temp    += varc;
2900         }
2901     }
2902     return 0;
2903 }
2904
2905 static void write_slice_end(MpegEncContext *s){
2906     if(CONFIG_MPEG4_ENCODER && s->codec_id==AV_CODEC_ID_MPEG4){
2907         if(s->partitioned_frame){
2908             ff_mpeg4_merge_partitions(s);
2909         }
2910
2911         ff_mpeg4_stuffing(&s->pb);
2912     }else if(CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG){
2913         ff_mjpeg_encode_stuffing(s);
2914     }
2915
2916     avpriv_align_put_bits(&s->pb);
2917     flush_put_bits(&s->pb);
2918
2919     if ((s->avctx->flags & AV_CODEC_FLAG_PASS1) && !s->partitioned_frame)
2920         s->misc_bits+= get_bits_diff(s);
2921 }
2922
2923 static void write_mb_info(MpegEncContext *s)
2924 {
2925     uint8_t *ptr = s->mb_info_ptr + s->mb_info_size - 12;
2926     int offset = put_bits_count(&s->pb);
2927     int mba  = s->mb_x + s->mb_width * (s->mb_y % s->gob_index);
2928     int gobn = s->mb_y / s->gob_index;
2929     int pred_x, pred_y;
2930     if (CONFIG_H263_ENCODER)
2931         ff_h263_pred_motion(s, 0, 0, &pred_x, &pred_y);
2932     bytestream_put_le32(&ptr, offset);
2933     bytestream_put_byte(&ptr, s->qscale);
2934     bytestream_put_byte(&ptr, gobn);
2935     bytestream_put_le16(&ptr, mba);
2936     bytestream_put_byte(&ptr, pred_x); /* hmv1 */
2937     bytestream_put_byte(&ptr, pred_y); /* vmv1 */
2938     /* 4MV not implemented */
2939     bytestream_put_byte(&ptr, 0); /* hmv2 */
2940     bytestream_put_byte(&ptr, 0); /* vmv2 */
2941 }
2942
2943 static void update_mb_info(MpegEncContext *s, int startcode)
2944 {
2945     if (!s->mb_info)
2946         return;
2947     if (put_bits_count(&s->pb) - s->prev_mb_info*8 >= s->mb_info*8) {
2948         s->mb_info_size += 12;
2949         s->prev_mb_info = s->last_mb_info;
2950     }
2951     if (startcode) {
2952         s->prev_mb_info = put_bits_count(&s->pb)/8;
2953         /* This might have incremented mb_info_size above, and we return without
2954          * actually writing any info into that slot yet. But in that case,
2955          * this will be called again at the start of the after writing the
2956          * start code, actually writing the mb info. */
2957         return;
2958     }
2959
2960     s->last_mb_info = put_bits_count(&s->pb)/8;
2961     if (!s->mb_info_size)
2962         s->mb_info_size += 12;
2963     write_mb_info(s);
2964 }
2965
2966 int ff_mpv_reallocate_putbitbuffer(MpegEncContext *s, size_t threshold, size_t size_increase)
2967 {
2968     if (   s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < threshold
2969         && s->slice_context_count == 1
2970         && s->pb.buf == s->avctx->internal->byte_buffer) {
2971         int lastgob_pos = s->ptr_lastgob - s->pb.buf;
2972         int vbv_pos     = s->vbv_delay_ptr - s->pb.buf;
2973
2974         uint8_t *new_buffer = NULL;
2975         int new_buffer_size = 0;
2976
2977         if ((s->avctx->internal->byte_buffer_size + size_increase) >= INT_MAX/8) {
2978             av_log(s->avctx, AV_LOG_ERROR, "Cannot reallocate putbit buffer\n");
2979             return AVERROR(ENOMEM);
2980         }
2981
2982         emms_c();
2983
2984         av_fast_padded_malloc(&new_buffer, &new_buffer_size,
2985                               s->avctx->internal->byte_buffer_size + size_increase);
2986         if (!new_buffer)
2987             return AVERROR(ENOMEM);
2988
2989         memcpy(new_buffer, s->avctx->internal->byte_buffer, s->avctx->internal->byte_buffer_size);
2990         av_free(s->avctx->internal->byte_buffer);
2991         s->avctx->internal->byte_buffer      = new_buffer;
2992         s->avctx->internal->byte_buffer_size = new_buffer_size;
2993         rebase_put_bits(&s->pb, new_buffer, new_buffer_size);
2994         s->ptr_lastgob   = s->pb.buf + lastgob_pos;
2995         s->vbv_delay_ptr = s->pb.buf + vbv_pos;
2996     }
2997     if (s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < threshold)
2998         return AVERROR(EINVAL);
2999     return 0;
3000 }
3001
3002 static int encode_thread(AVCodecContext *c, void *arg){
3003     MpegEncContext *s= *(void**)arg;
3004     int mb_x, mb_y;
3005     int chr_h= 16>>s->chroma_y_shift;
3006     int i, j;
3007     MpegEncContext best_s = { 0 }, backup_s;
3008     uint8_t bit_buf[2][MAX_MB_BYTES];
3009     uint8_t bit_buf2[2][MAX_MB_BYTES];
3010     uint8_t bit_buf_tex[2][MAX_MB_BYTES];
3011     PutBitContext pb[2], pb2[2], tex_pb[2];
3012
3013     ff_check_alignment();
3014
3015     for(i=0; i<2; i++){
3016         init_put_bits(&pb    [i], bit_buf    [i], MAX_MB_BYTES);
3017         init_put_bits(&pb2   [i], bit_buf2   [i], MAX_MB_BYTES);
3018         init_put_bits(&tex_pb[i], bit_buf_tex[i], MAX_MB_BYTES);
3019     }
3020
3021     s->last_bits= put_bits_count(&s->pb);
3022     s->mv_bits=0;
3023     s->misc_bits=0;
3024     s->i_tex_bits=0;
3025     s->p_tex_bits=0;
3026     s->i_count=0;
3027     s->f_count=0;
3028     s->b_count=0;
3029     s->skip_count=0;
3030
3031     for(i=0; i<3; i++){
3032         /* init last dc values */
3033         /* note: quant matrix value (8) is implied here */
3034         s->last_dc[i] = 128 << s->intra_dc_precision;
3035
3036         s->current_picture.encoding_error[i] = 0;
3037     }
3038     if(s->codec_id==AV_CODEC_ID_AMV){
3039         s->last_dc[0] = 128*8/13;
3040         s->last_dc[1] = 128*8/14;
3041         s->last_dc[2] = 128*8/14;
3042     }
3043     s->mb_skip_run = 0;
3044     memset(s->last_mv, 0, sizeof(s->last_mv));
3045
3046     s->last_mv_dir = 0;
3047
3048     switch(s->codec_id){
3049     case AV_CODEC_ID_H263:
3050     case AV_CODEC_ID_H263P:
3051     case AV_CODEC_ID_FLV1:
3052         if (CONFIG_H263_ENCODER)
3053             s->gob_index = H263_GOB_HEIGHT(s->height);
3054         break;
3055     case AV_CODEC_ID_MPEG4:
3056         if(CONFIG_MPEG4_ENCODER && s->partitioned_frame)
3057             ff_mpeg4_init_partitions(s);
3058         break;
3059     }
3060
3061     s->resync_mb_x=0;
3062     s->resync_mb_y=0;
3063     s->first_slice_line = 1;
3064     s->ptr_lastgob = s->pb.buf;
3065     for(mb_y= s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
3066         s->mb_x=0;
3067         s->mb_y= mb_y;
3068
3069         ff_set_qscale(s, s->qscale);
3070         ff_init_block_index(s);
3071
3072         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
3073             int xy= mb_y*s->mb_stride + mb_x; // removed const, H261 needs to adjust this
3074             int mb_type= s->mb_type[xy];
3075 //            int d;
3076             int dmin= INT_MAX;
3077             int dir;
3078             int size_increase =  s->avctx->internal->byte_buffer_size/4
3079                                + s->mb_width*MAX_MB_BYTES;
3080
3081             ff_mpv_reallocate_putbitbuffer(s, MAX_MB_BYTES, size_increase);
3082             if(s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < MAX_MB_BYTES){
3083                 av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
3084                 return -1;
3085             }
3086             if(s->data_partitioning){
3087                 if(   s->pb2   .buf_end - s->pb2   .buf - (put_bits_count(&s->    pb2)>>3) < MAX_MB_BYTES
3088                    || s->tex_pb.buf_end - s->tex_pb.buf - (put_bits_count(&s->tex_pb )>>3) < MAX_MB_BYTES){
3089                     av_log(s->avctx, AV_LOG_ERROR, "encoded partitioned frame too large\n");
3090                     return -1;
3091                 }
3092             }
3093
3094             s->mb_x = mb_x;
3095             s->mb_y = mb_y;  // moved into loop, can get changed by H.261
3096             ff_update_block_index(s);
3097
3098             if(CONFIG_H261_ENCODER && s->codec_id == AV_CODEC_ID_H261){
3099                 ff_h261_reorder_mb_index(s);
3100                 xy= s->mb_y*s->mb_stride + s->mb_x;
3101                 mb_type= s->mb_type[xy];
3102             }
3103
3104             /* write gob / video packet header  */
3105             if(s->rtp_mode){
3106                 int current_packet_size, is_gob_start;
3107
3108                 current_packet_size= ((put_bits_count(&s->pb)+7)>>3) - (s->ptr_lastgob - s->pb.buf);
3109
3110                 is_gob_start = s->rtp_payload_size &&
3111                                current_packet_size >= s->rtp_payload_size &&
3112                                mb_y + mb_x > 0;
3113
3114                 if(s->start_mb_y == mb_y && mb_y > 0 && mb_x==0) is_gob_start=1;
3115
3116                 switch(s->codec_id){
3117                 case AV_CODEC_ID_H263:
3118                 case AV_CODEC_ID_H263P:
3119                     if(!s->h263_slice_structured)
3120                         if(s->mb_x || s->mb_y%s->gob_index) is_gob_start=0;
3121                     break;
3122                 case AV_CODEC_ID_MPEG2VIDEO:
3123                     if(s->mb_x==0 && s->mb_y!=0) is_gob_start=1;
3124                 case AV_CODEC_ID_MPEG1VIDEO:
3125                     if(s->mb_skip_run) is_gob_start=0;
3126                     break;
3127                 case AV_CODEC_ID_MJPEG:
3128                     if(s->mb_x==0 && s->mb_y!=0) is_gob_start=1;
3129                     break;
3130                 }
3131
3132                 if(is_gob_start){
3133                     if(s->start_mb_y != mb_y || mb_x!=0){
3134                         write_slice_end(s);
3135
3136                         if(CONFIG_MPEG4_ENCODER && s->codec_id==AV_CODEC_ID_MPEG4 && s->partitioned_frame){
3137                             ff_mpeg4_init_partitions(s);
3138                         }
3139                     }
3140
3141                     av_assert2((put_bits_count(&s->pb)&7) == 0);
3142                     current_packet_size= put_bits_ptr(&s->pb) - s->ptr_lastgob;
3143
3144                     if (s->error_rate && s->resync_mb_x + s->resync_mb_y > 0) {
3145                         int r= put_bits_count(&s->pb)/8 + s->picture_number + 16 + s->mb_x + s->mb_y;
3146                         int d = 100 / s->error_rate;
3147                         if(r % d == 0){
3148                             current_packet_size=0;
3149                             s->pb.buf_ptr= s->ptr_lastgob;
3150                             assert(put_bits_ptr(&s->pb) == s->ptr_lastgob);
3151                         }
3152                     }
3153
3154 #if FF_API_RTP_CALLBACK
3155 FF_DISABLE_DEPRECATION_WARNINGS
3156                     if (s->avctx->rtp_callback){
3157                         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width + mb_x - s->resync_mb_x;
3158                         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, current_packet_size, number_mb);
3159                     }
3160 FF_ENABLE_DEPRECATION_WARNINGS
3161 #endif
3162                     update_mb_info(s, 1);
3163
3164                     switch(s->codec_id){
3165                     case AV_CODEC_ID_MPEG4:
3166                         if (CONFIG_MPEG4_ENCODER) {
3167                             ff_mpeg4_encode_video_packet_header(s);
3168                             ff_mpeg4_clean_buffers(s);
3169                         }
3170                     break;
3171                     case AV_CODEC_ID_MPEG1VIDEO:
3172                     case AV_CODEC_ID_MPEG2VIDEO:
3173                         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER) {
3174                             ff_mpeg1_encode_slice_header(s);
3175                             ff_mpeg1_clean_buffers(s);
3176                         }
3177                     break;
3178                     case AV_CODEC_ID_H263:
3179                     case AV_CODEC_ID_H263P:
3180                         if (CONFIG_H263_ENCODER)
3181                             ff_h263_encode_gob_header(s, mb_y);
3182                     break;
3183                     }
3184
3185                     if (s->avctx->flags & AV_CODEC_FLAG_PASS1) {
3186                         int bits= put_bits_count(&s->pb);
3187                         s->misc_bits+= bits - s->last_bits;
3188                         s->last_bits= bits;
3189                     }
3190
3191                     s->ptr_lastgob += current_packet_size;
3192                     s->first_slice_line=1;
3193                     s->resync_mb_x=mb_x;
3194                     s->resync_mb_y=mb_y;
3195                 }
3196             }
3197
3198             if(  (s->resync_mb_x   == s->mb_x)
3199                && s->resync_mb_y+1 == s->mb_y){
3200                 s->first_slice_line=0;
3201             }
3202
3203             s->mb_skipped=0;
3204             s->dquant=0; //only for QP_RD
3205
3206             update_mb_info(s, 0);
3207
3208             if (mb_type & (mb_type-1) || (s->mpv_flags & FF_MPV_FLAG_QP_RD)) { // more than 1 MB type possible or FF_MPV_FLAG_QP_RD
3209                 int next_block=0;
3210                 int pb_bits_count, pb2_bits_count, tex_pb_bits_count;
3211
3212                 copy_context_before_encode(&backup_s, s, -1);
3213                 backup_s.pb= s->pb;
3214                 best_s.data_partitioning= s->data_partitioning;
3215                 best_s.partitioned_frame= s->partitioned_frame;
3216                 if(s->data_partitioning){
3217                     backup_s.pb2= s->pb2;
3218                     backup_s.tex_pb= s->tex_pb;
3219                 }
3220
3221                 if(mb_type&CANDIDATE_MB_TYPE_INTER){
3222                     s->mv_dir = MV_DIR_FORWARD;
3223                     s->mv_type = MV_TYPE_16X16;
3224                     s->mb_intra= 0;
3225                     s->mv[0][0][0] = s->p_mv_table[xy][0];
3226                     s->mv[0][0][1] = s->p_mv_table[xy][1];
3227                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER, pb, pb2, tex_pb,
3228                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
3229                 }
3230                 if(mb_type&CANDIDATE_MB_TYPE_INTER_I){
3231                     s->mv_dir = MV_DIR_FORWARD;
3232                     s->mv_type = MV_TYPE_FIELD;
3233                     s->mb_intra= 0;
3234                     for(i=0; i<2; i++){
3235                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
3236                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
3237                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
3238                     }
3239                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER_I, pb, pb2, tex_pb,
3240                                  &dmin, &next_block, 0, 0);
3241                 }
3242                 if(mb_type&CANDIDATE_MB_TYPE_SKIPPED){
3243                     s->mv_dir = MV_DIR_FORWARD;
3244                     s->mv_type = MV_TYPE_16X16;
3245                     s->mb_intra= 0;
3246                     s->mv[0][0][0] = 0;
3247                     s->mv[0][0][1] = 0;
3248                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_SKIPPED, pb, pb2, tex_pb,
3249                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
3250                 }
3251                 if(mb_type&CANDIDATE_MB_TYPE_INTER4V){
3252                     s->mv_dir = MV_DIR_FORWARD;
3253                     s->mv_type = MV_TYPE_8X8;
3254                     s->mb_intra= 0;
3255                     for(i=0; i<4; i++){
3256                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
3257                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
3258                     }
3259                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER4V, pb, pb2, tex_pb,
3260                                  &dmin, &next_block, 0, 0);
3261                 }
3262                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD){
3263                     s->mv_dir = MV_DIR_FORWARD;
3264                     s->mv_type = MV_TYPE_16X16;
3265                     s->mb_intra= 0;
3266                     s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
3267                     s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
3268                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD, pb, pb2, tex_pb,
3269                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
3270                 }
3271                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD){
3272                     s->mv_dir = MV_DIR_BACKWARD;
3273                     s->mv_type = MV_TYPE_16X16;
3274                     s->mb_intra= 0;
3275                     s->mv[1][0][0] = s->b_back_mv_table[xy][0];
3276                     s->mv[1][0][1] = s->b_back_mv_table[xy][1];
3277                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD, pb, pb2, tex_pb,
3278                                  &dmin, &next_block, s->mv[1][0][0], s->mv[1][0][1]);
3279                 }
3280                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR){
3281                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
3282                     s->mv_type = MV_TYPE_16X16;
3283                     s->mb_intra= 0;
3284                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
3285                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
3286                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
3287                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
3288                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR, pb, pb2, tex_pb,
3289                                  &dmin, &next_block, 0, 0);
3290                 }
3291                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD_I){
3292                     s->mv_dir = MV_DIR_FORWARD;
3293                     s->mv_type = MV_TYPE_FIELD;
3294                     s->mb_intra= 0;
3295                     for(i=0; i<2; i++){
3296                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
3297                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
3298                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
3299                     }
3300                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD_I, pb, pb2, tex_pb,
3301                                  &dmin, &next_block, 0, 0);
3302                 }
3303                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD_I){
3304                     s->mv_dir = MV_DIR_BACKWARD;
3305                     s->mv_type = MV_TYPE_FIELD;
3306                     s->mb_intra= 0;
3307                     for(i=0; i<2; i++){
3308                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
3309                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
3310                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
3311                     }
3312                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD_I, pb, pb2, tex_pb,
3313                                  &dmin, &next_block, 0, 0);
3314                 }
3315                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR_I){
3316                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
3317                     s->mv_type = MV_TYPE_FIELD;
3318                     s->mb_intra= 0;
3319                     for(dir=0; dir<2; dir++){
3320                         for(i=0; i<2; i++){
3321                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
3322                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
3323                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
3324                         }
3325                     }
3326                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR_I, pb, pb2, tex_pb,
3327                                  &dmin, &next_block, 0, 0);
3328                 }
3329                 if(mb_type&CANDIDATE_MB_TYPE_INTRA){
3330                     s->mv_dir = 0;
3331                     s->mv_type = MV_TYPE_16X16;
3332                     s->mb_intra= 1;
3333                     s->mv[0][0][0] = 0;
3334                     s->mv[0][0][1] = 0;
3335                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTRA, pb, pb2, tex_pb,
3336                                  &dmin, &next_block, 0, 0);
3337                     if(s->h263_pred || s->h263_aic){
3338                         if(best_s.mb_intra)
3339                             s->mbintra_table[mb_x + mb_y*s->mb_stride]=1;
3340                         else
3341                             ff_clean_intra_table_entries(s); //old mode?
3342                     }
3343                 }
3344
3345                 if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) && dmin < INT_MAX) {
3346                     if(best_s.mv_type==MV_TYPE_16X16){ //FIXME move 4mv after QPRD
3347                         const int last_qp= backup_s.qscale;
3348                         int qpi, qp, dc[6];
3349                         int16_t ac[6][16];
3350                         const int mvdir= (best_s.mv_dir&MV_DIR_BACKWARD) ? 1 : 0;
3351                         static const int dquant_tab[4]={-1,1,-2,2};
3352                         int storecoefs = s->mb_intra && s->dc_val[0];
3353
3354                         av_assert2(backup_s.dquant == 0);
3355
3356                         //FIXME intra
3357                         s->mv_dir= best_s.mv_dir;
3358                         s->mv_type = MV_TYPE_16X16;
3359                         s->mb_intra= best_s.mb_intra;
3360                         s->mv[0][0][0] = best_s.mv[0][0][0];
3361                         s->mv[0][0][1] = best_s.mv[0][0][1];
3362                         s->mv[1][0][0] = best_s.mv[1][0][0];
3363                         s->mv[1][0][1] = best_s.mv[1][0][1];
3364
3365                         qpi = s->pict_type == AV_PICTURE_TYPE_B ? 2 : 0;
3366                         for(; qpi<4; qpi++){
3367                             int dquant= dquant_tab[qpi];
3368                             qp= last_qp + dquant;
3369                             if(qp < s->avctx->qmin || qp > s->avctx->qmax)
3370                                 continue;
3371                             backup_s.dquant= dquant;
3372                             if(storecoefs){
3373                                 for(i=0; i<6; i++){
3374                                     dc[i]= s->dc_val[0][ s->block_index[i] ];
3375                                     memcpy(ac[i], s->ac_val[0][s->block_index[i]], sizeof(int16_t)*16);
3376                                 }
3377                             }
3378
3379                             encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
3380                                          &dmin, &next_block, s->mv[mvdir][0][0], s->mv[mvdir][0][1]);
3381                             if(best_s.qscale != qp){
3382                                 if(storecoefs){
3383                                     for(i=0; i<6; i++){
3384                                         s->dc_val[0][ s->block_index[i] ]= dc[i];
3385                                         memcpy(s->ac_val[0][s->block_index[i]], ac[i], sizeof(int16_t)*16);
3386                                     }
3387                                 }
3388                             }
3389                         }
3390                     }
3391                 }
3392                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT){
3393                     int mx= s->b_direct_mv_table[xy][0];
3394                     int my= s->b_direct_mv_table[xy][1];
3395
3396                     backup_s.dquant = 0;
3397                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
3398                     s->mb_intra= 0;
3399                     ff_mpeg4_set_direct_mv(s, mx, my);
3400                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
3401                                  &dmin, &next_block, mx, my);
3402                 }
3403                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT0){
3404                     backup_s.dquant = 0;
3405                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
3406                     s->mb_intra= 0;
3407                     ff_mpeg4_set_direct_mv(s, 0, 0);
3408                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
3409                                  &dmin, &next_block, 0, 0);
3410                 }
3411                 if (!best_s.mb_intra && s->mpv_flags & FF_MPV_FLAG_SKIP_RD) {
3412                     int coded=0;
3413                     for(i=0; i<6; i++)
3414                         coded |= s->block_last_index[i];
3415                     if(coded){
3416                         int mx,my;
3417                         memcpy(s->mv, best_s.mv, sizeof(s->mv));
3418                         if(CONFIG_MPEG4_ENCODER && best_s.mv_dir & MV_DIRECT){
3419                             mx=my=0; //FIXME find the one we actually used
3420                             ff_mpeg4_set_direct_mv(s, mx, my);
3421                         }else if(best_s.mv_dir&MV_DIR_BACKWARD){
3422                             mx= s->mv[1][0][0];
3423                             my= s->mv[1][0][1];
3424                         }else{
3425                             mx= s->mv[0][0][0];
3426                             my= s->mv[0][0][1];
3427                         }
3428
3429                         s->mv_dir= best_s.mv_dir;
3430                         s->mv_type = best_s.mv_type;
3431                         s->mb_intra= 0;
3432 /*                        s->mv[0][0][0] = best_s.mv[0][0][0];
3433                         s->mv[0][0][1] = best_s.mv[0][0][1];
3434                         s->mv[1][0][0] = best_s.mv[1][0][0];
3435                         s->mv[1][0][1] = best_s.mv[1][0][1];*/
3436                         backup_s.dquant= 0;
3437                         s->skipdct=1;
3438                         encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
3439                                         &dmin, &next_block, mx, my);
3440                         s->skipdct=0;
3441                     }
3442                 }
3443
3444                 s->current_picture.qscale_table[xy] = best_s.qscale;
3445
3446                 copy_context_after_encode(s, &best_s, -1);
3447
3448                 pb_bits_count= put_bits_count(&s->pb);
3449                 flush_put_bits(&s->pb);
3450                 avpriv_copy_bits(&backup_s.pb, bit_buf[next_block^1], pb_bits_count);
3451                 s->pb= backup_s.pb;
3452
3453                 if(s->data_partitioning){
3454                     pb2_bits_count= put_bits_count(&s->pb2);
3455                     flush_put_bits(&s->pb2);
3456                     avpriv_copy_bits(&backup_s.pb2, bit_buf2[next_block^1], pb2_bits_count);
3457                     s->pb2= backup_s.pb2;
3458
3459                     tex_pb_bits_count= put_bits_count(&s->tex_pb);
3460                     flush_put_bits(&s->tex_pb);
3461                     avpriv_copy_bits(&backup_s.tex_pb, bit_buf_tex[next_block^1], tex_pb_bits_count);
3462                     s->tex_pb= backup_s.tex_pb;
3463                 }
3464                 s->last_bits= put_bits_count(&s->pb);
3465
3466                 if (CONFIG_H263_ENCODER &&
3467                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
3468                     ff_h263_update_motion_val(s);
3469
3470                 if(next_block==0){ //FIXME 16 vs linesize16
3471                     s->hdsp.put_pixels_tab[0][0](s->dest[0], s->sc.rd_scratchpad                     , s->linesize  ,16);
3472                     s->hdsp.put_pixels_tab[1][0](s->dest[1], s->sc.rd_scratchpad + 16*s->linesize    , s->uvlinesize, 8);
3473                     s->hdsp.put_pixels_tab[1][0](s->dest[2], s->sc.rd_scratchpad + 16*s->linesize + 8, s->uvlinesize, 8);
3474                 }
3475
3476                 if(s->avctx->mb_decision == FF_MB_DECISION_BITS)
3477                     ff_mpv_decode_mb(s, s->block);
3478             } else {
3479                 int motion_x = 0, motion_y = 0;
3480                 s->mv_type=MV_TYPE_16X16;
3481                 // only one MB-Type possible
3482
3483                 switch(mb_type){
3484                 case CANDIDATE_MB_TYPE_INTRA:
3485                     s->mv_dir = 0;
3486                     s->mb_intra= 1;
3487                     motion_x= s->mv[0][0][0] = 0;
3488                     motion_y= s->mv[0][0][1] = 0;
3489                     break;
3490                 case CANDIDATE_MB_TYPE_INTER:
3491                     s->mv_dir = MV_DIR_FORWARD;
3492                     s->mb_intra= 0;
3493                     motion_x= s->mv[0][0][0] = s->p_mv_table[xy][0];
3494                     motion_y= s->mv[0][0][1] = s->p_mv_table[xy][1];
3495                     break;
3496                 case CANDIDATE_MB_TYPE_INTER_I:
3497                     s->mv_dir = MV_DIR_FORWARD;
3498                     s->mv_type = MV_TYPE_FIELD;
3499                     s->mb_intra= 0;
3500                     for(i=0; i<2; i++){
3501                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
3502                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
3503                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
3504                     }
3505                     break;
3506                 case CANDIDATE_MB_TYPE_INTER4V:
3507                     s->mv_dir = MV_DIR_FORWARD;
3508                     s->mv_type = MV_TYPE_8X8;
3509                     s->mb_intra= 0;
3510                     for(i=0; i<4; i++){
3511                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
3512                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
3513                     }
3514                     break;
3515                 case CANDIDATE_MB_TYPE_DIRECT:
3516                     if (CONFIG_MPEG4_ENCODER) {
3517                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
3518                         s->mb_intra= 0;
3519                         motion_x=s->b_direct_mv_table[xy][0];
3520                         motion_y=s->b_direct_mv_table[xy][1];
3521                         ff_mpeg4_set_direct_mv(s, motion_x, motion_y);
3522                     }
3523                     break;
3524                 case CANDIDATE_MB_TYPE_DIRECT0:
3525                     if (CONFIG_MPEG4_ENCODER) {
3526                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
3527                         s->mb_intra= 0;
3528                         ff_mpeg4_set_direct_mv(s, 0, 0);
3529                     }
3530                     break;
3531                 case CANDIDATE_MB_TYPE_BIDIR:
3532                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
3533                     s->mb_intra= 0;
3534                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
3535                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
3536                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
3537                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
3538                     break;
3539                 case CANDIDATE_MB_TYPE_BACKWARD:
3540                     s->mv_dir = MV_DIR_BACKWARD;
3541                     s->mb_intra= 0;
3542                     motion_x= s->mv[1][0][0] = s->b_back_mv_table[xy][0];
3543                     motion_y= s->mv[1][0][1] = s->b_back_mv_table[xy][1];
3544                     break;
3545                 case CANDIDATE_MB_TYPE_FORWARD:
3546                     s->mv_dir = MV_DIR_FORWARD;
3547                     s->mb_intra= 0;
3548                     motion_x= s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
3549                     motion_y= s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
3550                     break;
3551                 case CANDIDATE_MB_TYPE_FORWARD_I:
3552                     s->mv_dir = MV_DIR_FORWARD;
3553                     s->mv_type = MV_TYPE_FIELD;
3554                     s->mb_intra= 0;
3555                     for(i=0; i<2; i++){
3556                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
3557                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
3558                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
3559                     }
3560                     break;
3561                 case CANDIDATE_MB_TYPE_BACKWARD_I:
3562                     s->mv_dir = MV_DIR_BACKWARD;
3563                     s->mv_type = MV_TYPE_FIELD;
3564                     s->mb_intra= 0;
3565                     for(i=0; i<2; i++){
3566                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
3567                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
3568                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
3569                     }
3570                     break;
3571                 case CANDIDATE_MB_TYPE_BIDIR_I:
3572                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
3573                     s->mv_type = MV_TYPE_FIELD;
3574                     s->mb_intra= 0;
3575                     for(dir=0; dir<2; dir++){
3576                         for(i=0; i<2; i++){
3577                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
3578                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
3579                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
3580                         }
3581                     }
3582                     break;
3583                 default:
3584                     av_log(s->avctx, AV_LOG_ERROR, "illegal MB type\n");
3585                 }
3586
3587                 encode_mb(s, motion_x, motion_y);
3588
3589                 // RAL: Update last macroblock type
3590                 s->last_mv_dir = s->mv_dir;
3591
3592                 if (CONFIG_H263_ENCODER &&
3593                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
3594                     ff_h263_update_motion_val(s);
3595
3596                 ff_mpv_decode_mb(s, s->block);
3597             }
3598
3599             /* clean the MV table in IPS frames for direct mode in B-frames */
3600             if(s->mb_intra /* && I,P,S_TYPE */){
3601                 s->p_mv_table[xy][0]=0;
3602                 s->p_mv_table[xy][1]=0;
3603             }
3604
3605             if (s->avctx->flags & AV_CODEC_FLAG_PSNR) {
3606                 int w= 16;
3607                 int h= 16;
3608
3609                 if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
3610                 if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
3611
3612                 s->current_picture.encoding_error[0] += sse(
3613                     s, s->new_picture.f->data[0] + s->mb_x*16 + s->mb_y*s->linesize*16,
3614                     s->dest[0], w, h, s->linesize);
3615                 s->current_picture.encoding_error[1] += sse(
3616                     s, s->new_picture.f->data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
3617                     s->dest[1], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
3618                 s->current_picture.encoding_error[2] += sse(
3619                     s, s->new_picture.f->data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
3620                     s->dest[2], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
3621             }
3622             if(s->loop_filter){
3623                 if(CONFIG_H263_ENCODER && s->out_format == FMT_H263)
3624                     ff_h263_loop_filter(s);
3625             }
3626             ff_dlog(s->avctx, "MB %d %d bits\n",
3627                     s->mb_x + s->mb_y * s->mb_stride, put_bits_count(&s->pb));
3628         }
3629     }
3630
3631     //not beautiful here but we must write it before flushing so it has to be here
3632     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version && s->msmpeg4_version<4 && s->pict_type == AV_PICTURE_TYPE_I)
3633         ff_msmpeg4_encode_ext_header(s);
3634
3635     write_slice_end(s);
3636
3637 #if FF_API_RTP_CALLBACK
3638 FF_DISABLE_DEPRECATION_WARNINGS
3639     /* Send the last GOB if RTP */
3640     if (s->avctx->rtp_callback) {
3641         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width - s->resync_mb_x;
3642         int pdif = put_bits_ptr(&s->pb) - s->ptr_lastgob;
3643         /* Call the RTP callback to send the last GOB */
3644         emms_c();
3645         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, pdif, number_mb);
3646     }
3647 FF_ENABLE_DEPRECATION_WARNINGS
3648 #endif
3649
3650     return 0;
3651 }
3652
3653 #define MERGE(field) dst->field += src->field; src->field=0
3654 static void merge_context_after_me(MpegEncContext *dst, MpegEncContext *src){
3655     MERGE(me.scene_change_score);
3656     MERGE(me.mc_mb_var_sum_temp);
3657     MERGE(me.mb_var_sum_temp);
3658 }
3659
3660 static void merge_context_after_encode(MpegEncContext *dst, MpegEncContext *src){
3661     int i;
3662
3663     MERGE(dct_count[0]); //note, the other dct vars are not part of the context
3664     MERGE(dct_count[1]);
3665     MERGE(mv_bits);
3666     MERGE(i_tex_bits);
3667     MERGE(p_tex_bits);
3668     MERGE(i_count);
3669     MERGE(f_count);
3670     MERGE(b_count);
3671     MERGE(skip_count);
3672     MERGE(misc_bits);
3673     MERGE(er.error_count);
3674     MERGE(padding_bug_score);
3675     MERGE(current_picture.encoding_error[0]);
3676     MERGE(current_picture.encoding_error[1]);
3677     MERGE(current_picture.encoding_error[2]);
3678
3679     if (dst->noise_reduction){
3680         for(i=0; i<64; i++){
3681             MERGE(dct_error_sum[0][i]);
3682             MERGE(dct_error_sum[1][i]);
3683         }
3684     }
3685
3686     assert(put_bits_count(&src->pb) % 8 ==0);
3687     assert(put_bits_count(&dst->pb) % 8 ==0);
3688     avpriv_copy_bits(&dst->pb, src->pb.buf, put_bits_count(&src->pb));
3689     flush_put_bits(&dst->pb);
3690 }
3691
3692 static int estimate_qp(MpegEncContext *s, int dry_run){
3693     if (s->next_lambda){
3694         s->current_picture_ptr->f->quality =
3695         s->current_picture.f->quality = s->next_lambda;
3696         if(!dry_run) s->next_lambda= 0;
3697     } else if (!s->fixed_qscale) {
3698         int quality;
3699 #if CONFIG_LIBXVID
3700         if ((s->avctx->flags & AV_CODEC_FLAG_PASS2) && s->rc_strategy == MPV_RC_STRATEGY_XVID)
3701             quality = ff_xvid_rate_estimate_qscale(s, dry_run);
3702         else
3703 #endif
3704         quality = ff_rate_estimate_qscale(s, dry_run);
3705         s->current_picture_ptr->f->quality =
3706         s->current_picture.f->quality = quality;
3707         if (s->current_picture.f->quality < 0)
3708             return -1;
3709     }
3710
3711     if(s->adaptive_quant){
3712         switch(s->codec_id){
3713         case AV_CODEC_ID_MPEG4:
3714             if (CONFIG_MPEG4_ENCODER)
3715                 ff_clean_mpeg4_qscales(s);
3716             break;
3717         case AV_CODEC_ID_H263:
3718         case AV_CODEC_ID_H263P:
3719         case AV_CODEC_ID_FLV1:
3720             if (CONFIG_H263_ENCODER)
3721                 ff_clean_h263_qscales(s);
3722             break;
3723         default:
3724             ff_init_qscale_tab(s);
3725         }
3726
3727         s->lambda= s->lambda_table[0];
3728         //FIXME broken
3729     }else
3730         s->lambda = s->current_picture.f->quality;
3731     update_qscale(s);
3732     return 0;
3733 }
3734
3735 /* must be called before writing the header */
3736 static void set_frame_distances(MpegEncContext * s){
3737     av_assert1(s->current_picture_ptr->f->pts != AV_NOPTS_VALUE);
3738     s->time = s->current_picture_ptr->f->pts * s->avctx->time_base.num;
3739
3740     if(s->pict_type==AV_PICTURE_TYPE_B){
3741         s->pb_time= s->pp_time - (s->last_non_b_time - s->time);
3742         assert(s->pb_time > 0 && s->pb_time < s->pp_time);
3743     }else{
3744         s->pp_time= s->time - s->last_non_b_time;
3745         s->last_non_b_time= s->time;
3746         assert(s->picture_number==0 || s->pp_time > 0);
3747     }
3748 }
3749
3750 static int encode_picture(MpegEncContext *s, int picture_number)
3751 {
3752     int i, ret;
3753     int bits;
3754     int context_count = s->slice_context_count;
3755
3756     s->picture_number = picture_number;
3757
3758     /* Reset the average MB variance */
3759     s->me.mb_var_sum_temp    =
3760     s->me.mc_mb_var_sum_temp = 0;
3761
3762     /* we need to initialize some time vars before we can encode B-frames */
3763     // RAL: Condition added for MPEG1VIDEO
3764     if (s->codec_id == AV_CODEC_ID_MPEG1VIDEO || s->codec_id == AV_CODEC_ID_MPEG2VIDEO || (s->h263_pred && !s->msmpeg4_version))
3765         set_frame_distances(s);
3766     if(CONFIG_MPEG4_ENCODER && s->codec_id == AV_CODEC_ID_MPEG4)
3767         ff_set_mpeg4_time(s);
3768
3769     s->me.scene_change_score=0;
3770
3771 //    s->lambda= s->current_picture_ptr->quality; //FIXME qscale / ... stuff for ME rate distortion
3772
3773     if(s->pict_type==AV_PICTURE_TYPE_I){
3774         if(s->msmpeg4_version >= 3) s->no_rounding=1;
3775         else                        s->no_rounding=0;
3776     }else if(s->pict_type!=AV_PICTURE_TYPE_B){
3777         if(s->flipflop_rounding || s->codec_id == AV_CODEC_ID_H263P || s->codec_id == AV_CODEC_ID_MPEG4)
3778             s->no_rounding ^= 1;
3779     }
3780
3781     if (s->avctx->flags & AV_CODEC_FLAG_PASS2) {
3782         if (estimate_qp(s,1) < 0)
3783             return -1;
3784         ff_get_2pass_fcode(s);
3785     } else if (!(s->avctx->flags & AV_CODEC_FLAG_QSCALE)) {
3786         if(s->pict_type==AV_PICTURE_TYPE_B)
3787             s->lambda= s->last_lambda_for[s->pict_type];
3788         else
3789             s->lambda= s->last_lambda_for[s->last_non_b_pict_type];
3790         update_qscale(s);
3791     }
3792
3793     if(s->codec_id != AV_CODEC_ID_AMV && s->codec_id != AV_CODEC_ID_MJPEG){
3794         if(s->q_chroma_intra_matrix   != s->q_intra_matrix  ) av_freep(&s->q_chroma_intra_matrix);
3795         if(s->q_chroma_intra_matrix16 != s->q_intra_matrix16) av_freep(&s->q_chroma_intra_matrix16);
3796         s->q_chroma_intra_matrix   = s->q_intra_matrix;
3797         s->q_chroma_intra_matrix16 = s->q_intra_matrix16;
3798     }
3799
3800     s->mb_intra=0; //for the rate distortion & bit compare functions
3801     for(i=1; i<context_count; i++){
3802         ret = ff_update_duplicate_context(s->thread_context[i], s);
3803         if (ret < 0)
3804             return ret;
3805     }
3806
3807     if(ff_init_me(s)<0)
3808         return -1;
3809
3810     /* Estimate motion for every MB */
3811     if(s->pict_type != AV_PICTURE_TYPE_I){
3812         s->lambda  = (s->lambda  * s->me_penalty_compensation + 128) >> 8;
3813         s->lambda2 = (s->lambda2 * (int64_t) s->me_penalty_compensation + 128) >> 8;
3814         if (s->pict_type != AV_PICTURE_TYPE_B) {
3815             if ((s->me_pre && s->last_non_b_pict_type == AV_PICTURE_TYPE_I) ||
3816                 s->me_pre == 2) {
3817                 s->avctx->execute(s->avctx, pre_estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3818             }
3819         }
3820
3821         s->avctx->execute(s->avctx, estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3822     }else /* if(s->pict_type == AV_PICTURE_TYPE_I) */{
3823         /* I-Frame */
3824         for(i=0; i<s->mb_stride*s->mb_height; i++)
3825             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
3826
3827         if(!s->fixed_qscale){
3828             /* finding spatial complexity for I-frame rate control */
3829             s->avctx->execute(s->avctx, mb_var_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3830         }
3831     }
3832     for(i=1; i<context_count; i++){
3833         merge_context_after_me(s, s->thread_context[i]);
3834     }
3835     s->current_picture.mc_mb_var_sum= s->current_picture_ptr->mc_mb_var_sum= s->me.mc_mb_var_sum_temp;
3836     s->current_picture.   mb_var_sum= s->current_picture_ptr->   mb_var_sum= s->me.   mb_var_sum_temp;
3837     emms_c();
3838
3839     if (s->me.scene_change_score > s->scenechange_threshold &&
3840         s->pict_type == AV_PICTURE_TYPE_P) {
3841         s->pict_type= AV_PICTURE_TYPE_I;
3842         for(i=0; i<s->mb_stride*s->mb_height; i++)
3843             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
3844         if(s->msmpeg4_version >= 3)
3845             s->no_rounding=1;
3846         ff_dlog(s, "Scene change detected, encoding as I Frame %"PRId64" %"PRId64"\n",
3847                 s->current_picture.mb_var_sum, s->current_picture.mc_mb_var_sum);
3848     }
3849
3850     if(!s->umvplus){
3851         if(s->pict_type==AV_PICTURE_TYPE_P || s->pict_type==AV_PICTURE_TYPE_S) {
3852             s->f_code= ff_get_best_fcode(s, s->p_mv_table, CANDIDATE_MB_TYPE_INTER);
3853
3854             if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_ME) {
3855                 int a,b;
3856                 a= ff_get_best_fcode(s, s->p_field_mv_table[0][0], CANDIDATE_MB_TYPE_INTER_I); //FIXME field_select
3857                 b= ff_get_best_fcode(s, s->p_field_mv_table[1][1], CANDIDATE_MB_TYPE_INTER_I);
3858                 s->f_code= FFMAX3(s->f_code, a, b);
3859             }
3860
3861             ff_fix_long_p_mvs(s);
3862             ff_fix_long_mvs(s, NULL, 0, s->p_mv_table, s->f_code, CANDIDATE_MB_TYPE_INTER, 0);
3863             if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_ME) {
3864                 int j;
3865                 for(i=0; i<2; i++){
3866                     for(j=0; j<2; j++)
3867                         ff_fix_long_mvs(s, s->p_field_select_table[i], j,
3868                                         s->p_field_mv_table[i][j], s->f_code, CANDIDATE_MB_TYPE_INTER_I, 0);
3869                 }
3870             }
3871         }
3872
3873         if(s->pict_type==AV_PICTURE_TYPE_B){
3874             int a, b;
3875
3876             a = ff_get_best_fcode(s, s->b_forw_mv_table, CANDIDATE_MB_TYPE_FORWARD);
3877             b = ff_get_best_fcode(s, s->b_bidir_forw_mv_table, CANDIDATE_MB_TYPE_BIDIR);
3878             s->f_code = FFMAX(a, b);
3879
3880             a = ff_get_best_fcode(s, s->b_back_mv_table, CANDIDATE_MB_TYPE_BACKWARD);
3881             b = ff_get_best_fcode(s, s->b_bidir_back_mv_table, CANDIDATE_MB_TYPE_BIDIR);
3882             s->b_code = FFMAX(a, b);
3883
3884             ff_fix_long_mvs(s, NULL, 0, s->b_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_FORWARD, 1);
3885             ff_fix_long_mvs(s, NULL, 0, s->b_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BACKWARD, 1);
3886             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_BIDIR, 1);
3887             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BIDIR, 1);
3888             if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_ME) {
3889                 int dir, j;
3890                 for(dir=0; dir<2; dir++){
3891                     for(i=0; i<2; i++){
3892                         for(j=0; j<2; j++){
3893                             int type= dir ? (CANDIDATE_MB_TYPE_BACKWARD_I|CANDIDATE_MB_TYPE_BIDIR_I)
3894                                           : (CANDIDATE_MB_TYPE_FORWARD_I |CANDIDATE_MB_TYPE_BIDIR_I);
3895                             ff_fix_long_mvs(s, s->b_field_select_table[dir][i], j,
3896                                             s->b_field_mv_table[dir][i][j], dir ? s->b_code : s->f_code, type, 1);
3897                         }
3898                     }
3899                 }
3900             }
3901         }
3902     }
3903
3904     if (estimate_qp(s, 0) < 0)
3905         return -1;
3906
3907     if (s->qscale < 3 && s->max_qcoeff <= 128 &&
3908         s->pict_type == AV_PICTURE_TYPE_I &&
3909         !(s->avctx->flags & AV_CODEC_FLAG_QSCALE))
3910         s->qscale= 3; //reduce clipping problems
3911
3912     if (s->out_format == FMT_MJPEG) {
3913         const uint16_t *  luma_matrix = ff_mpeg1_default_intra_matrix;
3914         const uint16_t *chroma_matrix = ff_mpeg1_default_intra_matrix;
3915
3916         if (s->avctx->intra_matrix) {
3917             chroma_matrix =
3918             luma_matrix = s->avctx->intra_matrix;
3919         }
3920         if (s->avctx->chroma_intra_matrix)
3921             chroma_matrix = s->avctx->chroma_intra_matrix;
3922
3923         /* for mjpeg, we do include qscale in the matrix */
3924         for(i=1;i<64;i++){
3925             int j = s->idsp.idct_permutation[i];
3926
3927             s->chroma_intra_matrix[j] = av_clip_uint8((chroma_matrix[i] * s->qscale) >> 3);
3928             s->       intra_matrix[j] = av_clip_uint8((  luma_matrix[i] * s->qscale) >> 3);
3929         }
3930         s->y_dc_scale_table=
3931         s->c_dc_scale_table= ff_mpeg2_dc_scale_table[s->intra_dc_precision];
3932         s->chroma_intra_matrix[0] =
3933         s->intra_matrix[0] = ff_mpeg2_dc_scale_table[s->intra_dc_precision][8];
3934         ff_convert_matrix(s, s->q_intra_matrix, s->q_intra_matrix16,
3935                        s->intra_matrix, s->intra_quant_bias, 8, 8, 1);
3936         ff_convert_matrix(s, s->q_chroma_intra_matrix, s->q_chroma_intra_matrix16,
3937                        s->chroma_intra_matrix, s->intra_quant_bias, 8, 8, 1);
3938         s->qscale= 8;
3939     }
3940     if(s->codec_id == AV_CODEC_ID_AMV){
3941         static const uint8_t y[32]={13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13};
3942         static const uint8_t c[32]={14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14};
3943         for(i=1;i<64;i++){
3944             int j= s->idsp.idct_permutation[ff_zigzag_direct[i]];
3945
3946             s->intra_matrix[j] = sp5x_quant_table[5*2+0][i];
3947             s->chroma_intra_matrix[j] = sp5x_quant_table[5*2+1][i];
3948         }
3949         s->y_dc_scale_table= y;
3950         s->c_dc_scale_table= c;
3951         s->intra_matrix[0] = 13;
3952         s->chroma_intra_matrix[0] = 14;
3953         ff_convert_matrix(s, s->q_intra_matrix, s->q_intra_matrix16,
3954                        s->intra_matrix, s->intra_quant_bias, 8, 8, 1);
3955         ff_convert_matrix(s, s->q_chroma_intra_matrix, s->q_chroma_intra_matrix16,
3956                        s->chroma_intra_matrix, s->intra_quant_bias, 8, 8, 1);
3957         s->qscale= 8;
3958     }
3959
3960     //FIXME var duplication
3961     s->current_picture_ptr->f->key_frame =
3962     s->current_picture.f->key_frame = s->pict_type == AV_PICTURE_TYPE_I; //FIXME pic_ptr
3963     s->current_picture_ptr->f->pict_type =
3964     s->current_picture.f->pict_type = s->pict_type;
3965
3966     if (s->current_picture.f->key_frame)
3967         s->picture_in_gop_number=0;
3968
3969     s->mb_x = s->mb_y = 0;
3970     s->last_bits= put_bits_count(&s->pb);
3971     switch(s->out_format) {
3972     case FMT_MJPEG:
3973         if (CONFIG_MJPEG_ENCODER && s->huffman != HUFFMAN_TABLE_OPTIMAL)
3974             ff_mjpeg_encode_picture_header(s->avctx, &s->pb, &s->intra_scantable,
3975                                            s->pred, s->intra_matrix, s->chroma_intra_matrix);
3976         break;
3977     case FMT_H261:
3978         if (CONFIG_H261_ENCODER)
3979             ff_h261_encode_picture_header(s, picture_number);
3980         break;
3981     case FMT_H263:
3982         if (CONFIG_WMV2_ENCODER && s->codec_id == AV_CODEC_ID_WMV2)
3983             ff_wmv2_encode_picture_header(s, picture_number);
3984         else if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
3985             ff_msmpeg4_encode_picture_header(s, picture_number);
3986         else if (CONFIG_MPEG4_ENCODER && s->h263_pred) {
3987             ret = ff_mpeg4_encode_picture_header(s, picture_number);
3988             if (ret < 0)
3989                 return ret;
3990         } else if (CONFIG_RV10_ENCODER && s->codec_id == AV_CODEC_ID_RV10) {
3991             ret = ff_rv10_encode_picture_header(s, picture_number);
3992             if (ret < 0)
3993                 return ret;
3994         }
3995         else if (CONFIG_RV20_ENCODER && s->codec_id == AV_CODEC_ID_RV20)
3996             ff_rv20_encode_picture_header(s, picture_number);
3997         else if (CONFIG_FLV_ENCODER && s->codec_id == AV_CODEC_ID_FLV1)
3998             ff_flv_encode_picture_header(s, picture_number);
3999         else if (CONFIG_H263_ENCODER)
4000             ff_h263_encode_picture_header(s, picture_number);
4001         break;
4002     case FMT_MPEG1:
4003         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
4004             ff_mpeg1_encode_picture_header(s, picture_number);
4005         break;
4006     default:
4007         av_assert0(0);
4008     }
4009     bits= put_bits_count(&s->pb);
4010     s->header_bits= bits - s->last_bits;
4011
4012     for(i=1; i<context_count; i++){
4013         update_duplicate_context_after_me(s->thread_context[i], s);
4014     }
4015     s->avctx->execute(s->avctx, encode_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
4016     for(i=1; i<context_count; i++){
4017         if (s->pb.buf_end == s->thread_context[i]->pb.buf)
4018             set_put_bits_buffer_size(&s->pb, FFMIN(s->thread_context[i]->pb.buf_end - s->pb.buf, INT_MAX/8-32));
4019         merge_context_after_encode(s, s->thread_context[i]);
4020     }
4021     emms_c();
4022     return 0;
4023 }
4024
4025 static void denoise_dct_c(MpegEncContext *s, int16_t *block){
4026     const int intra= s->mb_intra;
4027     int i;
4028
4029     s->dct_count[intra]++;
4030
4031     for(i=0; i<64; i++){
4032         int level= block[i];
4033
4034         if(level){
4035             if(level>0){
4036                 s->dct_error_sum[intra][i] += level;
4037                 level -= s->dct_offset[intra][i];
4038                 if(level<0) level=0;
4039             }else{
4040                 s->dct_error_sum[intra][i] -= level;
4041                 level += s->dct_offset[intra][i];
4042                 if(level>0) level=0;
4043             }
4044             block[i]= level;
4045         }
4046     }
4047 }
4048
4049 static int dct_quantize_trellis_c(MpegEncContext *s,
4050                                   int16_t *block, int n,
4051                                   int qscale, int *overflow){
4052     const int *qmat;
4053     const uint16_t *matrix;
4054     const uint8_t *scantable= s->intra_scantable.scantable;
4055     const uint8_t *perm_scantable= s->intra_scantable.permutated;
4056     int max=0;
4057     unsigned int threshold1, threshold2;
4058     int bias=0;
4059     int run_tab[65];
4060     int level_tab[65];
4061     int score_tab[65];
4062     int survivor[65];
4063     int survivor_count;
4064     int last_run=0;
4065     int last_level=0;
4066     int last_score= 0;
4067     int last_i;
4068     int coeff[2][64];
4069     int coeff_count[64];
4070     int qmul, qadd, start_i, last_non_zero, i, dc;
4071     const int esc_length= s->ac_esc_length;
4072     uint8_t * length;
4073     uint8_t * last_length;
4074     const int lambda= s->lambda2 >> (FF_LAMBDA_SHIFT - 6);
4075     int mpeg2_qscale;
4076
4077     s->fdsp.fdct(block);
4078
4079     if(s->dct_error_sum)
4080         s->denoise_dct(s, block);
4081     qmul= qscale*16;
4082     qadd= ((qscale-1)|1)*8;
4083
4084     if (s->q_scale_type) mpeg2_qscale = ff_mpeg2_non_linear_qscale[qscale];
4085     else                 mpeg2_qscale = qscale << 1;
4086
4087     if (s->mb_intra) {
4088         int q;
4089         if (!s->h263_aic) {
4090             if (n < 4)
4091                 q = s->y_dc_scale;
4092             else
4093                 q = s->c_dc_scale;
4094             q = q << 3;
4095         } else{
4096             /* For AIC we skip quant/dequant of INTRADC */
4097             q = 1 << 3;
4098             qadd=0;
4099         }
4100
4101         /* note: block[0] is assumed to be positive */
4102         block[0] = (block[0] + (q >> 1)) / q;
4103         start_i = 1;
4104         last_non_zero = 0;
4105         qmat = n < 4 ? s->q_intra_matrix[qscale] : s->q_chroma_intra_matrix[qscale];
4106         matrix = n < 4 ? s->intra_matrix : s->chroma_intra_matrix;
4107         if(s->mpeg_quant || s->out_format == FMT_MPEG1 || s->out_format == FMT_MJPEG)
4108             bias= 1<<(QMAT_SHIFT-1);
4109
4110         if (n > 3 && s->intra_chroma_ac_vlc_length) {
4111             length     = s->intra_chroma_ac_vlc_length;
4112             last_length= s->intra_chroma_ac_vlc_last_length;
4113         } else {
4114             length     = s->intra_ac_vlc_length;
4115             last_length= s->intra_ac_vlc_last_length;
4116         }
4117     } else {
4118         start_i = 0;
4119         last_non_zero = -1;
4120         qmat = s->q_inter_matrix[qscale];
4121         matrix = s->inter_matrix;
4122         length     = s->inter_ac_vlc_length;
4123         last_length= s->inter_ac_vlc_last_length;
4124     }
4125     last_i= start_i;
4126
4127     threshold1= (1<<QMAT_SHIFT) - bias - 1;
4128     threshold2= (threshold1<<1);
4129
4130     for(i=63; i>=start_i; i--) {
4131         const int j = scantable[i];
4132         int level = block[j] * qmat[j];
4133
4134         if(((unsigned)(level+threshold1))>threshold2){
4135             last_non_zero = i;
4136             break;
4137         }
4138     }
4139
4140     for(i=start_i; i<=last_non_zero; i++) {
4141         const int j = scantable[i];
4142         int level = block[j] * qmat[j];
4143
4144 //        if(   bias+level >= (1<<(QMAT_SHIFT - 3))
4145 //           || bias-level >= (1<<(QMAT_SHIFT - 3))){
4146         if(((unsigned)(level+threshold1))>threshold2){
4147             if(level>0){
4148                 level= (bias + level)>>QMAT_SHIFT;
4149                 coeff[0][i]= level;
4150                 coeff[1][i]= level-1;
4151 //                coeff[2][k]= level-2;
4152             }else{
4153                 level= (bias - level)>>QMAT_SHIFT;
4154                 coeff[0][i]= -level;
4155                 coeff[1][i]= -level+1;
4156 //                coeff[2][k]= -level+2;
4157             }
4158             coeff_count[i]= FFMIN(level, 2);
4159             av_assert2(coeff_count[i]);
4160             max |=level;
4161         }else{
4162             coeff[0][i]= (level>>31)|1;
4163             coeff_count[i]= 1;
4164         }
4165     }
4166
4167     *overflow= s->max_qcoeff < max; //overflow might have happened
4168
4169     if(last_non_zero < start_i){
4170         memset(block + start_i, 0, (64-start_i)*sizeof(int16_t));
4171         return last_non_zero;
4172     }
4173
4174     score_tab[start_i]= 0;
4175     survivor[0]= start_i;
4176     survivor_count= 1;
4177
4178     for(i=start_i; i<=last_non_zero; i++){
4179         int level_index, j, zero_distortion;
4180         int dct_coeff= FFABS(block[ scantable[i] ]);
4181         int best_score=256*256*256*120;
4182
4183         if (s->fdsp.fdct == ff_fdct_ifast)
4184             dct_coeff= (dct_coeff*ff_inv_aanscales[ scantable[i] ]) >> 12;
4185         zero_distortion= dct_coeff*dct_coeff;
4186
4187         for(level_index=0; level_index < coeff_count[i]; level_index++){
4188             int distortion;
4189             int level= coeff[level_index][i];
4190             const int alevel= FFABS(level);
4191             int unquant_coeff;
4192
4193             av_assert2(level);
4194
4195             if(s->out_format == FMT_H263 || s->out_format == FMT_H261){
4196                 unquant_coeff= alevel*qmul + qadd;
4197             } else if(s->out_format == FMT_MJPEG) {
4198                 j = s->idsp.idct_permutation[scantable[i]];
4199                 unquant_coeff = alevel * matrix[j] * 8;
4200             }else{ // MPEG-1
4201                 j = s->idsp.idct_permutation[scantable[i]]; // FIXME: optimize
4202                 if(s->mb_intra){
4203                         unquant_coeff = (int)(  alevel  * mpeg2_qscale * matrix[j]) >> 4;
4204                         unquant_coeff =   (unquant_coeff - 1) | 1;
4205                 }else{
4206                         unquant_coeff = (((  alevel  << 1) + 1) * mpeg2_qscale * ((int) matrix[j])) >> 5;
4207                         unquant_coeff =   (unquant_coeff - 1) | 1;
4208                 }
4209                 unquant_coeff<<= 3;
4210             }
4211
4212             distortion= (unquant_coeff - dct_coeff) * (unquant_coeff - dct_coeff) - zero_distortion;
4213             level+=64;
4214             if((level&(~127)) == 0){
4215                 for(j=survivor_count-1; j>=0; j--){
4216                     int run= i - survivor[j];
4217                     int score= distortion + length[UNI_AC_ENC_INDEX(run, level)]*lambda;
4218                     score += score_tab[i-run];
4219
4220                     if(score < best_score){
4221                         best_score= score;
4222                         run_tab[i+1]= run;
4223                         level_tab[i+1]= level-64;
4224                     }
4225                 }
4226
4227                 if(s->out_format == FMT_H263 || s->out_format == FMT_H261){
4228                     for(j=survivor_count-1; j>=0; j--){
4229                         int run= i - survivor[j];
4230                         int score= distortion + last_length[UNI_AC_ENC_INDEX(run, level)]*lambda;
4231                         score += score_tab[i-run];
4232                         if(score < last_score){
4233                             last_score= score;
4234                             last_run= run;
4235                             last_level= level-64;
4236                             last_i= i+1;
4237                         }
4238                     }
4239                 }
4240             }else{
4241                 distortion += esc_length*lambda;
4242                 for(j=survivor_count-1; j>=0; j--){
4243                     int run= i - survivor[j];
4244                     int score= distortion + score_tab[i-run];
4245
4246                     if(score < best_score){
4247                         best_score= score;
4248                         run_tab[i+1]= run;
4249                         level_tab[i+1]= level-64;
4250                     }
4251                 }
4252
4253                 if(s->out_format == FMT_H263 || s->out_format == FMT_H261){
4254                   for(j=survivor_count-1; j>=0; j--){
4255                         int run= i - survivor[j];
4256                         int score= distortion + score_tab[i-run];
4257                         if(score < last_score){
4258                             last_score= score;
4259                             last_run= run;
4260                             last_level= level-64;
4261                             last_i= i+1;
4262                         }
4263                     }
4264                 }
4265             }
4266         }
4267
4268         score_tab[i+1]= best_score;
4269
4270         // Note: there is a vlc code in MPEG-4 which is 1 bit shorter then another one with a shorter run and the same level
4271         if(last_non_zero <= 27){
4272             for(; survivor_count; survivor_count--){
4273                 if(score_tab[ survivor[survivor_count-1] ] <= best_score)
4274                     break;
4275             }
4276         }else{
4277             for(; survivor_count; survivor_count--){
4278                 if(score_tab[ survivor[survivor_count-1] ] <= best_score + lambda)
4279                     break;
4280             }
4281         }
4282
4283         survivor[ survivor_count++ ]= i+1;
4284     }
4285
4286     if(s->out_format != FMT_H263 && s->out_format != FMT_H261){
4287         last_score= 256*256*256*120;
4288         for(i= survivor[0]; i<=last_non_zero + 1; i++){
4289             int score= score_tab[i];
4290             if (i)
4291                 score += lambda * 2; // FIXME more exact?
4292
4293             if(score < last_score){
4294                 last_score= score;
4295                 last_i= i;
4296                 last_level= level_tab[i];
4297                 last_run= run_tab[i];
4298             }
4299         }
4300     }
4301
4302     s->coded_score[n] = last_score;
4303
4304     dc= FFABS(block[0]);
4305     last_non_zero= last_i - 1;
4306     memset(block + start_i, 0, (64-start_i)*sizeof(int16_t));
4307
4308     if(last_non_zero < start_i)
4309         return last_non_zero;
4310
4311     if(last_non_zero == 0 && start_i == 0){
4312         int best_level= 0;
4313         int best_score= dc * dc;
4314
4315         for(i=0; i<coeff_count[0]; i++){
4316             int level= coeff[i][0];
4317             int alevel= FFABS(level);
4318             int unquant_coeff, score, distortion;
4319
4320             if(s->out_format == FMT_H263 || s->out_format == FMT_H261){
4321                     unquant_coeff= (alevel*qmul + qadd)>>3;
4322             } else{ // MPEG-1
4323                     unquant_coeff = (((  alevel  << 1) + 1) * mpeg2_qscale * ((int) matrix[0])) >> 5;
4324                     unquant_coeff =   (unquant_coeff - 1) | 1;
4325             }
4326             unquant_coeff = (unquant_coeff + 4) >> 3;
4327             unquant_coeff<<= 3 + 3;
4328
4329             distortion= (unquant_coeff - dc) * (unquant_coeff - dc);
4330             level+=64;
4331             if((level&(~127)) == 0) score= distortion + last_length[UNI_AC_ENC_INDEX(0, level)]*lambda;
4332             else                    score= distortion + esc_length*lambda;
4333
4334             if(score < best_score){
4335                 best_score= score;
4336                 best_level= level - 64;
4337             }
4338         }
4339         block[0]= best_level;
4340         s->coded_score[n] = best_score - dc*dc;
4341         if(best_level == 0) return -1;
4342         else                return last_non_zero;
4343     }
4344
4345     i= last_i;
4346     av_assert2(last_level);
4347
4348     block[ perm_scantable[last_non_zero] ]= last_level;
4349     i -= last_run + 1;
4350
4351     for(; i>start_i; i -= run_tab[i] + 1){
4352         block[ perm_scantable[i-1] ]= level_tab[i];
4353     }
4354
4355     return last_non_zero;
4356 }
4357
4358 //#define REFINE_STATS 1
4359 static int16_t basis[64][64];
4360
4361 static void build_basis(uint8_t *perm){
4362     int i, j, x, y;
4363     emms_c();
4364     for(i=0; i<8; i++){
4365         for(j=0; j<8; j++){
4366             for(y=0; y<8; y++){
4367                 for(x=0; x<8; x++){
4368                     double s= 0.25*(1<<BASIS_SHIFT);
4369                     int index= 8*i + j;
4370                     int perm_index= perm[index];
4371                     if(i==0) s*= sqrt(0.5);
4372                     if(j==0) s*= sqrt(0.5);
4373                     basis[perm_index][8*x + y]= lrintf(s * cos((M_PI/8.0)*i*(x+0.5)) * cos((M_PI/8.0)*j*(y+0.5)));
4374                 }
4375             }
4376         }
4377     }
4378 }
4379
4380 static int dct_quantize_refine(MpegEncContext *s, //FIXME breaks denoise?
4381                         int16_t *block, int16_t *weight, int16_t *orig,
4382                         int n, int qscale){
4383     int16_t rem[64];
4384     LOCAL_ALIGNED_16(int16_t, d1, [64]);
4385     const uint8_t *scantable= s->intra_scantable.scantable;
4386     const uint8_t *perm_scantable= s->intra_scantable.permutated;
4387 //    unsigned int threshold1, threshold2;
4388 //    int bias=0;
4389     int run_tab[65];
4390     int prev_run=0;
4391     int prev_level=0;
4392     int qmul, qadd, start_i, last_non_zero, i, dc;
4393     uint8_t * length;
4394     uint8_t * last_length;
4395     int lambda;
4396     int rle_index, run, q = 1, sum; //q is only used when s->mb_intra is true
4397 #ifdef REFINE_STATS
4398 static int count=0;
4399 static int after_last=0;
4400 static int to_zero=0;
4401 static int from_zero=0;
4402 static int raise=0;
4403 static int lower=0;
4404 static int messed_sign=0;
4405 #endif
4406
4407     if(basis[0][0] == 0)
4408         build_basis(s->idsp.idct_permutation);
4409
4410     qmul= qscale*2;
4411     qadd= (qscale-1)|1;
4412     if (s->mb_intra) {
4413         if (!s->h263_aic) {
4414             if (n < 4)
4415                 q = s->y_dc_scale;
4416             else
4417                 q = s->c_dc_scale;
4418         } else{
4419             /* For AIC we skip quant/dequant of INTRADC */
4420             q = 1;
4421             qadd=0;
4422         }
4423         q <<= RECON_SHIFT-3;
4424         /* note: block[0] is assumed to be positive */
4425         dc= block[0]*q;
4426 //        block[0] = (block[0] + (q >> 1)) / q;
4427         start_i = 1;
4428 //        if(s->mpeg_quant || s->out_format == FMT_MPEG1)
4429 //            bias= 1<<(QMAT_SHIFT-1);
4430         if (n > 3 && s->intra_chroma_ac_vlc_length) {
4431             length     = s->intra_chroma_ac_vlc_length;
4432             last_length= s->intra_chroma_ac_vlc_last_length;
4433         } else {
4434             length     = s->intra_ac_vlc_length;
4435             last_length= s->intra_ac_vlc_last_length;
4436         }
4437     } else {
4438         dc= 0;
4439         start_i = 0;
4440         length     = s->inter_ac_vlc_length;
4441         last_length= s->inter_ac_vlc_last_length;
4442     }
4443     last_non_zero = s->block_last_index[n];
4444
4445 #ifdef REFINE_STATS
4446 {START_TIMER
4447 #endif
4448     dc += (1<<(RECON_SHIFT-1));
4449     for(i=0; i<64; i++){
4450         rem[i] = dc - (orig[i] << RECON_SHIFT); // FIXME use orig directly instead of copying to rem[]
4451     }
4452 #ifdef REFINE_STATS
4453 STOP_TIMER("memset rem[]")}
4454 #endif
4455     sum=0;
4456     for(i=0; i<64; i++){
4457         int one= 36;
4458         int qns=4;
4459         int w;
4460
4461         w= FFABS(weight[i]) + qns*one;
4462         w= 15 + (48*qns*one + w/2)/w; // 16 .. 63
4463
4464         weight[i] = w;
4465 //        w=weight[i] = (63*qns + (w/2)) / w;
4466
4467         av_assert2(w>0);
4468         av_assert2(w<(1<<6));
4469         sum += w*w;
4470     }
4471     lambda= sum*(uint64_t)s->lambda2 >> (FF_LAMBDA_SHIFT - 6 + 6 + 6 + 6);
4472 #ifdef REFINE_STATS
4473 {START_TIMER
4474 #endif
4475     run=0;
4476     rle_index=0;
4477     for(i=start_i; i<=last_non_zero; i++){
4478         int j= perm_scantable[i];
4479         const int level= block[j];
4480         int coeff;
4481
4482         if(level){
4483             if(level<0) coeff= qmul*level - qadd;
4484             else        coeff= qmul*level + qadd;
4485             run_tab[rle_index++]=run;
4486             run=0;
4487
4488             s->mpvencdsp.add_8x8basis(rem, basis[j], coeff);
4489         }else{
4490             run++;
4491         }
4492     }
4493 #ifdef REFINE_STATS
4494 if(last_non_zero>0){
4495 STOP_TIMER("init rem[]")
4496 }
4497 }
4498
4499 {START_TIMER
4500 #endif
4501     for(;;){
4502         int best_score = s->mpvencdsp.try_8x8basis(rem, weight, basis[0], 0);
4503         int best_coeff=0;
4504         int best_change=0;
4505         int run2, best_unquant_change=0, analyze_gradient;
4506 #ifdef REFINE_STATS
4507 {START_TIMER
4508 #endif
4509         analyze_gradient = last_non_zero > 2 || s->quantizer_noise_shaping >= 3;
4510
4511         if(analyze_gradient){
4512 #ifdef REFINE_STATS
4513 {START_TIMER
4514 #endif
4515             for(i=0; i<64; i++){
4516                 int w= weight[i];
4517
4518                 d1[i] = (rem[i]*w*w + (1<<(RECON_SHIFT+12-1)))>>(RECON_SHIFT+12);
4519             }
4520 #ifdef REFINE_STATS
4521 STOP_TIMER("rem*w*w")}
4522 {START_TIMER
4523 #endif
4524             s->fdsp.fdct(d1);
4525 #ifdef REFINE_STATS
4526 STOP_TIMER("dct")}
4527 #endif
4528         }
4529
4530         if(start_i){
4531             const int level= block[0];
4532             int change, old_coeff;
4533
4534             av_assert2(s->mb_intra);
4535
4536             old_coeff= q*level;
4537
4538             for(change=-1; change<=1; change+=2){
4539                 int new_level= level + change;
4540                 int score, new_coeff;
4541
4542                 new_coeff= q*new_level;
4543                 if(new_coeff >= 2048 || new_coeff < 0)
4544                     continue;
4545
4546                 score = s->mpvencdsp.try_8x8basis(rem, weight, basis[0],
4547                                                   new_coeff - old_coeff);
4548                 if(score<best_score){
4549                     best_score= score;
4550                     best_coeff= 0;
4551                     best_change= change;
4552                     best_unquant_change= new_coeff - old_coeff;
4553                 }
4554             }
4555         }
4556
4557         run=0;
4558         rle_index=0;
4559         run2= run_tab[rle_index++];
4560         prev_level=0;
4561         prev_run=0;
4562
4563         for(i=start_i; i<64; i++){
4564             int j= perm_scantable[i];
4565             const int level= block[j];
4566             int change, old_coeff;
4567
4568             if(s->quantizer_noise_shaping < 3 && i > last_non_zero + 1)
4569                 break;
4570
4571             if(level){
4572                 if(level<0) old_coeff= qmul*level - qadd;
4573                 else        old_coeff= qmul*level + qadd;
4574                 run2= run_tab[rle_index++]; //FIXME ! maybe after last
4575             }else{
4576                 old_coeff=0;
4577                 run2--;
4578                 av_assert2(run2>=0 || i >= last_non_zero );
4579             }
4580
4581             for(change=-1; change<=1; change+=2){
4582                 int new_level= level + change;
4583                 int score, new_coeff, unquant_change;
4584
4585                 score=0;
4586                 if(s->quantizer_noise_shaping < 2 && FFABS(new_level) > FFABS(level))
4587                    continue;
4588
4589                 if(new_level){
4590                     if(new_level<0) new_coeff= qmul*new_level - qadd;
4591                     else            new_coeff= qmul*new_level + qadd;
4592                     if(new_coeff >= 2048 || new_coeff <= -2048)
4593                         continue;
4594                     //FIXME check for overflow
4595
4596                     if(level){
4597                         if(level < 63 && level > -63){
4598                             if(i < last_non_zero)
4599                                 score +=   length[UNI_AC_ENC_INDEX(run, new_level+64)]
4600                                          - length[UNI_AC_ENC_INDEX(run, level+64)];
4601                             else
4602                                 score +=   last_length[UNI_AC_ENC_INDEX(run, new_level+64)]
4603                                          - last_length[UNI_AC_ENC_INDEX(run, level+64)];
4604                         }
4605                     }else{
4606                         av_assert2(FFABS(new_level)==1);
4607
4608                         if(analyze_gradient){
4609                             int g= d1[ scantable[i] ];
4610                             if(g && (g^new_level) >= 0)
4611                                 continue;
4612                         }
4613
4614                         if(i < last_non_zero){
4615                             int next_i= i + run2 + 1;
4616                             int next_level= block[ perm_scantable[next_i] ] + 64;
4617
4618                             if(next_level&(~127))
4619                                 next_level= 0;
4620
4621                             if(next_i < last_non_zero)
4622                                 score +=   length[UNI_AC_ENC_INDEX(run, 65)]
4623                                          + length[UNI_AC_ENC_INDEX(run2, next_level)]
4624                                          - length[UNI_AC_ENC_INDEX(run + run2 + 1, next_level)];
4625                             else
4626                                 score +=  length[UNI_AC_ENC_INDEX(run, 65)]
4627                                         + last_length[UNI_AC_ENC_INDEX(run2, next_level)]
4628                                         - last_length[UNI_AC_ENC_INDEX(run + run2 + 1, next_level)];
4629                         }else{
4630                             score += last_length[UNI_AC_ENC_INDEX(run, 65)];
4631                             if(prev_level){
4632                                 score +=  length[UNI_AC_ENC_INDEX(prev_run, prev_level)]
4633                                         - last_length[UNI_AC_ENC_INDEX(prev_run, prev_level)];
4634                             }
4635                         }
4636                     }
4637                 }else{
4638                     new_coeff=0;
4639                     av_assert2(FFABS(level)==1);
4640
4641                     if(i < last_non_zero){
4642                         int next_i= i + run2 + 1;
4643                         int next_level= block[ perm_scantable[next_i] ] + 64;
4644
4645                         if(next_level&(~127))
4646                             next_level= 0;
4647
4648                         if(next_i < last_non_zero)
4649                             score +=   length[UNI_AC_ENC_INDEX(run + run2 + 1, next_level)]
4650                                      - length[UNI_AC_ENC_INDEX(run2, next_level)]
4651                                      - length[UNI_AC_ENC_INDEX(run, 65)];
4652                         else
4653                             score +=   last_length[UNI_AC_ENC_INDEX(run + run2 + 1, next_level)]
4654                                      - last_length[UNI_AC_ENC_INDEX(run2, next_level)]
4655                                      - length[UNI_AC_ENC_INDEX(run, 65)];
4656                     }else{
4657                         score += -last_length[UNI_AC_ENC_INDEX(run, 65)];
4658                         if(prev_level){
4659                             score +=  last_length[UNI_AC_ENC_INDEX(prev_run, prev_level)]
4660                                     - length[UNI_AC_ENC_INDEX(prev_run, prev_level)];
4661                         }
4662                     }
4663                 }
4664
4665                 score *= lambda;
4666
4667                 unquant_change= new_coeff - old_coeff;
4668                 av_assert2((score < 100*lambda && score > -100*lambda) || lambda==0);
4669
4670                 score += s->mpvencdsp.try_8x8basis(rem, weight, basis[j],
4671                                                    unquant_change);
4672                 if(score<best_score){
4673                     best_score= score;
4674                     best_coeff= i;
4675                     best_change= change;
4676                     best_unquant_change= unquant_change;
4677                 }
4678             }
4679             if(level){
4680                 prev_level= level + 64;
4681                 if(prev_level&(~127))
4682                     prev_level= 0;
4683                 prev_run= run;
4684                 run=0;
4685             }else{
4686                 run++;
4687             }
4688         }
4689 #ifdef REFINE_STATS
4690 STOP_TIMER("iterative step")}
4691 #endif
4692
4693         if(best_change){
4694             int j= perm_scantable[ best_coeff ];
4695
4696             block[j] += best_change;
4697
4698             if(best_coeff > last_non_zero){
4699                 last_non_zero= best_coeff;
4700                 av_assert2(block[j]);
4701 #ifdef REFINE_STATS
4702 after_last++;
4703 #endif
4704             }else{
4705 #ifdef REFINE_STATS
4706 if(block[j]){
4707     if(block[j] - best_change){
4708         if(FFABS(block[j]) > FFABS(block[j] - best_change)){
4709             raise++;
4710         }else{
4711             lower++;
4712         }
4713     }else{
4714         from_zero++;
4715     }
4716 }else{
4717     to_zero++;
4718 }
4719 #endif
4720                 for(; last_non_zero>=start_i; last_non_zero--){
4721                     if(block[perm_scantable[last_non_zero]])
4722                         break;
4723                 }
4724             }
4725 #ifdef REFINE_STATS
4726 count++;
4727 if(256*256*256*64 % count == 0){
4728     av_log(s->avctx, AV_LOG_DEBUG, "after_last:%d to_zero:%d from_zero:%d raise:%d lower:%d sign:%d xyp:%d/%d/%d\n", after_last, to_zero, from_zero, raise, lower, messed_sign, s->mb_x, s->mb_y, s->picture_number);
4729 }
4730 #endif
4731             run=0;
4732             rle_index=0;
4733             for(i=start_i; i<=last_non_zero; i++){
4734                 int j= perm_scantable[i];
4735                 const int level= block[j];
4736
4737                  if(level){
4738                      run_tab[rle_index++]=run;
4739                      run=0;
4740                  }else{
4741                      run++;
4742                  }
4743             }
4744
4745             s->mpvencdsp.add_8x8basis(rem, basis[j], best_unquant_change);
4746         }else{
4747             break;
4748         }
4749     }
4750 #ifdef REFINE_STATS
4751 if(last_non_zero>0){
4752 STOP_TIMER("iterative search")
4753 }
4754 }
4755 #endif
4756
4757     return last_non_zero;
4758 }
4759
4760 /**
4761  * Permute an 8x8 block according to permutation.
4762  * @param block the block which will be permuted according to
4763  *              the given permutation vector
4764  * @param permutation the permutation vector
4765  * @param last the last non zero coefficient in scantable order, used to
4766  *             speed the permutation up
4767  * @param scantable the used scantable, this is only used to speed the
4768  *                  permutation up, the block is not (inverse) permutated
4769  *                  to scantable order!
4770  */
4771 void ff_block_permute(int16_t *block, uint8_t *permutation,
4772                       const uint8_t *scantable, int last)
4773 {
4774     int i;
4775     int16_t temp[64];
4776
4777     if (last <= 0)
4778         return;
4779     //FIXME it is ok but not clean and might fail for some permutations
4780     // if (permutation[1] == 1)
4781     // return;
4782
4783     for (i = 0; i <= last; i++) {
4784         const int j = scantable[i];
4785         temp[j] = block[j];
4786         block[j] = 0;
4787     }
4788
4789     for (i = 0; i <= last; i++) {
4790         const int j = scantable[i];
4791         const int perm_j = permutation[j];
4792         block[perm_j] = temp[j];
4793     }
4794 }
4795
4796 int ff_dct_quantize_c(MpegEncContext *s,
4797                         int16_t *block, int n,
4798                         int qscale, int *overflow)
4799 {
4800     int i, j, level, last_non_zero, q, start_i;
4801     const int *qmat;
4802     const uint8_t *scantable= s->intra_scantable.scantable;
4803     int bias;
4804     int max=0;
4805     unsigned int threshold1, threshold2;
4806
4807     s->fdsp.fdct(block);
4808
4809     if(s->dct_error_sum)
4810         s->denoise_dct(s, block);
4811
4812     if (s->mb_intra) {
4813         if (!s->h263_aic) {
4814             if (n < 4)
4815                 q = s->y_dc_scale;
4816             else
4817                 q = s->c_dc_scale;
4818             q = q << 3;
4819         } else
4820             /* For AIC we skip quant/dequant of INTRADC */
4821             q = 1 << 3;
4822
4823         /* note: block[0] is assumed to be positive */
4824         block[0] = (block[0] + (q >> 1)) / q;
4825         start_i = 1;
4826         last_non_zero = 0;
4827         qmat = n < 4 ? s->q_intra_matrix[qscale] : s->q_chroma_intra_matrix[qscale];
4828         bias= s->intra_quant_bias*(1<<(QMAT_SHIFT - QUANT_BIAS_SHIFT));
4829     } else {
4830         start_i = 0;
4831         last_non_zero = -1;
4832         qmat = s->q_inter_matrix[qscale];
4833         bias= s->inter_quant_bias*(1<<(QMAT_SHIFT - QUANT_BIAS_SHIFT));
4834     }
4835     threshold1= (1<<QMAT_SHIFT) - bias - 1;
4836     threshold2= (threshold1<<1);
4837     for(i=63;i>=start_i;i--) {
4838         j = scantable[i];
4839         level = block[j] * qmat[j];
4840
4841         if(((unsigned)(level+threshold1))>threshold2){
4842             last_non_zero = i;
4843             break;
4844         }else{
4845             block[j]=0;
4846         }
4847     }
4848     for(i=start_i; i<=last_non_zero; i++) {
4849         j = scantable[i];
4850         level = block[j] * qmat[j];
4851
4852 //        if(   bias+level >= (1<<QMAT_SHIFT)
4853 //           || bias-level >= (1<<QMAT_SHIFT)){
4854         if(((unsigned)(level+threshold1))>threshold2){
4855             if(level>0){
4856                 level= (bias + level)>>QMAT_SHIFT;
4857                 block[j]= level;
4858             }else{
4859                 level= (bias - level)>>QMAT_SHIFT;
4860                 block[j]= -level;
4861             }
4862             max |=level;
4863         }else{
4864             block[j]=0;
4865         }
4866     }
4867     *overflow= s->max_qcoeff < max; //overflow might have happened
4868
4869     /* we need this permutation so that we correct the IDCT, we only permute the !=0 elements */
4870     if (s->idsp.perm_type != FF_IDCT_PERM_NONE)
4871         ff_block_permute(block, s->idsp.idct_permutation,
4872                       scantable, last_non_zero);
4873
4874     return last_non_zero;
4875 }
4876
4877 #define OFFSET(x) offsetof(MpegEncContext, x)
4878 #define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM
4879 static const AVOption h263_options[] = {
4880     { "obmc",         "use overlapped block motion compensation.", OFFSET(obmc), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE },
4881     { "mb_info",      "emit macroblock info for RFC 2190 packetization, the parameter value is the maximum payload size", OFFSET(mb_info), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, VE },
4882     FF_MPV_COMMON_OPTS
4883     { NULL },
4884 };
4885
4886 static const AVClass h263_class = {
4887     .class_name = "H.263 encoder",
4888     .item_name  = av_default_item_name,
4889     .option     = h263_options,
4890     .version    = LIBAVUTIL_VERSION_INT,
4891 };
4892
4893 AVCodec ff_h263_encoder = {
4894     .name           = "h263",
4895     .long_name      = NULL_IF_CONFIG_SMALL("H.263 / H.263-1996"),
4896     .type           = AVMEDIA_TYPE_VIDEO,
4897     .id             = AV_CODEC_ID_H263,
4898     .priv_data_size = sizeof(MpegEncContext),
4899     .init           = ff_mpv_encode_init,
4900     .encode2        = ff_mpv_encode_picture,
4901     .close          = ff_mpv_encode_end,
4902     .pix_fmts= (const enum AVPixelFormat[]){AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE},
4903     .priv_class     = &h263_class,
4904 };
4905
4906 static const AVOption h263p_options[] = {
4907     { "umv",        "Use unlimited motion vectors.",    OFFSET(umvplus),       AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE },
4908     { "aiv",        "Use alternative inter VLC.",       OFFSET(alt_inter_vlc), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE },
4909     { "obmc",       "use overlapped block motion compensation.", OFFSET(obmc), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE },
4910     { "structured_slices", "Write slice start position at every GOB header instead of just GOB number.", OFFSET(h263_slice_structured), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE},
4911     FF_MPV_COMMON_OPTS
4912     { NULL },
4913 };
4914 static const AVClass h263p_class = {
4915     .class_name = "H.263p encoder",
4916     .item_name  = av_default_item_name,
4917     .option     = h263p_options,
4918     .version    = LIBAVUTIL_VERSION_INT,
4919 };
4920
4921 AVCodec ff_h263p_encoder = {
4922     .name           = "h263p",
4923     .long_name      = NULL_IF_CONFIG_SMALL("H.263+ / H.263-1998 / H.263 version 2"),
4924     .type           = AVMEDIA_TYPE_VIDEO,
4925     .id             = AV_CODEC_ID_H263P,
4926     .priv_data_size = sizeof(MpegEncContext),
4927     .init           = ff_mpv_encode_init,
4928     .encode2        = ff_mpv_encode_picture,
4929     .close          = ff_mpv_encode_end,
4930     .capabilities   = AV_CODEC_CAP_SLICE_THREADS,
4931     .pix_fmts       = (const enum AVPixelFormat[]){ AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE },
4932     .priv_class     = &h263p_class,
4933 };
4934
4935 static const AVClass msmpeg4v2_class = {
4936     .class_name = "msmpeg4v2 encoder",
4937     .item_name  = av_default_item_name,
4938     .option     = ff_mpv_generic_options,
4939     .version    = LIBAVUTIL_VERSION_INT,
4940 };
4941
4942 AVCodec ff_msmpeg4v2_encoder = {
4943     .name           = "msmpeg4v2",
4944     .long_name      = NULL_IF_CONFIG_SMALL("MPEG-4 part 2 Microsoft variant version 2"),
4945     .type           = AVMEDIA_TYPE_VIDEO,
4946     .id             = AV_CODEC_ID_MSMPEG4V2,
4947     .priv_data_size = sizeof(MpegEncContext),
4948     .init           = ff_mpv_encode_init,
4949     .encode2        = ff_mpv_encode_picture,
4950     .close          = ff_mpv_encode_end,
4951     .pix_fmts       = (const enum AVPixelFormat[]){ AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE },
4952     .priv_class     = &msmpeg4v2_class,
4953 };
4954
4955 static const AVClass msmpeg4v3_class = {
4956     .class_name = "msmpeg4v3 encoder",
4957     .item_name  = av_default_item_name,
4958     .option     = ff_mpv_generic_options,
4959     .version    = LIBAVUTIL_VERSION_INT,
4960 };
4961
4962 AVCodec ff_msmpeg4v3_encoder = {
4963     .name           = "msmpeg4",
4964     .long_name      = NULL_IF_CONFIG_SMALL("MPEG-4 part 2 Microsoft variant version 3"),
4965     .type           = AVMEDIA_TYPE_VIDEO,
4966     .id             = AV_CODEC_ID_MSMPEG4V3,
4967     .priv_data_size = sizeof(MpegEncContext),
4968     .init           = ff_mpv_encode_init,
4969     .encode2        = ff_mpv_encode_picture,
4970     .close          = ff_mpv_encode_end,
4971     .pix_fmts       = (const enum AVPixelFormat[]){ AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE },
4972     .priv_class     = &msmpeg4v3_class,
4973 };
4974
4975 static const AVClass wmv1_class = {
4976     .class_name = "wmv1 encoder",
4977     .item_name  = av_default_item_name,
4978     .option     = ff_mpv_generic_options,
4979     .version    = LIBAVUTIL_VERSION_INT,
4980 };
4981
4982 AVCodec ff_wmv1_encoder = {
4983     .name           = "wmv1",
4984     .long_name      = NULL_IF_CONFIG_SMALL("Windows Media Video 7"),
4985     .type           = AVMEDIA_TYPE_VIDEO,
4986     .id             = AV_CODEC_ID_WMV1,
4987     .priv_data_size = sizeof(MpegEncContext),
4988     .init           = ff_mpv_encode_init,
4989     .encode2        = ff_mpv_encode_picture,
4990     .close          = ff_mpv_encode_end,
4991     .pix_fmts       = (const enum AVPixelFormat[]){ AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE },
4992     .priv_class     = &wmv1_class,
4993 };