git.sesse.net Git - ffmpeg/blob - libavcodec/mpegvideo_enc.c

   1 /*
   2  * The simplest mpeg encoder (well, it was the simplest!)
   3  * Copyright (c) 2000,2001 Fabrice Bellard
   4  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
   5  *
   6  * 4MV & hq & B-frame encoding stuff by Michael Niedermayer <michaelni@gmx.at>
   7  *
   8  * This file is part of Libav.
   9  *
  10  * Libav is free software; you can redistribute it and/or
  11  * modify it under the terms of the GNU Lesser General Public
  12  * License as published by the Free Software Foundation; either
  13  * version 2.1 of the License, or (at your option) any later version.
  14  *
  15  * Libav is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18  * Lesser General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU Lesser General Public
  21  * License along with Libav; if not, write to the Free Software
  22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  23  */
  24
  25 /**
  26  * @file
  27  * The simplest mpeg encoder (well, it was the simplest!).
  28  */
  29
  30 #include <stdint.h>
  31
  32 #include "libavutil/internal.h"
  33 #include "libavutil/intmath.h"
  34 #include "libavutil/mathematics.h"
  35 #include "libavutil/pixdesc.h"
  36 #include "libavutil/opt.h"
  37 #include "libavutil/timer.h"
  38 #include "avcodec.h"
  39 #include "dct.h"
  40 #include "idctdsp.h"
  41 #include "mpeg12.h"
  42 #include "mpegvideo.h"
  43 #include "mpegvideodata.h"
  44 #include "h261.h"
  45 #include "h263.h"
  46 #include "h263data.h"
  47 #include "mjpegenc_common.h"
  48 #include "mathops.h"
  49 #include "mpegutils.h"
  50 #include "mjpegenc.h"
  51 #include "msmpeg4.h"
  52 #include "pixblockdsp.h"
  53 #include "qpeldsp.h"
  54 #include "faandct.h"
  55 #include "thread.h"
  56 #include "aandcttab.h"
  57 #include "flv.h"
  58 #include "mpeg4video.h"
  59 #include "internal.h"
  60 #include "bytestream.h"
  61 #include "wmv2.h"
  62 #include "rv10.h"
  63 #include <limits.h>
  64
  65 #define QUANT_BIAS_SHIFT 8
  66
  67 #define QMAT_SHIFT_MMX 16
  68 #define QMAT_SHIFT 22
  69
  70 static int encode_picture(MpegEncContext *s, int picture_number);
  71 static int dct_quantize_refine(MpegEncContext *s, int16_t *block, int16_t *weight, int16_t *orig, int n, int qscale);
  72 static int sse_mb(MpegEncContext *s);
  73 static void denoise_dct_c(MpegEncContext *s, int16_t *block);
  74 static int dct_quantize_trellis_c(MpegEncContext *s, int16_t *block, int n, int qscale, int *overflow);
  75
  76 static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_MV * 2 + 1];
  77 static uint8_t default_fcode_tab[MAX_MV * 2 + 1];
  78
  79 const AVOption ff_mpv_generic_options[] = {
  80     FF_MPV_COMMON_OPTS
  81     { NULL },
  82 };
  83
  84 void ff_convert_matrix(MpegEncContext *s, int (*qmat)[64],
  85                        uint16_t (*qmat16)[2][64],
  86                        const uint16_t *quant_matrix,
  87                        int bias, int qmin, int qmax, int intra)
  88 {
  89     FDCTDSPContext *fdsp = &s->fdsp;
  90     int qscale;
  91     int shift = 0;
  92
  93     for (qscale = qmin; qscale <= qmax; qscale++) {
  94         int i;
  95         if (fdsp->fdct == ff_jpeg_fdct_islow_8  ||
  96 #if CONFIG_FAANDCT
  97             fdsp->fdct == ff_faandct            ||
  98 #endif /* CONFIG_FAANDCT */
  99             fdsp->fdct == ff_jpeg_fdct_islow_10) {
 100             for (i = 0; i < 64; i++) {
 101                 const int j = s->idsp.idct_permutation[i];
 102                 int64_t den = (int64_t) qscale * quant_matrix[j];
 103                 /* 16 <= qscale * quant_matrix[i] <= 7905
 104                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
 105                  *             19952 <=              x  <= 249205026
 106                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
 107                  *           3444240 >= (1 << 36) / (x) >= 275 */
 108
 109                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) / den);
 110             }
 111         } else if (fdsp->fdct == ff_fdct_ifast) {
 112             for (i = 0; i < 64; i++) {
 113                 const int j = s->idsp.idct_permutation[i];
 114                 int64_t den = ff_aanscales[i] * (int64_t) qscale * quant_matrix[j];
 115                 /* 16 <= qscale * quant_matrix[i] <= 7905
 116                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
 117                  *             19952 <=              x  <= 249205026
 118                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
 119                  *           3444240 >= (1 << 36) / (x) >= 275 */
 120
 121                 qmat[qscale][i] = (int)((UINT64_C(1) << (QMAT_SHIFT + 14)) / den);
 122             }
 123         } else {
 124             for (i = 0; i < 64; i++) {
 125                 const int j = s->idsp.idct_permutation[i];
 126                 int64_t den = (int64_t) qscale * quant_matrix[j];
 127                 /* We can safely suppose that 16 <= quant_matrix[i] <= 255
 128                  * Assume x = qscale * quant_matrix[i]
 129                  * So             16 <=              x  <= 7905
 130                  * so (1 << 19) / 16 >= (1 << 19) / (x) >= (1 << 19) / 7905
 131                  * so          32768 >= (1 << 19) / (x) >= 67 */
 132                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) / den);
 133                 //qmat  [qscale][i] = (1 << QMAT_SHIFT_MMX) /
 134                 //                    (qscale * quant_matrix[i]);
 135                 qmat16[qscale][0][i] = (1 << QMAT_SHIFT_MMX) / den;
 136
 137                 if (qmat16[qscale][0][i] == 0 ||
 138                     qmat16[qscale][0][i] == 128 * 256)
 139                     qmat16[qscale][0][i] = 128 * 256 - 1;
 140                 qmat16[qscale][1][i] =
 141                     ROUNDED_DIV(bias << (16 - QUANT_BIAS_SHIFT),
 142                                 qmat16[qscale][0][i]);
 143             }
 144         }
 145
 146         for (i = intra; i < 64; i++) {
 147             int64_t max = 8191;
 148             if (fdsp->fdct == ff_fdct_ifast) {
 149                 max = (8191LL * ff_aanscales[i]) >> 14;
 150             }
 151             while (((max * qmat[qscale][i]) >> shift) > INT_MAX) {
 152                 shift++;
 153             }
 154         }
 155     }
 156     if (shift) {
 157         av_log(NULL, AV_LOG_INFO,
 158                "Warning, QMAT_SHIFT is larger than %d, overflows possible\n",
 159                QMAT_SHIFT - shift);
 160     }
 161 }
 162
 163 static inline void update_qscale(MpegEncContext *s)
 164 {
 165     s->qscale = (s->lambda * 139 + FF_LAMBDA_SCALE * 64) >>
 166                 (FF_LAMBDA_SHIFT + 7);
 167     s->qscale = av_clip(s->qscale, s->avctx->qmin, s->avctx->qmax);
 168
 169     s->lambda2 = (s->lambda * s->lambda + FF_LAMBDA_SCALE / 2) >>
 170                  FF_LAMBDA_SHIFT;
 171 }
 172
 173 void ff_write_quant_matrix(PutBitContext *pb, uint16_t *matrix)
 174 {
 175     int i;
 176
 177     if (matrix) {
 178         put_bits(pb, 1, 1);
 179         for (i = 0; i < 64; i++) {
 180             put_bits(pb, 8, matrix[ff_zigzag_direct[i]]);
 181         }
 182     } else
 183         put_bits(pb, 1, 0);
 184 }
 185
 186 /**
 187  * init s->current_picture.qscale_table from s->lambda_table
 188  */
 189 void ff_init_qscale_tab(MpegEncContext *s)
 190 {
 191     int8_t * const qscale_table = s->current_picture.qscale_table;
 192     int i;
 193
 194     for (i = 0; i < s->mb_num; i++) {
 195         unsigned int lam = s->lambda_table[s->mb_index2xy[i]];
 196         int qp = (lam * 139 + FF_LAMBDA_SCALE * 64) >> (FF_LAMBDA_SHIFT + 7);
 197         qscale_table[s->mb_index2xy[i]] = av_clip(qp, s->avctx->qmin,
 198                                                   s->avctx->qmax);
 199     }
 200 }
 201
 202 static void update_duplicate_context_after_me(MpegEncContext *dst,
 203                                               MpegEncContext *src)
 204 {
 205 #define COPY(a) dst->a= src->a
 206     COPY(pict_type);
 207     COPY(current_picture);
 208     COPY(f_code);
 209     COPY(b_code);
 210     COPY(qscale);
 211     COPY(lambda);
 212     COPY(lambda2);
 213     COPY(picture_in_gop_number);
 214     COPY(gop_picture_number);
 215     COPY(frame_pred_frame_dct); // FIXME don't set in encode_header
 216     COPY(progressive_frame);    // FIXME don't set in encode_header
 217     COPY(partitioned_frame);    // FIXME don't set in encode_header
 218 #undef COPY
 219 }
 220
 221 /**
 222  * Set the given MpegEncContext to defaults for encoding.
 223  * the changed fields will not depend upon the prior state of the MpegEncContext.
 224  */
 225 static void mpv_encode_defaults(MpegEncContext *s)
 226 {
 227     int i;
 228     ff_mpv_common_defaults(s);
 229
 230     for (i = -16; i < 16; i++) {
 231         default_fcode_tab[i + MAX_MV] = 1;
 232     }
 233     s->me.mv_penalty = default_mv_penalty;
 234     s->fcode_tab     = default_fcode_tab;
 235
 236     s->input_picture_number  = 0;
 237     s->picture_in_gop_number = 0;
 238 }
 239
 240 /* init video encoder */
 241 av_cold int ff_mpv_encode_init(AVCodecContext *avctx)
 242 {
 243     MpegEncContext *s = avctx->priv_data;
 244     AVCPBProperties *cpb_props;
 245     int i, ret, format_supported;
 246
 247     mpv_encode_defaults(s);
 248
 249     switch (avctx->codec_id) {
 250     case AV_CODEC_ID_MPEG2VIDEO:
 251         if (avctx->pix_fmt != AV_PIX_FMT_YUV420P &&
 252             avctx->pix_fmt != AV_PIX_FMT_YUV422P) {
 253             av_log(avctx, AV_LOG_ERROR,
 254                    "only YUV420 and YUV422 are supported\n");
 255             return -1;
 256         }
 257         break;
 258     case AV_CODEC_ID_MJPEG:
 259         format_supported = 0;
 260         /* JPEG color space */
 261         if (avctx->pix_fmt == AV_PIX_FMT_YUVJ420P ||
 262             avctx->pix_fmt == AV_PIX_FMT_YUVJ422P ||
 263             (avctx->color_range == AVCOL_RANGE_JPEG &&
 264              (avctx->pix_fmt == AV_PIX_FMT_YUV420P ||
 265               avctx->pix_fmt == AV_PIX_FMT_YUV422P)))
 266             format_supported = 1;
 267         /* MPEG color space */
 268         else if (avctx->strict_std_compliance <= FF_COMPLIANCE_UNOFFICIAL &&
 269                  (avctx->pix_fmt == AV_PIX_FMT_YUV420P ||
 270                   avctx->pix_fmt == AV_PIX_FMT_YUV422P))
 271             format_supported = 1;
 272
 273         if (!format_supported) {
 274             av_log(avctx, AV_LOG_ERROR, "colorspace not supported in jpeg\n");
 275             return -1;
 276         }
 277         break;
 278     default:
 279         if (avctx->pix_fmt != AV_PIX_FMT_YUV420P) {
 280             av_log(avctx, AV_LOG_ERROR, "only YUV420 is supported\n");
 281             return -1;
 282         }
 283     }
 284
 285     switch (avctx->pix_fmt) {
 286     case AV_PIX_FMT_YUVJ422P:
 287     case AV_PIX_FMT_YUV422P:
 288         s->chroma_format = CHROMA_422;
 289         break;
 290     case AV_PIX_FMT_YUVJ420P:
 291     case AV_PIX_FMT_YUV420P:
 292     default:
 293         s->chroma_format = CHROMA_420;
 294         break;
 295     }
 296
 297 #if FF_API_PRIVATE_OPT
 298 FF_DISABLE_DEPRECATION_WARNINGS
 299     if (avctx->rtp_payload_size)
 300         s->rtp_payload_size = avctx->rtp_payload_size;
 301     if (avctx->me_penalty_compensation)
 302         s->me_penalty_compensation = avctx->me_penalty_compensation;
 303     if (avctx->pre_me)
 304         s->me_pre = avctx->pre_me;
 305 FF_ENABLE_DEPRECATION_WARNINGS
 306 #endif
 307
 308     s->bit_rate = avctx->bit_rate;
 309     s->width    = avctx->width;
 310     s->height   = avctx->height;
 311     if (avctx->gop_size > 600 &&
 312         avctx->strict_std_compliance > FF_COMPLIANCE_EXPERIMENTAL) {
 313         av_log(avctx, AV_LOG_ERROR,
 314                "Warning keyframe interval too large! reducing it ...\n");
 315         avctx->gop_size = 600;
 316     }
 317     s->gop_size     = avctx->gop_size;
 318     s->avctx        = avctx;
 319     if (avctx->max_b_frames > MAX_B_FRAMES) {
 320         av_log(avctx, AV_LOG_ERROR, "Too many B-frames requested, maximum "
 321                "is %d.\n", MAX_B_FRAMES);
 322     }
 323     s->max_b_frames = avctx->max_b_frames;
 324     s->codec_id     = avctx->codec->id;
 325     s->strict_std_compliance = avctx->strict_std_compliance;
 326     s->quarter_sample     = (avctx->flags & AV_CODEC_FLAG_QPEL) != 0;
 327     s->rtp_mode           = !!s->rtp_payload_size;
 328     s->intra_dc_precision = avctx->intra_dc_precision;
 329     s->user_specified_pts = AV_NOPTS_VALUE;
 330
 331     if (s->gop_size <= 1) {
 332         s->intra_only = 1;
 333         s->gop_size   = 12;
 334     } else {
 335         s->intra_only = 0;
 336     }
 337
 338     /* Fixed QSCALE */
 339     s->fixed_qscale = !!(avctx->flags & AV_CODEC_FLAG_QSCALE);
 340
 341     s->adaptive_quant = (s->avctx->lumi_masking ||
 342                          s->avctx->dark_masking ||
 343                          s->avctx->temporal_cplx_masking ||
 344                          s->avctx->spatial_cplx_masking  ||
 345                          s->avctx->p_masking      ||
 346                          s->border_masking ||
 347                          (s->mpv_flags & FF_MPV_FLAG_QP_RD)) &&
 348                         !s->fixed_qscale;
 349
 350     s->loop_filter = !!(s->avctx->flags & AV_CODEC_FLAG_LOOP_FILTER);
 351
 352     if (avctx->rc_max_rate && !avctx->rc_buffer_size) {
 353         av_log(avctx, AV_LOG_ERROR,
 354                "a vbv buffer size is needed, "
 355                "for encoding with a maximum bitrate\n");
 356         return -1;
 357     }
 358
 359     if (avctx->rc_min_rate && avctx->rc_max_rate != avctx->rc_min_rate) {
 360         av_log(avctx, AV_LOG_INFO,
 361                "Warning min_rate > 0 but min_rate != max_rate isn't recommended!\n");
 362     }
 363
 364     if (avctx->rc_min_rate && avctx->rc_min_rate > avctx->bit_rate) {
 365         av_log(avctx, AV_LOG_ERROR, "bitrate below min bitrate\n");
 366         return -1;
 367     }
 368
 369     if (avctx->rc_max_rate && avctx->rc_max_rate < avctx->bit_rate) {
 370         av_log(avctx, AV_LOG_INFO, "bitrate above max bitrate\n");
 371         return -1;
 372     }
 373
 374     if (avctx->rc_max_rate &&
 375         avctx->rc_max_rate == avctx->bit_rate &&
 376         avctx->rc_max_rate != avctx->rc_min_rate) {
 377         av_log(avctx, AV_LOG_INFO,
 378                "impossible bitrate constraints, this will fail\n");
 379     }
 380
 381     if (avctx->rc_buffer_size &&
 382         avctx->bit_rate * (int64_t)avctx->time_base.num >
 383             avctx->rc_buffer_size * (int64_t)avctx->time_base.den) {
 384         av_log(avctx, AV_LOG_ERROR, "VBV buffer too small for bitrate\n");
 385         return -1;
 386     }
 387
 388     if (!s->fixed_qscale &&
 389         avctx->bit_rate * av_q2d(avctx->time_base) >
 390             avctx->bit_rate_tolerance) {
 391         av_log(avctx, AV_LOG_ERROR,
 392                "bitrate tolerance too small for bitrate\n");
 393         return -1;
 394     }
 395
 396     if (s->avctx->rc_max_rate &&
 397         s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
 398         (s->codec_id == AV_CODEC_ID_MPEG1VIDEO ||
 399          s->codec_id == AV_CODEC_ID_MPEG2VIDEO) &&
 400         90000LL * (avctx->rc_buffer_size - 1) >
 401             s->avctx->rc_max_rate * 0xFFFFLL) {
 402         av_log(avctx, AV_LOG_INFO,
 403                "Warning vbv_delay will be set to 0xFFFF (=VBR) as the "
 404                "specified vbv buffer is too large for the given bitrate!\n");
 405     }
 406
 407     if ((s->avctx->flags & AV_CODEC_FLAG_4MV) && s->codec_id != AV_CODEC_ID_MPEG4 &&
 408         s->codec_id != AV_CODEC_ID_H263 && s->codec_id != AV_CODEC_ID_H263P &&
 409         s->codec_id != AV_CODEC_ID_FLV1) {
 410         av_log(avctx, AV_LOG_ERROR, "4MV not supported by codec\n");
 411         return -1;
 412     }
 413
 414     if (s->obmc && s->avctx->mb_decision != FF_MB_DECISION_SIMPLE) {
 415         av_log(avctx, AV_LOG_ERROR,
 416                "OBMC is only supported with simple mb decision\n");
 417         return -1;
 418     }
 419
 420     if (s->quarter_sample && s->codec_id != AV_CODEC_ID_MPEG4) {
 421         av_log(avctx, AV_LOG_ERROR, "qpel not supported by codec\n");
 422         return -1;
 423     }
 424
 425     if (s->max_b_frames                    &&
 426         s->codec_id != AV_CODEC_ID_MPEG4      &&
 427         s->codec_id != AV_CODEC_ID_MPEG1VIDEO &&
 428         s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
 429         av_log(avctx, AV_LOG_ERROR, "B-frames not supported by codec\n");
 430         return -1;
 431     }
 432
 433     if ((s->codec_id == AV_CODEC_ID_MPEG4 ||
 434          s->codec_id == AV_CODEC_ID_H263  ||
 435          s->codec_id == AV_CODEC_ID_H263P) &&
 436         (avctx->sample_aspect_ratio.num > 255 ||
 437          avctx->sample_aspect_ratio.den > 255)) {
 438         av_log(avctx, AV_LOG_ERROR,
 439                "Invalid pixel aspect ratio %i/%i, limit is 255/255\n",
 440                avctx->sample_aspect_ratio.num, avctx->sample_aspect_ratio.den);
 441         return -1;
 442     }
 443
 444     if ((s->avctx->flags & (AV_CODEC_FLAG_INTERLACED_DCT | AV_CODEC_FLAG_INTERLACED_ME)) &&
 445         s->codec_id != AV_CODEC_ID_MPEG4 && s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
 446         av_log(avctx, AV_LOG_ERROR, "interlacing not supported by codec\n");
 447         return -1;
 448     }
 449
 450 #if FF_API_PRIVATE_OPT
 451     FF_DISABLE_DEPRECATION_WARNINGS
 452     if (avctx->mpeg_quant)
 453         s->mpeg_quant = avctx->mpeg_quant;
 454     FF_ENABLE_DEPRECATION_WARNINGS
 455 #endif
 456
 457     // FIXME mpeg2 uses that too
 458     if (s->mpeg_quant && s->codec_id != AV_CODEC_ID_MPEG4) {
 459         av_log(avctx, AV_LOG_ERROR,
 460                "mpeg2 style quantization not supported by codec\n");
 461         return -1;
 462     }
 463
 464     if ((s->mpv_flags & FF_MPV_FLAG_CBP_RD) && !avctx->trellis) {
 465         av_log(avctx, AV_LOG_ERROR, "CBP RD needs trellis quant\n");
 466         return -1;
 467     }
 468
 469     if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) &&
 470         s->avctx->mb_decision != FF_MB_DECISION_RD) {
 471         av_log(avctx, AV_LOG_ERROR, "QP RD needs mbd=2\n");
 472         return -1;
 473     }
 474
 475 #if FF_API_PRIVATE_OPT
 476 FF_DISABLE_DEPRECATION_WARNINGS
 477     if (avctx->scenechange_threshold)
 478         s->scenechange_threshold = avctx->scenechange_threshold;
 479 FF_ENABLE_DEPRECATION_WARNINGS
 480 #endif
 481
 482     if (s->scenechange_threshold < 1000000000 &&
 483         (s->avctx->flags & AV_CODEC_FLAG_CLOSED_GOP)) {
 484         av_log(avctx, AV_LOG_ERROR,
 485                "closed gop with scene change detection are not supported yet, "
 486                "set threshold to 1000000000\n");
 487         return -1;
 488     }
 489
 490     if (s->avctx->flags & AV_CODEC_FLAG_LOW_DELAY) {
 491         if (s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
 492             av_log(avctx, AV_LOG_ERROR,
 493                   "low delay forcing is only available for mpeg2\n");
 494             return -1;
 495         }
 496         if (s->max_b_frames != 0) {
 497             av_log(avctx, AV_LOG_ERROR,
 498                    "B-frames cannot be used with low delay\n");
 499             return -1;
 500         }
 501     }
 502
 503     if (s->q_scale_type == 1) {
 504         if (avctx->qmax > 12) {
 505             av_log(avctx, AV_LOG_ERROR,
 506                    "non linear quant only supports qmax <= 12 currently\n");
 507             return -1;
 508         }
 509     }
 510
 511     if (avctx->slices > 1 &&
 512         (avctx->codec_id == AV_CODEC_ID_FLV1 || avctx->codec_id == AV_CODEC_ID_H261)) {
 513         av_log(avctx, AV_LOG_ERROR, "Multiple slices are not supported by this codec\n");
 514         return AVERROR(EINVAL);
 515     }
 516
 517     if (s->avctx->thread_count > 1         &&
 518         s->codec_id != AV_CODEC_ID_MPEG4      &&
 519         s->codec_id != AV_CODEC_ID_MPEG1VIDEO &&
 520         s->codec_id != AV_CODEC_ID_MPEG2VIDEO &&
 521         (s->codec_id != AV_CODEC_ID_H263P)) {
 522         av_log(avctx, AV_LOG_ERROR,
 523                "multi threaded encoding not supported by codec\n");
 524         return -1;
 525     }
 526
 527     if (s->avctx->thread_count < 1) {
 528         av_log(avctx, AV_LOG_ERROR,
 529                "automatic thread number detection not supported by codec,"
 530                "patch welcome\n");
 531         return -1;
 532     }
 533
 534     if (!avctx->time_base.den || !avctx->time_base.num) {
 535         av_log(avctx, AV_LOG_ERROR, "framerate not set\n");
 536         return -1;
 537     }
 538
 539 #if FF_API_PRIVATE_OPT
 540 FF_DISABLE_DEPRECATION_WARNINGS
 541     if (avctx->b_frame_strategy)
 542         s->b_frame_strategy = avctx->b_frame_strategy;
 543     if (avctx->b_sensitivity != 40)
 544         s->b_sensitivity = avctx->b_sensitivity;
 545 FF_ENABLE_DEPRECATION_WARNINGS
 546 #endif
 547
 548     if (s->b_frame_strategy && (avctx->flags & AV_CODEC_FLAG_PASS2)) {
 549         av_log(avctx, AV_LOG_INFO,
 550                "notice: b_frame_strategy only affects the first pass\n");
 551         s->b_frame_strategy = 0;
 552     }
 553
 554     i = av_gcd(avctx->time_base.den, avctx->time_base.num);
 555     if (i > 1) {
 556         av_log(avctx, AV_LOG_INFO, "removing common factors from framerate\n");
 557         avctx->time_base.den /= i;
 558         avctx->time_base.num /= i;
 559         //return -1;
 560     }
 561
 562     if (s->mpeg_quant || s->codec_id == AV_CODEC_ID_MPEG1VIDEO ||
 563         s->codec_id == AV_CODEC_ID_MPEG2VIDEO || s->codec_id == AV_CODEC_ID_MJPEG) {
 564         // (a + x * 3 / 8) / x
 565         s->intra_quant_bias = 3 << (QUANT_BIAS_SHIFT - 3);
 566         s->inter_quant_bias = 0;
 567     } else {
 568         s->intra_quant_bias = 0;
 569         // (a - x / 4) / x
 570         s->inter_quant_bias = -(1 << (QUANT_BIAS_SHIFT - 2));
 571     }
 572
 573     if (avctx->codec_id == AV_CODEC_ID_MPEG4 &&
 574         s->avctx->time_base.den > (1 << 16) - 1) {
 575         av_log(avctx, AV_LOG_ERROR,
 576                "timebase %d/%d not supported by MPEG 4 standard, "
 577                "the maximum admitted value for the timebase denominator "
 578                "is %d\n", s->avctx->time_base.num, s->avctx->time_base.den,
 579                (1 << 16) - 1);
 580         return -1;
 581     }
 582     s->time_increment_bits = av_log2(s->avctx->time_base.den - 1) + 1;
 583
 584     switch (avctx->codec->id) {
 585     case AV_CODEC_ID_MPEG1VIDEO:
 586         s->out_format = FMT_MPEG1;
 587         s->low_delay  = !!(s->avctx->flags & AV_CODEC_FLAG_LOW_DELAY);
 588         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
 589         break;
 590     case AV_CODEC_ID_MPEG2VIDEO:
 591         s->out_format = FMT_MPEG1;
 592         s->low_delay  = !!(s->avctx->flags & AV_CODEC_FLAG_LOW_DELAY);
 593         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
 594         s->rtp_mode   = 1;
 595         break;
 596     case AV_CODEC_ID_MJPEG:
 597         s->out_format = FMT_MJPEG;
 598         s->intra_only = 1; /* force intra only for jpeg */
 599         if (!CONFIG_MJPEG_ENCODER ||
 600             ff_mjpeg_encode_init(s) < 0)
 601             return -1;
 602         avctx->delay = 0;
 603         s->low_delay = 1;
 604         break;
 605     case AV_CODEC_ID_H261:
 606         if (!CONFIG_H261_ENCODER)
 607             return -1;
 608         if (ff_h261_get_picture_format(s->width, s->height) < 0) {
 609             av_log(avctx, AV_LOG_ERROR,
 610                    "The specified picture size of %dx%d is not valid for the "
 611                    "H.261 codec.\nValid sizes are 176x144, 352x288\n",
 612                     s->width, s->height);
 613             return -1;
 614         }
 615         s->out_format = FMT_H261;
 616         avctx->delay  = 0;
 617         s->low_delay  = 1;
 618         s->rtp_mode   = 0; /* Sliced encoding not supported */
 619         break;
 620     case AV_CODEC_ID_H263:
 621         if (!CONFIG_H263_ENCODER)
 622         return -1;
 623         if (ff_match_2uint16(ff_h263_format, FF_ARRAY_ELEMS(ff_h263_format),
 624                              s->width, s->height) == 8) {
 625             av_log(avctx, AV_LOG_INFO,
 626                    "The specified picture size of %dx%d is not valid for "
 627                    "the H.263 codec.\nValid sizes are 128x96, 176x144, "
 628                    "352x288, 704x576, and 1408x1152."
 629                    "Try H.263+.\n", s->width, s->height);
 630             return -1;
 631         }
 632         s->out_format = FMT_H263;
 633         avctx->delay  = 0;
 634         s->low_delay  = 1;
 635         break;
 636     case AV_CODEC_ID_H263P:
 637         s->out_format = FMT_H263;
 638         s->h263_plus  = 1;
 639         /* Fx */
 640         s->h263_aic        = (avctx->flags & AV_CODEC_FLAG_AC_PRED) ? 1 : 0;
 641         s->modified_quant  = s->h263_aic;
 642         s->loop_filter     = (avctx->flags & AV_CODEC_FLAG_LOOP_FILTER) ? 1 : 0;
 643         s->unrestricted_mv = s->obmc || s->loop_filter || s->umvplus;
 644
 645         /* /Fx */
 646         /* These are just to be sure */
 647         avctx->delay = 0;
 648         s->low_delay = 1;
 649         break;
 650     case AV_CODEC_ID_FLV1:
 651         s->out_format      = FMT_H263;
 652         s->h263_flv        = 2; /* format = 1; 11-bit codes */
 653         s->unrestricted_mv = 1;
 654         s->rtp_mode  = 0; /* don't allow GOB */
 655         avctx->delay = 0;
 656         s->low_delay = 1;
 657         break;
 658     case AV_CODEC_ID_RV10:
 659         s->out_format = FMT_H263;
 660         avctx->delay  = 0;
 661         s->low_delay  = 1;
 662         break;
 663     case AV_CODEC_ID_RV20:
 664         s->out_format      = FMT_H263;
 665         avctx->delay       = 0;
 666         s->low_delay       = 1;
 667         s->modified_quant  = 1;
 668         s->h263_aic        = 1;
 669         s->h263_plus       = 1;
 670         s->loop_filter     = 1;
 671         s->unrestricted_mv = 0;
 672         break;
 673     case AV_CODEC_ID_MPEG4:
 674         s->out_format      = FMT_H263;
 675         s->h263_pred       = 1;
 676         s->unrestricted_mv = 1;
 677         s->low_delay       = s->max_b_frames ? 0 : 1;
 678         avctx->delay       = s->low_delay ? 0 : (s->max_b_frames + 1);
 679         break;
 680     case AV_CODEC_ID_MSMPEG4V2:
 681         s->out_format      = FMT_H263;
 682         s->h263_pred       = 1;
 683         s->unrestricted_mv = 1;
 684         s->msmpeg4_version = 2;
 685         avctx->delay       = 0;
 686         s->low_delay       = 1;
 687         break;
 688     case AV_CODEC_ID_MSMPEG4V3:
 689         s->out_format        = FMT_H263;
 690         s->h263_pred         = 1;
 691         s->unrestricted_mv   = 1;
 692         s->msmpeg4_version   = 3;
 693         s->flipflop_rounding = 1;
 694         avctx->delay         = 0;
 695         s->low_delay         = 1;
 696         break;
 697     case AV_CODEC_ID_WMV1:
 698         s->out_format        = FMT_H263;
 699         s->h263_pred         = 1;
 700         s->unrestricted_mv   = 1;
 701         s->msmpeg4_version   = 4;
 702         s->flipflop_rounding = 1;
 703         avctx->delay         = 0;
 704         s->low_delay         = 1;
 705         break;
 706     case AV_CODEC_ID_WMV2:
 707         s->out_format        = FMT_H263;
 708         s->h263_pred         = 1;
 709         s->unrestricted_mv   = 1;
 710         s->msmpeg4_version   = 5;
 711         s->flipflop_rounding = 1;
 712         avctx->delay         = 0;
 713         s->low_delay         = 1;
 714         break;
 715     default:
 716         return -1;
 717     }
 718
 719 #if FF_API_PRIVATE_OPT
 720     FF_DISABLE_DEPRECATION_WARNINGS
 721     if (avctx->noise_reduction)
 722         s->noise_reduction = avctx->noise_reduction;
 723     FF_ENABLE_DEPRECATION_WARNINGS
 724 #endif
 725
 726     avctx->has_b_frames = !s->low_delay;
 727
 728     s->encoding = 1;
 729
 730     s->progressive_frame    =
 731     s->progressive_sequence = !(avctx->flags & (AV_CODEC_FLAG_INTERLACED_DCT |
 732                                                 AV_CODEC_FLAG_INTERLACED_ME) ||
 733                                 s->alternate_scan);
 734
 735     /* init */
 736     ff_mpv_idct_init(s);
 737     if (ff_mpv_common_init(s) < 0)
 738         return -1;
 739
 740     if (ARCH_X86)
 741         ff_mpv_encode_init_x86(s);
 742
 743     ff_fdctdsp_init(&s->fdsp, avctx);
 744     ff_me_cmp_init(&s->mecc, avctx);
 745     ff_mpegvideoencdsp_init(&s->mpvencdsp, avctx);
 746     ff_pixblockdsp_init(&s->pdsp, avctx);
 747     ff_qpeldsp_init(&s->qdsp);
 748
 749     if (s->msmpeg4_version) {
 750         FF_ALLOCZ_OR_GOTO(s->avctx, s->ac_stats,
 751                           2 * 2 * (MAX_LEVEL + 1) *
 752                           (MAX_RUN + 1) * 2 * sizeof(int), fail);
 753     }
 754     FF_ALLOCZ_OR_GOTO(s->avctx, s->avctx->stats_out, 256, fail);
 755
 756     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_intra_matrix,   64 * 32 * sizeof(int), fail);
 757     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_inter_matrix,   64 * 32 * sizeof(int), fail);
 758     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_intra_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
 759     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_inter_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
 760     FF_ALLOCZ_OR_GOTO(s->avctx, s->input_picture,
 761                       MAX_PICTURE_COUNT * sizeof(Picture *), fail);
 762     FF_ALLOCZ_OR_GOTO(s->avctx, s->reordered_input_picture,
 763                       MAX_PICTURE_COUNT * sizeof(Picture *), fail);
 764
 765
 766     if (s->noise_reduction) {
 767         FF_ALLOCZ_OR_GOTO(s->avctx, s->dct_offset,
 768                           2 * 64 * sizeof(uint16_t), fail);
 769     }
 770
 771     if (CONFIG_H263_ENCODER)
 772         ff_h263dsp_init(&s->h263dsp);
 773     if (!s->dct_quantize)
 774         s->dct_quantize = ff_dct_quantize_c;
 775     if (!s->denoise_dct)
 776         s->denoise_dct  = denoise_dct_c;
 777     s->fast_dct_quantize = s->dct_quantize;
 778     if (avctx->trellis)
 779         s->dct_quantize  = dct_quantize_trellis_c;
 780
 781     if ((CONFIG_H263P_ENCODER || CONFIG_RV20_ENCODER) && s->modified_quant)
 782         s->chroma_qscale_table = ff_h263_chroma_qscale_table;
 783
 784     if (s->slice_context_count > 1) {
 785         s->rtp_mode = 1;
 786
 787         if (avctx->codec_id == AV_CODEC_ID_H263 || avctx->codec_id == AV_CODEC_ID_H263P)
 788             s->h263_slice_structured = 1;
 789     }
 790
 791     s->quant_precision = 5;
 792
 793 #if FF_API_PRIVATE_OPT
 794 FF_DISABLE_DEPRECATION_WARNINGS
 795     if (avctx->frame_skip_threshold)
 796         s->frame_skip_threshold = avctx->frame_skip_threshold;
 797     if (avctx->frame_skip_factor)
 798         s->frame_skip_factor = avctx->frame_skip_factor;
 799     if (avctx->frame_skip_exp)
 800         s->frame_skip_exp = avctx->frame_skip_exp;
 801     if (avctx->frame_skip_cmp != FF_CMP_DCTMAX)
 802         s->frame_skip_cmp = avctx->frame_skip_cmp;
 803 FF_ENABLE_DEPRECATION_WARNINGS
 804 #endif
 805
 806     ff_set_cmp(&s->mecc, s->mecc.ildct_cmp,      s->avctx->ildct_cmp);
 807     ff_set_cmp(&s->mecc, s->mecc.frame_skip_cmp, s->frame_skip_cmp);
 808
 809     if (CONFIG_H261_ENCODER && s->out_format == FMT_H261)
 810         ff_h261_encode_init(s);
 811     if (CONFIG_H263_ENCODER && s->out_format == FMT_H263)
 812         ff_h263_encode_init(s);
 813     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
 814         if ((ret = ff_msmpeg4_encode_init(s)) < 0)
 815             return ret;
 816     if ((CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
 817         && s->out_format == FMT_MPEG1)
 818         ff_mpeg1_encode_init(s);
 819
 820     /* init q matrix */
 821     for (i = 0; i < 64; i++) {
 822         int j = s->idsp.idct_permutation[i];
 823         if (CONFIG_MPEG4_ENCODER && s->codec_id == AV_CODEC_ID_MPEG4 &&
 824             s->mpeg_quant) {
 825             s->intra_matrix[j] = ff_mpeg4_default_intra_matrix[i];
 826             s->inter_matrix[j] = ff_mpeg4_default_non_intra_matrix[i];
 827         } else if (s->out_format == FMT_H263 || s->out_format == FMT_H261) {
 828             s->intra_matrix[j] =
 829             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
 830         } else {
 831             /* MPEG-1/2 */
 832             s->intra_matrix[j] = ff_mpeg1_default_intra_matrix[i];
 833             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
 834         }
 835         if (s->avctx->intra_matrix)
 836             s->intra_matrix[j] = s->avctx->intra_matrix[i];
 837         if (s->avctx->inter_matrix)
 838             s->inter_matrix[j] = s->avctx->inter_matrix[i];
 839     }
 840
 841     /* precompute matrix */
 842     /* for mjpeg, we do include qscale in the matrix */
 843     if (s->out_format != FMT_MJPEG) {
 844         ff_convert_matrix(s, s->q_intra_matrix, s->q_intra_matrix16,
 845                           s->intra_matrix, s->intra_quant_bias, avctx->qmin,
 846                           31, 1);
 847         ff_convert_matrix(s, s->q_inter_matrix, s->q_inter_matrix16,
 848                           s->inter_matrix, s->inter_quant_bias, avctx->qmin,
 849                           31, 0);
 850     }
 851
 852     if (ff_rate_control_init(s) < 0)
 853         return -1;
 854
 855 #if FF_API_PRIVATE_OPT
 856     FF_DISABLE_DEPRECATION_WARNINGS
 857     if (avctx->brd_scale)
 858         s->brd_scale = avctx->brd_scale;
 859
 860     if (avctx->prediction_method)
 861         s->pred = avctx->prediction_method + 1;
 862     FF_ENABLE_DEPRECATION_WARNINGS
 863 #endif
 864
 865     if (s->b_frame_strategy == 2) {
 866         for (i = 0; i < s->max_b_frames + 2; i++) {
 867             s->tmp_frames[i] = av_frame_alloc();
 868             if (!s->tmp_frames[i])
 869                 return AVERROR(ENOMEM);
 870
 871             s->tmp_frames[i]->format = AV_PIX_FMT_YUV420P;
 872             s->tmp_frames[i]->width  = s->width  >> s->brd_scale;
 873             s->tmp_frames[i]->height = s->height >> s->brd_scale;
 874
 875             ret = av_frame_get_buffer(s->tmp_frames[i], 32);
 876             if (ret < 0)
 877                 return ret;
 878         }
 879     }
 880
 881     cpb_props = ff_add_cpb_side_data(avctx);
 882     if (!cpb_props)
 883         return AVERROR(ENOMEM);
 884     cpb_props->max_bitrate = avctx->rc_max_rate;
 885     cpb_props->min_bitrate = avctx->rc_min_rate;
 886     cpb_props->avg_bitrate = avctx->bit_rate;
 887     cpb_props->buffer_size = avctx->rc_buffer_size;
 888
 889     return 0;
 890 fail:
 891     ff_mpv_encode_end(avctx);
 892     return AVERROR_UNKNOWN;
 893 }
 894
 895 av_cold int ff_mpv_encode_end(AVCodecContext *avctx)
 896 {
 897     MpegEncContext *s = avctx->priv_data;
 898     int i;
 899
 900     ff_rate_control_uninit(s);
 901     ff_mpv_common_end(s);
 902     if (CONFIG_MJPEG_ENCODER &&
 903         s->out_format == FMT_MJPEG)
 904         ff_mjpeg_encode_close(s);
 905
 906     av_freep(&avctx->extradata);
 907
 908     for (i = 0; i < FF_ARRAY_ELEMS(s->tmp_frames); i++)
 909         av_frame_free(&s->tmp_frames[i]);
 910
 911     ff_free_picture_tables(&s->new_picture);
 912     ff_mpeg_unref_picture(s->avctx, &s->new_picture);
 913
 914     av_freep(&s->avctx->stats_out);
 915     av_freep(&s->ac_stats);
 916
 917     av_freep(&s->q_intra_matrix);
 918     av_freep(&s->q_inter_matrix);
 919     av_freep(&s->q_intra_matrix16);
 920     av_freep(&s->q_inter_matrix16);
 921     av_freep(&s->input_picture);
 922     av_freep(&s->reordered_input_picture);
 923     av_freep(&s->dct_offset);
 924
 925     return 0;
 926 }
 927
 928 static int get_sae(uint8_t *src, int ref, int stride)
 929 {
 930     int x,y;
 931     int acc = 0;
 932
 933     for (y = 0; y < 16; y++) {
 934         for (x = 0; x < 16; x++) {
 935             acc += FFABS(src[x + y * stride] - ref);
 936         }
 937     }
 938
 939     return acc;
 940 }
 941
 942 static int get_intra_count(MpegEncContext *s, uint8_t *src,
 943                            uint8_t *ref, int stride)
 944 {
 945     int x, y, w, h;
 946     int acc = 0;
 947
 948     w = s->width  & ~15;
 949     h = s->height & ~15;
 950
 951     for (y = 0; y < h; y += 16) {
 952         for (x = 0; x < w; x += 16) {
 953             int offset = x + y * stride;
 954             int sad  = s->mecc.sad[0](NULL, src + offset, ref + offset,
 955                                       stride, 16);
 956             int mean = (s->mpvencdsp.pix_sum(src + offset, stride) + 128) >> 8;
 957             int sae  = get_sae(src + offset, mean, stride);
 958
 959             acc += sae + 500 < sad;
 960         }
 961     }
 962     return acc;
 963 }
 964
 965 static int alloc_picture(MpegEncContext *s, Picture *pic, int shared)
 966 {
 967     return ff_alloc_picture(s->avctx, pic, &s->me, &s->sc, shared, 1,
 968                             s->chroma_x_shift, s->chroma_y_shift, s->out_format,
 969                             s->mb_stride, s->mb_height, s->b8_stride,
 970                             &s->linesize, &s->uvlinesize);
 971 }
 972
 973 static int load_input_picture(MpegEncContext *s, const AVFrame *pic_arg)
 974 {
 975     Picture *pic = NULL;
 976     int64_t pts;
 977     int i, display_picture_number = 0, ret;
 978     int encoding_delay = s->max_b_frames ? s->max_b_frames
 979                                          : (s->low_delay ? 0 : 1);
 980     int flush_offset = 1;
 981     int direct = 1;
 982
 983     if (pic_arg) {
 984         pts = pic_arg->pts;
 985         display_picture_number = s->input_picture_number++;
 986
 987         if (pts != AV_NOPTS_VALUE) {
 988             if (s->user_specified_pts != AV_NOPTS_VALUE) {
 989                 int64_t time = pts;
 990                 int64_t last = s->user_specified_pts;
 991
 992                 if (time <= last) {
 993                     av_log(s->avctx, AV_LOG_ERROR,
 994                            "Error, Invalid timestamp=%"PRId64", "
 995                            "last=%"PRId64"\n", pts, s->user_specified_pts);
 996                     return -1;
 997                 }
 998
 999                 if (!s->low_delay && display_picture_number == 1)
1000                     s->dts_delta = time - last;
1001             }
1002             s->user_specified_pts = pts;
1003         } else {
1004             if (s->user_specified_pts != AV_NOPTS_VALUE) {
1005                 s->user_specified_pts =
1006                 pts = s->user_specified_pts + 1;
1007                 av_log(s->avctx, AV_LOG_INFO,
1008                        "Warning: AVFrame.pts=? trying to guess (%"PRId64")\n",
1009                        pts);
1010             } else {
1011                 pts = display_picture_number;
1012             }
1013         }
1014
1015         if (!pic_arg->buf[0] ||
1016             pic_arg->linesize[0] != s->linesize ||
1017             pic_arg->linesize[1] != s->uvlinesize ||
1018             pic_arg->linesize[2] != s->uvlinesize)
1019             direct = 0;
1020         if ((s->width & 15) || (s->height & 15))
1021             direct = 0;
1022
1023         ff_dlog(s->avctx, "%d %d %td %td\n", pic_arg->linesize[0],
1024                 pic_arg->linesize[1], s->linesize, s->uvlinesize);
1025
1026         i = ff_find_unused_picture(s->avctx, s->picture, direct);
1027         if (i < 0)
1028             return i;
1029
1030         pic = &s->picture[i];
1031         pic->reference = 3;
1032
1033         if (direct) {
1034             if ((ret = av_frame_ref(pic->f, pic_arg)) < 0)
1035                 return ret;
1036         }
1037         ret = alloc_picture(s, pic, direct);
1038         if (ret < 0)
1039             return ret;
1040
1041         if (!direct) {
1042             if (pic->f->data[0] + INPLACE_OFFSET == pic_arg->data[0] &&
1043                 pic->f->data[1] + INPLACE_OFFSET == pic_arg->data[1] &&
1044                 pic->f->data[2] + INPLACE_OFFSET == pic_arg->data[2]) {
1045                 // empty
1046             } else {
1047                 int h_chroma_shift, v_chroma_shift;
1048                 av_pix_fmt_get_chroma_sub_sample(s->avctx->pix_fmt,
1049                                                  &h_chroma_shift,
1050                                                  &v_chroma_shift);
1051
1052                 for (i = 0; i < 3; i++) {
1053                     int src_stride = pic_arg->linesize[i];
1054                     int dst_stride = i ? s->uvlinesize : s->linesize;
1055                     int h_shift = i ? h_chroma_shift : 0;
1056                     int v_shift = i ? v_chroma_shift : 0;
1057                     int w = s->width  >> h_shift;
1058                     int h = s->height >> v_shift;
1059                     uint8_t *src = pic_arg->data[i];
1060                     uint8_t *dst = pic->f->data[i];
1061
1062                     if (!s->avctx->rc_buffer_size)
1063                         dst += INPLACE_OFFSET;
1064
1065                     if (src_stride == dst_stride)
1066                         memcpy(dst, src, src_stride * h);
1067                     else {
1068                         int h2 = h;
1069                         uint8_t *dst2 = dst;
1070                         while (h2--) {
1071                             memcpy(dst2, src, w);
1072                             dst2 += dst_stride;
1073                             src += src_stride;
1074                         }
1075                     }
1076                     if ((s->width & 15) || (s->height & 15)) {
1077                         s->mpvencdsp.draw_edges(dst, dst_stride,
1078                                                 w, h,
1079                                                 16 >> h_shift,
1080                                                 16 >> v_shift,
1081                                                 EDGE_BOTTOM);
1082                     }
1083                 }
1084             }
1085         }
1086         ret = av_frame_copy_props(pic->f, pic_arg);
1087         if (ret < 0)
1088             return ret;
1089
1090         pic->f->display_picture_number = display_picture_number;
1091         pic->f->pts = pts; // we set this here to avoid modifying pic_arg
1092     } else {
1093         /* Flushing: When we have not received enough input frames,
1094          * ensure s->input_picture[0] contains the first picture */
1095         for (flush_offset = 0; flush_offset < encoding_delay + 1; flush_offset++)
1096             if (s->input_picture[flush_offset])
1097                 break;
1098
1099         if (flush_offset <= 1)
1100             flush_offset = 1;
1101         else
1102             encoding_delay = encoding_delay - flush_offset + 1;
1103     }
1104
1105     /* shift buffer entries */
1106     for (i = flush_offset; i < MAX_PICTURE_COUNT /*s->encoding_delay + 1*/; i++)
1107         s->input_picture[i - flush_offset] = s->input_picture[i];
1108
1109     s->input_picture[encoding_delay] = (Picture*) pic;
1110
1111     return 0;
1112 }
1113
1114 static int skip_check(MpegEncContext *s, Picture *p, Picture *ref)
1115 {
1116     int x, y, plane;
1117     int score = 0;
1118     int64_t score64 = 0;
1119
1120     for (plane = 0; plane < 3; plane++) {
1121         const int stride = p->f->linesize[plane];
1122         const int bw = plane ? 1 : 2;
1123         for (y = 0; y < s->mb_height * bw; y++) {
1124             for (x = 0; x < s->mb_width * bw; x++) {
1125                 int off = p->shared ? 0 : 16;
1126                 uint8_t *dptr = p->f->data[plane] + 8 * (x + y * stride) + off;
1127                 uint8_t *rptr = ref->f->data[plane] + 8 * (x + y * stride);
1128                 int v = s->mecc.frame_skip_cmp[1](s, dptr, rptr, stride, 8);
1129
1130                 switch (s->frame_skip_exp) {
1131                 case 0: score    =  FFMAX(score, v);          break;
1132                 case 1: score   += FFABS(v);                  break;
1133                 case 2: score   += v * v;                     break;
1134                 case 3: score64 += FFABS(v * v * (int64_t)v); break;
1135                 case 4: score64 += v * v * (int64_t)(v * v);  break;
1136                 }
1137             }
1138         }
1139     }
1140
1141     if (score)
1142         score64 = score;
1143
1144     if (score64 < s->frame_skip_threshold)
1145         return 1;
1146     if (score64 < ((s->frame_skip_factor * (int64_t) s->lambda) >> 8))
1147         return 1;
1148     return 0;
1149 }
1150
1151 static int encode_frame(AVCodecContext *c, AVFrame *frame)
1152 {
1153     AVPacket pkt = { 0 };
1154     int ret;
1155     int size = 0;
1156
1157     av_init_packet(&pkt);
1158
1159     ret = avcodec_send_frame(c, frame);
1160     if (ret < 0)
1161         return ret;
1162
1163     do {
1164         ret = avcodec_receive_packet(c, &pkt);
1165         if (ret >= 0) {
1166             size += pkt.size;
1167             av_packet_unref(&pkt);
1168         } else if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF)
1169             return ret;
1170     } while (ret >= 0);
1171
1172     return size;
1173 }
1174
1175 static int estimate_best_b_count(MpegEncContext *s)
1176 {
1177     const AVCodec *codec = avcodec_find_encoder(s->avctx->codec_id);
1178     const int scale = s->brd_scale;
1179     int width  = s->width  >> scale;
1180     int height = s->height >> scale;
1181     int i, j, out_size, p_lambda, b_lambda, lambda2;
1182     int64_t best_rd  = INT64_MAX;
1183     int best_b_count = -1;
1184     int ret = 0;
1185
1186     assert(scale >= 0 && scale <= 3);
1187
1188     //emms_c();
1189     //s->next_picture_ptr->quality;
1190     p_lambda = s->last_lambda_for[AV_PICTURE_TYPE_P];
1191     //p_lambda * FFABS(s->avctx->b_quant_factor) + s->avctx->b_quant_offset;
1192     b_lambda = s->last_lambda_for[AV_PICTURE_TYPE_B];
1193     if (!b_lambda) // FIXME we should do this somewhere else
1194         b_lambda = p_lambda;
1195     lambda2  = (b_lambda * b_lambda + (1 << FF_LAMBDA_SHIFT) / 2) >>
1196                FF_LAMBDA_SHIFT;
1197
1198     for (i = 0; i < s->max_b_frames + 2; i++) {
1199         Picture pre_input, *pre_input_ptr = i ? s->input_picture[i - 1] :
1200                                                 s->next_picture_ptr;
1201
1202         if (pre_input_ptr && (!i || s->input_picture[i - 1])) {
1203             pre_input = *pre_input_ptr;
1204
1205             if (!pre_input.shared && i) {
1206                 pre_input.f->data[0] += INPLACE_OFFSET;
1207                 pre_input.f->data[1] += INPLACE_OFFSET;
1208                 pre_input.f->data[2] += INPLACE_OFFSET;
1209             }
1210
1211             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[0],
1212                                        s->tmp_frames[i]->linesize[0],
1213                                        pre_input.f->data[0],
1214                                        pre_input.f->linesize[0],
1215                                        width, height);
1216             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[1],
1217                                        s->tmp_frames[i]->linesize[1],
1218                                        pre_input.f->data[1],
1219                                        pre_input.f->linesize[1],
1220                                        width >> 1, height >> 1);
1221             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[2],
1222                                        s->tmp_frames[i]->linesize[2],
1223                                        pre_input.f->data[2],
1224                                        pre_input.f->linesize[2],
1225                                        width >> 1, height >> 1);
1226         }
1227     }
1228
1229     for (j = 0; j < s->max_b_frames + 1; j++) {
1230         AVCodecContext *c;
1231         int64_t rd = 0;
1232
1233         if (!s->input_picture[j])
1234             break;
1235
1236         c = avcodec_alloc_context3(NULL);
1237         if (!c)
1238             return AVERROR(ENOMEM);
1239
1240         c->width        = width;
1241         c->height       = height;
1242         c->flags        = AV_CODEC_FLAG_QSCALE | AV_CODEC_FLAG_PSNR;
1243         c->flags       |= s->avctx->flags & AV_CODEC_FLAG_QPEL;
1244         c->mb_decision  = s->avctx->mb_decision;
1245         c->me_cmp       = s->avctx->me_cmp;
1246         c->mb_cmp       = s->avctx->mb_cmp;
1247         c->me_sub_cmp   = s->avctx->me_sub_cmp;
1248         c->pix_fmt      = AV_PIX_FMT_YUV420P;
1249         c->time_base    = s->avctx->time_base;
1250         c->max_b_frames = s->max_b_frames;
1251
1252         ret = avcodec_open2(c, codec, NULL);
1253         if (ret < 0)
1254             goto fail;
1255
1256         s->tmp_frames[0]->pict_type = AV_PICTURE_TYPE_I;
1257         s->tmp_frames[0]->quality   = 1 * FF_QP2LAMBDA;
1258
1259         out_size = encode_frame(c, s->tmp_frames[0]);
1260         if (out_size < 0) {
1261             ret = out_size;
1262             goto fail;
1263         }
1264
1265         //rd += (out_size * lambda2) >> FF_LAMBDA_SHIFT;
1266
1267         for (i = 0; i < s->max_b_frames + 1; i++) {
1268             int is_p = i % (j + 1) == j || i == s->max_b_frames;
1269
1270             s->tmp_frames[i + 1]->pict_type = is_p ?
1271                                      AV_PICTURE_TYPE_P : AV_PICTURE_TYPE_B;
1272             s->tmp_frames[i + 1]->quality   = is_p ? p_lambda : b_lambda;
1273
1274             out_size = encode_frame(c, s->tmp_frames[i + 1]);
1275             if (out_size < 0) {
1276                 ret = out_size;
1277                 goto fail;
1278             }
1279
1280             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1281         }
1282
1283         /* get the delayed frames */
1284         out_size = encode_frame(c, NULL);
1285         if (out_size < 0) {
1286             ret = out_size;
1287             goto fail;
1288         }
1289         rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1290
1291         rd += c->error[0] + c->error[1] + c->error[2];
1292
1293         if (rd < best_rd) {
1294             best_rd = rd;
1295             best_b_count = j;
1296         }
1297
1298 fail:
1299         avcodec_free_context(&c);
1300         if (ret < 0)
1301             return ret;
1302     }
1303
1304     return best_b_count;
1305 }
1306
1307 static int select_input_picture(MpegEncContext *s)
1308 {
1309     int i, ret;
1310
1311     for (i = 1; i < MAX_PICTURE_COUNT; i++)
1312         s->reordered_input_picture[i - 1] = s->reordered_input_picture[i];
1313     s->reordered_input_picture[MAX_PICTURE_COUNT - 1] = NULL;
1314
1315     /* set next picture type & ordering */
1316     if (!s->reordered_input_picture[0] && s->input_picture[0]) {
1317         if (/*s->picture_in_gop_number >= s->gop_size ||*/
1318             !s->next_picture_ptr || s->intra_only) {
1319             s->reordered_input_picture[0] = s->input_picture[0];
1320             s->reordered_input_picture[0]->f->pict_type = AV_PICTURE_TYPE_I;
1321             s->reordered_input_picture[0]->f->coded_picture_number =
1322                 s->coded_picture_number++;
1323         } else {
1324             int b_frames = 0;
1325
1326             if (s->frame_skip_threshold || s->frame_skip_factor) {
1327                 if (s->picture_in_gop_number < s->gop_size &&
1328                     skip_check(s, s->input_picture[0], s->next_picture_ptr)) {
1329                     // FIXME check that the gop check above is +-1 correct
1330                     av_frame_unref(s->input_picture[0]->f);
1331
1332                     emms_c();
1333                     ff_vbv_update(s, 0);
1334
1335                     goto no_output_pic;
1336                 }
1337             }
1338
1339             if (s->avctx->flags & AV_CODEC_FLAG_PASS2) {
1340                 for (i = 0; i < s->max_b_frames + 1; i++) {
1341                     int pict_num = s->input_picture[0]->f->display_picture_number + i;
1342
1343                     if (pict_num >= s->rc_context.num_entries)
1344                         break;
1345                     if (!s->input_picture[i]) {
1346                         s->rc_context.entry[pict_num - 1].new_pict_type = AV_PICTURE_TYPE_P;
1347                         break;
1348                     }
1349
1350                     s->input_picture[i]->f->pict_type =
1351                         s->rc_context.entry[pict_num].new_pict_type;
1352                 }
1353             }
1354
1355             if (s->b_frame_strategy == 0) {
1356                 b_frames = s->max_b_frames;
1357                 while (b_frames && !s->input_picture[b_frames])
1358                     b_frames--;
1359             } else if (s->b_frame_strategy == 1) {
1360                 for (i = 1; i < s->max_b_frames + 1; i++) {
1361                     if (s->input_picture[i] &&
1362                         s->input_picture[i]->b_frame_score == 0) {
1363                         s->input_picture[i]->b_frame_score =
1364                             get_intra_count(s,
1365                                             s->input_picture[i    ]->f->data[0],
1366                                             s->input_picture[i - 1]->f->data[0],
1367                                             s->linesize) + 1;
1368                     }
1369                 }
1370                 for (i = 0; i < s->max_b_frames + 1; i++) {
1371                     if (!s->input_picture[i] ||
1372                         s->input_picture[i]->b_frame_score - 1 >
1373                             s->mb_num / s->b_sensitivity)
1374                         break;
1375                 }
1376
1377                 b_frames = FFMAX(0, i - 1);
1378
1379                 /* reset scores */
1380                 for (i = 0; i < b_frames + 1; i++) {
1381                     s->input_picture[i]->b_frame_score = 0;
1382                 }
1383             } else if (s->b_frame_strategy == 2) {
1384                 b_frames = estimate_best_b_count(s);
1385                 if (b_frames < 0)
1386                     return b_frames;
1387             }
1388
1389             emms_c();
1390
1391             for (i = b_frames - 1; i >= 0; i--) {
1392                 int type = s->input_picture[i]->f->pict_type;
1393                 if (type && type != AV_PICTURE_TYPE_B)
1394                     b_frames = i;
1395             }
1396             if (s->input_picture[b_frames]->f->pict_type == AV_PICTURE_TYPE_B &&
1397                 b_frames == s->max_b_frames) {
1398                 av_log(s->avctx, AV_LOG_ERROR,
1399                        "warning, too many B-frames in a row\n");
1400             }
1401
1402             if (s->picture_in_gop_number + b_frames >= s->gop_size) {
1403                 if ((s->mpv_flags & FF_MPV_FLAG_STRICT_GOP) &&
1404                     s->gop_size > s->picture_in_gop_number) {
1405                     b_frames = s->gop_size - s->picture_in_gop_number - 1;
1406                 } else {
1407                     if (s->avctx->flags & AV_CODEC_FLAG_CLOSED_GOP)
1408                         b_frames = 0;
1409                     s->input_picture[b_frames]->f->pict_type = AV_PICTURE_TYPE_I;
1410                 }
1411             }
1412
1413             if ((s->avctx->flags & AV_CODEC_FLAG_CLOSED_GOP) && b_frames &&
1414                 s->input_picture[b_frames]->f->pict_type == AV_PICTURE_TYPE_I)
1415                 b_frames--;
1416
1417             s->reordered_input_picture[0] = s->input_picture[b_frames];
1418             if (s->reordered_input_picture[0]->f->pict_type != AV_PICTURE_TYPE_I)
1419                 s->reordered_input_picture[0]->f->pict_type = AV_PICTURE_TYPE_P;
1420             s->reordered_input_picture[0]->f->coded_picture_number =
1421                 s->coded_picture_number++;
1422             for (i = 0; i < b_frames; i++) {
1423                 s->reordered_input_picture[i + 1] = s->input_picture[i];
1424                 s->reordered_input_picture[i + 1]->f->pict_type =
1425                     AV_PICTURE_TYPE_B;
1426                 s->reordered_input_picture[i + 1]->f->coded_picture_number =
1427                     s->coded_picture_number++;
1428             }
1429         }
1430     }
1431 no_output_pic:
1432     ff_mpeg_unref_picture(s->avctx, &s->new_picture);
1433
1434     if (s->reordered_input_picture[0]) {
1435         s->reordered_input_picture[0]->reference =
1436            s->reordered_input_picture[0]->f->pict_type !=
1437                AV_PICTURE_TYPE_B ? 3 : 0;
1438
1439         if ((ret = ff_mpeg_ref_picture(s->avctx, &s->new_picture, s->reordered_input_picture[0])))
1440             return ret;
1441
1442         if (s->reordered_input_picture[0]->shared || s->avctx->rc_buffer_size) {
1443             // input is a shared pix, so we can't modify it -> allocate a new
1444             // one & ensure that the shared one is reuseable
1445
1446             Picture *pic;
1447             int i = ff_find_unused_picture(s->avctx, s->picture, 0);
1448             if (i < 0)
1449                 return i;
1450             pic = &s->picture[i];
1451
1452             pic->reference = s->reordered_input_picture[0]->reference;
1453             if (alloc_picture(s, pic, 0) < 0) {
1454                 return -1;
1455             }
1456
1457             ret = av_frame_copy_props(pic->f, s->reordered_input_picture[0]->f);
1458             if (ret < 0)
1459                 return ret;
1460
1461             /* mark us unused / free shared pic */
1462             av_frame_unref(s->reordered_input_picture[0]->f);
1463             s->reordered_input_picture[0]->shared = 0;
1464
1465             s->current_picture_ptr = pic;
1466         } else {
1467             // input is not a shared pix -> reuse buffer for current_pix
1468             s->current_picture_ptr = s->reordered_input_picture[0];
1469             for (i = 0; i < 4; i++) {
1470                 s->new_picture.f->data[i] += INPLACE_OFFSET;
1471             }
1472         }
1473         ff_mpeg_unref_picture(s->avctx, &s->current_picture);
1474         if ((ret = ff_mpeg_ref_picture(s->avctx, &s->current_picture,
1475                                        s->current_picture_ptr)) < 0)
1476             return ret;
1477
1478         s->picture_number = s->new_picture.f->display_picture_number;
1479     }
1480     return 0;
1481 }
1482
1483 static void frame_end(MpegEncContext *s)
1484 {
1485     int i;
1486
1487     if (s->unrestricted_mv &&
1488         s->current_picture.reference &&
1489         !s->intra_only) {
1490         const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(s->avctx->pix_fmt);
1491         int hshift = desc->log2_chroma_w;
1492         int vshift = desc->log2_chroma_h;
1493         s->mpvencdsp.draw_edges(s->current_picture.f->data[0], s->linesize,
1494                                 s->h_edge_pos, s->v_edge_pos,
1495                                 EDGE_WIDTH, EDGE_WIDTH,
1496                                 EDGE_TOP | EDGE_BOTTOM);
1497         s->mpvencdsp.draw_edges(s->current_picture.f->data[1], s->uvlinesize,
1498                                 s->h_edge_pos >> hshift,
1499                                 s->v_edge_pos >> vshift,
1500                                 EDGE_WIDTH >> hshift,
1501                                 EDGE_WIDTH >> vshift,
1502                                 EDGE_TOP | EDGE_BOTTOM);
1503         s->mpvencdsp.draw_edges(s->current_picture.f->data[2], s->uvlinesize,
1504                                 s->h_edge_pos >> hshift,
1505                                 s->v_edge_pos >> vshift,
1506                                 EDGE_WIDTH >> hshift,
1507                                 EDGE_WIDTH >> vshift,
1508                                 EDGE_TOP | EDGE_BOTTOM);
1509     }
1510
1511     emms_c();
1512
1513     s->last_pict_type                 = s->pict_type;
1514     s->last_lambda_for [s->pict_type] = s->current_picture_ptr->f->quality;
1515     if (s->pict_type!= AV_PICTURE_TYPE_B)
1516         s->last_non_b_pict_type = s->pict_type;
1517
1518     if (s->encoding) {
1519         /* release non-reference frames */
1520         for (i = 0; i < MAX_PICTURE_COUNT; i++) {
1521             if (!s->picture[i].reference)
1522                 ff_mpeg_unref_picture(s->avctx, &s->picture[i]);
1523         }
1524     }
1525
1526 #if FF_API_CODED_FRAME
1527 FF_DISABLE_DEPRECATION_WARNINGS
1528     av_frame_copy_props(s->avctx->coded_frame, s->current_picture.f);
1529 FF_ENABLE_DEPRECATION_WARNINGS
1530 #endif
1531 #if FF_API_ERROR_FRAME
1532 FF_DISABLE_DEPRECATION_WARNINGS
1533     memcpy(s->current_picture.f->error, s->current_picture.encoding_error,
1534            sizeof(s->current_picture.encoding_error));
1535 FF_ENABLE_DEPRECATION_WARNINGS
1536 #endif
1537 }
1538
1539 static void update_noise_reduction(MpegEncContext *s)
1540 {
1541     int intra, i;
1542
1543     for (intra = 0; intra < 2; intra++) {
1544         if (s->dct_count[intra] > (1 << 16)) {
1545             for (i = 0; i < 64; i++) {
1546                 s->dct_error_sum[intra][i] >>= 1;
1547             }
1548             s->dct_count[intra] >>= 1;
1549         }
1550
1551         for (i = 0; i < 64; i++) {
1552             s->dct_offset[intra][i] = (s->noise_reduction *
1553                                        s->dct_count[intra] +
1554                                        s->dct_error_sum[intra][i] / 2) /
1555                                       (s->dct_error_sum[intra][i] + 1);
1556         }
1557     }
1558 }
1559
1560 static int frame_start(MpegEncContext *s)
1561 {
1562     int ret;
1563
1564     /* mark & release old frames */
1565     if (s->pict_type != AV_PICTURE_TYPE_B && s->last_picture_ptr &&
1566         s->last_picture_ptr != s->next_picture_ptr &&
1567         s->last_picture_ptr->f->buf[0]) {
1568         ff_mpeg_unref_picture(s->avctx, s->last_picture_ptr);
1569     }
1570
1571     s->current_picture_ptr->f->pict_type = s->pict_type;
1572     s->current_picture_ptr->f->key_frame = s->pict_type == AV_PICTURE_TYPE_I;
1573
1574     ff_mpeg_unref_picture(s->avctx, &s->current_picture);
1575     if ((ret = ff_mpeg_ref_picture(s->avctx, &s->current_picture,
1576                                    s->current_picture_ptr)) < 0)
1577         return ret;
1578
1579     if (s->pict_type != AV_PICTURE_TYPE_B) {
1580         s->last_picture_ptr = s->next_picture_ptr;
1581         if (!s->droppable)
1582             s->next_picture_ptr = s->current_picture_ptr;
1583     }
1584
1585     if (s->last_picture_ptr) {
1586         ff_mpeg_unref_picture(s->avctx, &s->last_picture);
1587         if (s->last_picture_ptr->f->buf[0] &&
1588             (ret = ff_mpeg_ref_picture(s->avctx, &s->last_picture,
1589                                        s->last_picture_ptr)) < 0)
1590             return ret;
1591     }
1592     if (s->next_picture_ptr) {
1593         ff_mpeg_unref_picture(s->avctx, &s->next_picture);
1594         if (s->next_picture_ptr->f->buf[0] &&
1595             (ret = ff_mpeg_ref_picture(s->avctx, &s->next_picture,
1596                                        s->next_picture_ptr)) < 0)
1597             return ret;
1598     }
1599
1600     if (s->picture_structure!= PICT_FRAME) {
1601         int i;
1602         for (i = 0; i < 4; i++) {
1603             if (s->picture_structure == PICT_BOTTOM_FIELD) {
1604                 s->current_picture.f->data[i] +=
1605                     s->current_picture.f->linesize[i];
1606             }
1607             s->current_picture.f->linesize[i] *= 2;
1608             s->last_picture.f->linesize[i]    *= 2;
1609             s->next_picture.f->linesize[i]    *= 2;
1610         }
1611     }
1612
1613     if (s->mpeg_quant || s->codec_id == AV_CODEC_ID_MPEG2VIDEO) {
1614         s->dct_unquantize_intra = s->dct_unquantize_mpeg2_intra;
1615         s->dct_unquantize_inter = s->dct_unquantize_mpeg2_inter;
1616     } else if (s->out_format == FMT_H263 || s->out_format == FMT_H261) {
1617         s->dct_unquantize_intra = s->dct_unquantize_h263_intra;
1618         s->dct_unquantize_inter = s->dct_unquantize_h263_inter;
1619     } else {
1620         s->dct_unquantize_intra = s->dct_unquantize_mpeg1_intra;
1621         s->dct_unquantize_inter = s->dct_unquantize_mpeg1_inter;
1622     }
1623
1624     if (s->dct_error_sum) {
1625         assert(s->noise_reduction && s->encoding);
1626         update_noise_reduction(s);
1627     }
1628
1629     return 0;
1630 }
1631
1632 static void write_pass1_stats(MpegEncContext *s)
1633 {
1634     snprintf(s->avctx->stats_out, 256,
1635              "in:%d out:%d type:%d q:%d itex:%d ptex:%d mv:%d misc:%d "
1636              "fcode:%d bcode:%d mc-var:%d var:%d icount:%d skipcount:%d "
1637              "hbits:%d;\n",
1638              s->current_picture_ptr->f->display_picture_number,
1639              s->current_picture_ptr->f->coded_picture_number,
1640              s->pict_type,
1641              s->current_picture.f->quality,
1642              s->i_tex_bits,
1643              s->p_tex_bits,
1644              s->mv_bits,
1645              s->misc_bits,
1646              s->f_code,
1647              s->b_code,
1648              s->current_picture.mc_mb_var_sum,
1649              s->current_picture.mb_var_sum,
1650              s->i_count, s->skip_count,
1651              s->header_bits);
1652 }
1653
1654 int ff_mpv_encode_picture(AVCodecContext *avctx, AVPacket *pkt,
1655                           const AVFrame *pic_arg, int *got_packet)
1656 {
1657     MpegEncContext *s = avctx->priv_data;
1658     int i, stuffing_count, ret;
1659     int context_count = s->slice_context_count;
1660
1661     s->picture_in_gop_number++;
1662
1663     if (load_input_picture(s, pic_arg) < 0)
1664         return -1;
1665
1666     if (select_input_picture(s) < 0) {
1667         return -1;
1668     }
1669
1670     /* output? */
1671     if (s->new_picture.f->data[0]) {
1672         uint8_t *sd;
1673         if (!pkt->data &&
1674             (ret = ff_alloc_packet(pkt, s->mb_width*s->mb_height*MAX_MB_BYTES)) < 0)
1675             return ret;
1676         if (s->mb_info) {
1677             s->mb_info_ptr = av_packet_new_side_data(pkt,
1678                                  AV_PKT_DATA_H263_MB_INFO,
1679                                  s->mb_width*s->mb_height*12);
1680             s->prev_mb_info = s->last_mb_info = s->mb_info_size = 0;
1681         }
1682
1683         for (i = 0; i < context_count; i++) {
1684             int start_y = s->thread_context[i]->start_mb_y;
1685             int   end_y = s->thread_context[i]->  end_mb_y;
1686             int h       = s->mb_height;
1687             uint8_t *start = pkt->data + (size_t)(((int64_t) pkt->size) * start_y / h);
1688             uint8_t *end   = pkt->data + (size_t)(((int64_t) pkt->size) *   end_y / h);
1689
1690             init_put_bits(&s->thread_context[i]->pb, start, end - start);
1691         }
1692
1693         s->pict_type = s->new_picture.f->pict_type;
1694         //emms_c();
1695         ret = frame_start(s);
1696         if (ret < 0)
1697             return ret;
1698 vbv_retry:
1699         if (encode_picture(s, s->picture_number) < 0)
1700             return -1;
1701
1702 #if FF_API_STAT_BITS
1703 FF_DISABLE_DEPRECATION_WARNINGS
1704         avctx->header_bits = s->header_bits;
1705         avctx->mv_bits     = s->mv_bits;
1706         avctx->misc_bits   = s->misc_bits;
1707         avctx->i_tex_bits  = s->i_tex_bits;
1708         avctx->p_tex_bits  = s->p_tex_bits;
1709         avctx->i_count     = s->i_count;
1710         // FIXME f/b_count in avctx
1711         avctx->p_count     = s->mb_num - s->i_count - s->skip_count;
1712         avctx->skip_count  = s->skip_count;
1713 FF_ENABLE_DEPRECATION_WARNINGS
1714 #endif
1715
1716         frame_end(s);
1717
1718         sd = av_packet_new_side_data(pkt, AV_PKT_DATA_QUALITY_FACTOR,
1719                                      sizeof(int));
1720         if (!sd)
1721             return AVERROR(ENOMEM);
1722         *(int *)sd = s->current_picture.f->quality;
1723
1724         if (CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG)
1725             ff_mjpeg_encode_picture_trailer(&s->pb, s->header_bits);
1726
1727         if (avctx->rc_buffer_size) {
1728             RateControlContext *rcc = &s->rc_context;
1729             int max_size = rcc->buffer_index * avctx->rc_max_available_vbv_use;
1730
1731             if (put_bits_count(&s->pb) > max_size &&
1732                 s->lambda < s->lmax) {
1733                 s->next_lambda = FFMAX(s->lambda + 1, s->lambda *
1734                                        (s->qscale + 1) / s->qscale);
1735                 if (s->adaptive_quant) {
1736                     int i;
1737                     for (i = 0; i < s->mb_height * s->mb_stride; i++)
1738                         s->lambda_table[i] =
1739                             FFMAX(s->lambda_table[i] + 1,
1740                                   s->lambda_table[i] * (s->qscale + 1) /
1741                                   s->qscale);
1742                 }
1743                 s->mb_skipped = 0;        // done in frame_start()
1744                 // done in encode_picture() so we must undo it
1745                 if (s->pict_type == AV_PICTURE_TYPE_P) {
1746                     if (s->flipflop_rounding          ||
1747                         s->codec_id == AV_CODEC_ID_H263P ||
1748                         s->codec_id == AV_CODEC_ID_MPEG4)
1749                         s->no_rounding ^= 1;
1750                 }
1751                 if (s->pict_type != AV_PICTURE_TYPE_B) {
1752                     s->time_base       = s->last_time_base;
1753                     s->last_non_b_time = s->time - s->pp_time;
1754                 }
1755                 for (i = 0; i < context_count; i++) {
1756                     PutBitContext *pb = &s->thread_context[i]->pb;
1757                     init_put_bits(pb, pb->buf, pb->buf_end - pb->buf);
1758                 }
1759                 goto vbv_retry;
1760             }
1761
1762             assert(s->avctx->rc_max_rate);
1763         }
1764
1765         if (s->avctx->flags & AV_CODEC_FLAG_PASS1)
1766             write_pass1_stats(s);
1767
1768         for (i = 0; i < 4; i++) {
1769             s->current_picture_ptr->encoding_error[i] = s->current_picture.encoding_error[i];
1770             avctx->error[i] += s->current_picture_ptr->encoding_error[i];
1771         }
1772
1773         if (s->avctx->flags & AV_CODEC_FLAG_PASS1)
1774             assert(put_bits_count(&s->pb) == s->header_bits + s->mv_bits +
1775                                              s->misc_bits + s->i_tex_bits +
1776                                              s->p_tex_bits);
1777         flush_put_bits(&s->pb);
1778         s->frame_bits  = put_bits_count(&s->pb);
1779
1780         stuffing_count = ff_vbv_update(s, s->frame_bits);
1781         if (stuffing_count) {
1782             if (s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb) >> 3) <
1783                     stuffing_count + 50) {
1784                 av_log(s->avctx, AV_LOG_ERROR, "stuffing too large\n");
1785                 return -1;
1786             }
1787
1788             switch (s->codec_id) {
1789             case AV_CODEC_ID_MPEG1VIDEO:
1790             case AV_CODEC_ID_MPEG2VIDEO:
1791                 while (stuffing_count--) {
1792                     put_bits(&s->pb, 8, 0);
1793                 }
1794             break;
1795             case AV_CODEC_ID_MPEG4:
1796                 put_bits(&s->pb, 16, 0);
1797                 put_bits(&s->pb, 16, 0x1C3);
1798                 stuffing_count -= 4;
1799                 while (stuffing_count--) {
1800                     put_bits(&s->pb, 8, 0xFF);
1801                 }
1802             break;
1803             default:
1804                 av_log(s->avctx, AV_LOG_ERROR, "vbv buffer overflow\n");
1805             }
1806             flush_put_bits(&s->pb);
1807             s->frame_bits  = put_bits_count(&s->pb);
1808         }
1809
1810         /* update MPEG-1/2 vbv_delay for CBR */
1811         if (s->avctx->rc_max_rate                          &&
1812             s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
1813             s->out_format == FMT_MPEG1                     &&
1814             90000LL * (avctx->rc_buffer_size - 1) <=
1815                 s->avctx->rc_max_rate * 0xFFFFLL) {
1816             AVCPBProperties *props;
1817             size_t props_size;
1818
1819             int vbv_delay, min_delay;
1820             double inbits  = s->avctx->rc_max_rate *
1821                              av_q2d(s->avctx->time_base);
1822             int    minbits = s->frame_bits - 8 *
1823                              (s->vbv_delay_ptr - s->pb.buf - 1);
1824             double bits    = s->rc_context.buffer_index + minbits - inbits;
1825
1826             if (bits < 0)
1827                 av_log(s->avctx, AV_LOG_ERROR,
1828                        "Internal error, negative bits\n");
1829
1830             assert(s->repeat_first_field == 0);
1831
1832             vbv_delay = bits * 90000 / s->avctx->rc_max_rate;
1833             min_delay = (minbits * 90000LL + s->avctx->rc_max_rate - 1) /
1834                         s->avctx->rc_max_rate;
1835
1836             vbv_delay = FFMAX(vbv_delay, min_delay);
1837
1838             assert(vbv_delay < 0xFFFF);
1839
1840             s->vbv_delay_ptr[0] &= 0xF8;
1841             s->vbv_delay_ptr[0] |= vbv_delay >> 13;
1842             s->vbv_delay_ptr[1]  = vbv_delay >> 5;
1843             s->vbv_delay_ptr[2] &= 0x07;
1844             s->vbv_delay_ptr[2] |= vbv_delay << 3;
1845
1846             props = av_cpb_properties_alloc(&props_size);
1847             if (!props)
1848                 return AVERROR(ENOMEM);
1849             props->vbv_delay = vbv_delay * 300;
1850
1851             ret = av_packet_add_side_data(pkt, AV_PKT_DATA_CPB_PROPERTIES,
1852                                           (uint8_t*)props, props_size);
1853             if (ret < 0) {
1854                 av_freep(&props);
1855                 return ret;
1856             }
1857
1858 #if FF_API_VBV_DELAY
1859 FF_DISABLE_DEPRECATION_WARNINGS
1860             avctx->vbv_delay     = vbv_delay * 300;
1861 FF_ENABLE_DEPRECATION_WARNINGS
1862 #endif
1863         }
1864         s->total_bits     += s->frame_bits;
1865 #if FF_API_STAT_BITS
1866 FF_DISABLE_DEPRECATION_WARNINGS
1867         avctx->frame_bits  = s->frame_bits;
1868 FF_ENABLE_DEPRECATION_WARNINGS
1869 #endif
1870
1871
1872         pkt->pts = s->current_picture.f->pts;
1873         if (!s->low_delay && s->pict_type != AV_PICTURE_TYPE_B) {
1874             if (!s->current_picture.f->coded_picture_number)
1875                 pkt->dts = pkt->pts - s->dts_delta;
1876             else
1877                 pkt->dts = s->reordered_pts;
1878             s->reordered_pts = pkt->pts;
1879         } else
1880             pkt->dts = pkt->pts;
1881         if (s->current_picture.f->key_frame)
1882             pkt->flags |= AV_PKT_FLAG_KEY;
1883         if (s->mb_info)
1884             av_packet_shrink_side_data(pkt, AV_PKT_DATA_H263_MB_INFO, s->mb_info_size);
1885     } else {
1886         s->frame_bits = 0;
1887     }
1888     assert((s->frame_bits & 7) == 0);
1889
1890     pkt->size = s->frame_bits / 8;
1891     *got_packet = !!pkt->size;
1892     return 0;
1893 }
1894
1895 static inline void dct_single_coeff_elimination(MpegEncContext *s,
1896                                                 int n, int threshold)
1897 {
1898     static const char tab[64] = {
1899         3, 2, 2, 1, 1, 1, 1, 1,
1900         1, 1, 1, 1, 1, 1, 1, 1,
1901         1, 1, 1, 1, 1, 1, 1, 1,
1902         0, 0, 0, 0, 0, 0, 0, 0,
1903         0, 0, 0, 0, 0, 0, 0, 0,
1904         0, 0, 0, 0, 0, 0, 0, 0,
1905         0, 0, 0, 0, 0, 0, 0, 0,
1906         0, 0, 0, 0, 0, 0, 0, 0
1907     };
1908     int score = 0;
1909     int run = 0;
1910     int i;
1911     int16_t *block = s->block[n];
1912     const int last_index = s->block_last_index[n];
1913     int skip_dc;
1914
1915     if (threshold < 0) {
1916         skip_dc = 0;
1917         threshold = -threshold;
1918     } else
1919         skip_dc = 1;
1920
1921     /* Are all we could set to zero already zero? */
1922     if (last_index <= skip_dc - 1)
1923         return;
1924
1925     for (i = 0; i <= last_index; i++) {
1926         const int j = s->intra_scantable.permutated[i];
1927         const int level = FFABS(block[j]);
1928         if (level == 1) {
1929             if (skip_dc && i == 0)
1930                 continue;
1931             score += tab[run];
1932             run = 0;
1933         } else if (level > 1) {
1934             return;
1935         } else {
1936             run++;
1937         }
1938     }
1939     if (score >= threshold)
1940         return;
1941     for (i = skip_dc; i <= last_index; i++) {
1942         const int j = s->intra_scantable.permutated[i];
1943         block[j] = 0;
1944     }
1945     if (block[0])
1946         s->block_last_index[n] = 0;
1947     else
1948         s->block_last_index[n] = -1;
1949 }
1950
1951 static inline void clip_coeffs(MpegEncContext *s, int16_t *block,
1952                                int last_index)
1953 {
1954     int i;
1955     const int maxlevel = s->max_qcoeff;
1956     const int minlevel = s->min_qcoeff;
1957     int overflow = 0;
1958
1959     if (s->mb_intra) {
1960         i = 1; // skip clipping of intra dc
1961     } else
1962         i = 0;
1963
1964     for (; i <= last_index; i++) {
1965         const int j = s->intra_scantable.permutated[i];
1966         int level = block[j];
1967
1968         if (level > maxlevel) {
1969             level = maxlevel;
1970             overflow++;
1971         } else if (level < minlevel) {
1972             level = minlevel;
1973             overflow++;
1974         }
1975
1976         block[j] = level;
1977     }
1978
1979     if (overflow && s->avctx->mb_decision == FF_MB_DECISION_SIMPLE)
1980         av_log(s->avctx, AV_LOG_INFO,
1981                "warning, clipping %d dct coefficients to %d..%d\n",
1982                overflow, minlevel, maxlevel);
1983 }
1984
1985 static void get_visual_weight(int16_t *weight, uint8_t *ptr, int stride)
1986 {
1987     int x, y;
1988     // FIXME optimize
1989     for (y = 0; y < 8; y++) {
1990         for (x = 0; x < 8; x++) {
1991             int x2, y2;
1992             int sum = 0;
1993             int sqr = 0;
1994             int count = 0;
1995
1996             for (y2 = FFMAX(y - 1, 0); y2 < FFMIN(8, y + 2); y2++) {
1997                 for (x2= FFMAX(x - 1, 0); x2 < FFMIN(8, x + 2); x2++) {
1998                     int v = ptr[x2 + y2 * stride];
1999                     sum += v;
2000                     sqr += v * v;
2001                     count++;
2002                 }
2003             }
2004             weight[x + 8 * y]= (36 * ff_sqrt(count * sqr - sum * sum)) / count;
2005         }
2006     }
2007 }
2008
2009 static av_always_inline void encode_mb_internal(MpegEncContext *s,
2010                                                 int motion_x, int motion_y,
2011                                                 int mb_block_height,
2012                                                 int mb_block_count)
2013 {
2014     int16_t weight[8][64];
2015     int16_t orig[8][64];
2016     const int mb_x = s->mb_x;
2017     const int mb_y = s->mb_y;
2018     int i;
2019     int skip_dct[8];
2020     int dct_offset = s->linesize * 8; // default for progressive frames
2021     uint8_t *ptr_y, *ptr_cb, *ptr_cr;
2022     ptrdiff_t wrap_y, wrap_c;
2023
2024     for (i = 0; i < mb_block_count; i++)
2025         skip_dct[i] = s->skipdct;
2026
2027     if (s->adaptive_quant) {
2028         const int last_qp = s->qscale;
2029         const int mb_xy = mb_x + mb_y * s->mb_stride;
2030
2031         s->lambda = s->lambda_table[mb_xy];
2032         update_qscale(s);
2033
2034         if (!(s->mpv_flags & FF_MPV_FLAG_QP_RD)) {
2035             s->qscale = s->current_picture_ptr->qscale_table[mb_xy];
2036             s->dquant = s->qscale - last_qp;
2037
2038             if (s->out_format == FMT_H263) {
2039                 s->dquant = av_clip(s->dquant, -2, 2);
2040
2041                 if (s->codec_id == AV_CODEC_ID_MPEG4) {
2042                     if (!s->mb_intra) {
2043                         if (s->pict_type == AV_PICTURE_TYPE_B) {
2044                             if (s->dquant & 1 || s->mv_dir & MV_DIRECT)
2045                                 s->dquant = 0;
2046                         }
2047                         if (s->mv_type == MV_TYPE_8X8)
2048                             s->dquant = 0;
2049                     }
2050                 }
2051             }
2052         }
2053         ff_set_qscale(s, last_qp + s->dquant);
2054     } else if (s->mpv_flags & FF_MPV_FLAG_QP_RD)
2055         ff_set_qscale(s, s->qscale + s->dquant);
2056
2057     wrap_y = s->linesize;
2058     wrap_c = s->uvlinesize;
2059     ptr_y  = s->new_picture.f->data[0] +
2060              (mb_y * 16 * wrap_y)              + mb_x * 16;
2061     ptr_cb = s->new_picture.f->data[1] +
2062              (mb_y * mb_block_height * wrap_c) + mb_x * 8;
2063     ptr_cr = s->new_picture.f->data[2] +
2064              (mb_y * mb_block_height * wrap_c) + mb_x * 8;
2065
2066     if (mb_x * 16 + 16 > s->width || mb_y * 16 + 16 > s->height) {
2067         uint8_t *ebuf = s->sc.edge_emu_buffer + 32;
2068         s->vdsp.emulated_edge_mc(ebuf, ptr_y,
2069                                  wrap_y, wrap_y,
2070                                  16, 16, mb_x * 16, mb_y * 16,
2071                                  s->width, s->height);
2072         ptr_y = ebuf;
2073         s->vdsp.emulated_edge_mc(ebuf + 18 * wrap_y, ptr_cb,
2074                                  wrap_c, wrap_c,
2075                                  8, mb_block_height, mb_x * 8, mb_y * 8,
2076                                  s->width >> 1, s->height >> 1);
2077         ptr_cb = ebuf + 18 * wrap_y;
2078         s->vdsp.emulated_edge_mc(ebuf + 18 * wrap_y + 8, ptr_cr,
2079                                  wrap_c, wrap_c,
2080                                  8, mb_block_height, mb_x * 8, mb_y * 8,
2081                                  s->width >> 1, s->height >> 1);
2082         ptr_cr = ebuf + 18 * wrap_y + 8;
2083     }
2084
2085     if (s->mb_intra) {
2086         if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_DCT) {
2087             int progressive_score, interlaced_score;
2088
2089             s->interlaced_dct = 0;
2090             progressive_score = s->mecc.ildct_cmp[4](s, ptr_y, NULL, wrap_y, 8) +
2091                                 s->mecc.ildct_cmp[4](s, ptr_y + wrap_y * 8,
2092                                                      NULL, wrap_y, 8) - 400;
2093
2094             if (progressive_score > 0) {
2095                 interlaced_score = s->mecc.ildct_cmp[4](s, ptr_y,
2096                                                         NULL, wrap_y * 2, 8) +
2097                                    s->mecc.ildct_cmp[4](s, ptr_y + wrap_y,
2098                                                         NULL, wrap_y * 2, 8);
2099                 if (progressive_score > interlaced_score) {
2100                     s->interlaced_dct = 1;
2101
2102                     dct_offset = wrap_y;
2103                     wrap_y <<= 1;
2104                     if (s->chroma_format == CHROMA_422)
2105                         wrap_c <<= 1;
2106                 }
2107             }
2108         }
2109
2110         s->pdsp.get_pixels(s->block[0], ptr_y,                  wrap_y);
2111         s->pdsp.get_pixels(s->block[1], ptr_y + 8,              wrap_y);
2112         s->pdsp.get_pixels(s->block[2], ptr_y + dct_offset,     wrap_y);
2113         s->pdsp.get_pixels(s->block[3], ptr_y + dct_offset + 8, wrap_y);
2114
2115         if (s->avctx->flags & AV_CODEC_FLAG_GRAY) {
2116             skip_dct[4] = 1;
2117             skip_dct[5] = 1;
2118         } else {
2119             s->pdsp.get_pixels(s->block[4], ptr_cb, wrap_c);
2120             s->pdsp.get_pixels(s->block[5], ptr_cr, wrap_c);
2121             if (!s->chroma_y_shift) { /* 422 */
2122                 s->pdsp.get_pixels(s->block[6],
2123                                    ptr_cb + (dct_offset >> 1), wrap_c);
2124                 s->pdsp.get_pixels(s->block[7],
2125                                    ptr_cr + (dct_offset >> 1), wrap_c);
2126             }
2127         }
2128     } else {
2129         op_pixels_func (*op_pix)[4];
2130         qpel_mc_func (*op_qpix)[16];
2131         uint8_t *dest_y, *dest_cb, *dest_cr;
2132
2133         dest_y  = s->dest[0];
2134         dest_cb = s->dest[1];
2135         dest_cr = s->dest[2];
2136
2137         if ((!s->no_rounding) || s->pict_type == AV_PICTURE_TYPE_B) {
2138             op_pix  = s->hdsp.put_pixels_tab;
2139             op_qpix = s->qdsp.put_qpel_pixels_tab;
2140         } else {
2141             op_pix  = s->hdsp.put_no_rnd_pixels_tab;
2142             op_qpix = s->qdsp.put_no_rnd_qpel_pixels_tab;
2143         }
2144
2145         if (s->mv_dir & MV_DIR_FORWARD) {
2146             ff_mpv_motion(s, dest_y, dest_cb, dest_cr, 0,
2147                           s->last_picture.f->data,
2148                           op_pix, op_qpix);
2149             op_pix  = s->hdsp.avg_pixels_tab;
2150             op_qpix = s->qdsp.avg_qpel_pixels_tab;
2151         }
2152         if (s->mv_dir & MV_DIR_BACKWARD) {
2153             ff_mpv_motion(s, dest_y, dest_cb, dest_cr, 1,
2154                           s->next_picture.f->data,
2155                           op_pix, op_qpix);
2156         }
2157
2158         if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_DCT) {
2159             int progressive_score, interlaced_score;
2160
2161             s->interlaced_dct = 0;
2162             progressive_score = s->mecc.ildct_cmp[0](s, dest_y, ptr_y, wrap_y, 8) +
2163                                 s->mecc.ildct_cmp[0](s, dest_y + wrap_y * 8,
2164                                                      ptr_y + wrap_y * 8,
2165                                                      wrap_y, 8) - 400;
2166
2167             if (s->avctx->ildct_cmp == FF_CMP_VSSE)
2168                 progressive_score -= 400;
2169
2170             if (progressive_score > 0) {
2171                 interlaced_score = s->mecc.ildct_cmp[0](s, dest_y, ptr_y,
2172                                                         wrap_y * 2, 8) +
2173                                    s->mecc.ildct_cmp[0](s, dest_y + wrap_y,
2174                                                         ptr_y + wrap_y,
2175                                                         wrap_y * 2, 8);
2176
2177                 if (progressive_score > interlaced_score) {
2178                     s->interlaced_dct = 1;
2179
2180                     dct_offset = wrap_y;
2181                     wrap_y <<= 1;
2182                     if (s->chroma_format == CHROMA_422)
2183                         wrap_c <<= 1;
2184                 }
2185             }
2186         }
2187
2188         s->pdsp.diff_pixels(s->block[0], ptr_y, dest_y, wrap_y);
2189         s->pdsp.diff_pixels(s->block[1], ptr_y + 8, dest_y + 8, wrap_y);
2190         s->pdsp.diff_pixels(s->block[2], ptr_y + dct_offset,
2191                             dest_y + dct_offset, wrap_y);
2192         s->pdsp.diff_pixels(s->block[3], ptr_y + dct_offset + 8,
2193                             dest_y + dct_offset + 8, wrap_y);
2194
2195         if (s->avctx->flags & AV_CODEC_FLAG_GRAY) {
2196             skip_dct[4] = 1;
2197             skip_dct[5] = 1;
2198         } else {
2199             s->pdsp.diff_pixels(s->block[4], ptr_cb, dest_cb, wrap_c);
2200             s->pdsp.diff_pixels(s->block[5], ptr_cr, dest_cr, wrap_c);
2201             if (!s->chroma_y_shift) { /* 422 */
2202                 s->pdsp.diff_pixels(s->block[6], ptr_cb + (dct_offset >> 1),
2203                                     dest_cb + (dct_offset >> 1), wrap_c);
2204                 s->pdsp.diff_pixels(s->block[7], ptr_cr + (dct_offset >> 1),
2205                                     dest_cr + (dct_offset >> 1), wrap_c);
2206             }
2207         }
2208         /* pre quantization */
2209         if (s->current_picture.mc_mb_var[s->mb_stride * mb_y + mb_x] <
2210                 2 * s->qscale * s->qscale) {
2211             // FIXME optimize
2212             if (s->mecc.sad[1](NULL, ptr_y, dest_y, wrap_y, 8) < 20 * s->qscale)
2213                 skip_dct[0] = 1;
2214             if (s->mecc.sad[1](NULL, ptr_y + 8, dest_y + 8, wrap_y, 8) < 20 * s->qscale)
2215                 skip_dct[1] = 1;
2216             if (s->mecc.sad[1](NULL, ptr_y + dct_offset, dest_y + dct_offset,
2217                                wrap_y, 8) < 20 * s->qscale)
2218                 skip_dct[2] = 1;
2219             if (s->mecc.sad[1](NULL, ptr_y + dct_offset + 8, dest_y + dct_offset + 8,
2220                                wrap_y, 8) < 20 * s->qscale)
2221                 skip_dct[3] = 1;
2222             if (s->mecc.sad[1](NULL, ptr_cb, dest_cb, wrap_c, 8) < 20 * s->qscale)
2223                 skip_dct[4] = 1;
2224             if (s->mecc.sad[1](NULL, ptr_cr, dest_cr, wrap_c, 8) < 20 * s->qscale)
2225                 skip_dct[5] = 1;
2226             if (!s->chroma_y_shift) { /* 422 */
2227                 if (s->mecc.sad[1](NULL, ptr_cb + (dct_offset >> 1),
2228                                    dest_cb + (dct_offset >> 1),
2229                                    wrap_c, 8) < 20 * s->qscale)
2230                     skip_dct[6] = 1;
2231                 if (s->mecc.sad[1](NULL, ptr_cr + (dct_offset >> 1),
2232                                    dest_cr + (dct_offset >> 1),
2233                                    wrap_c, 8) < 20 * s->qscale)
2234                     skip_dct[7] = 1;
2235             }
2236         }
2237     }
2238
2239     if (s->quantizer_noise_shaping) {
2240         if (!skip_dct[0])
2241             get_visual_weight(weight[0], ptr_y                 , wrap_y);
2242         if (!skip_dct[1])
2243             get_visual_weight(weight[1], ptr_y              + 8, wrap_y);
2244         if (!skip_dct[2])
2245             get_visual_weight(weight[2], ptr_y + dct_offset    , wrap_y);
2246         if (!skip_dct[3])
2247             get_visual_weight(weight[3], ptr_y + dct_offset + 8, wrap_y);
2248         if (!skip_dct[4])
2249             get_visual_weight(weight[4], ptr_cb                , wrap_c);
2250         if (!skip_dct[5])
2251             get_visual_weight(weight[5], ptr_cr                , wrap_c);
2252         if (!s->chroma_y_shift) { /* 422 */
2253             if (!skip_dct[6])
2254                 get_visual_weight(weight[6], ptr_cb + (dct_offset >> 1),
2255                                   wrap_c);
2256             if (!skip_dct[7])
2257                 get_visual_weight(weight[7], ptr_cr + (dct_offset >> 1),
2258                                   wrap_c);
2259         }
2260         memcpy(orig[0], s->block[0], sizeof(int16_t) * 64 * mb_block_count);
2261     }
2262
2263     /* DCT & quantize */
2264     assert(s->out_format != FMT_MJPEG || s->qscale == 8);
2265     {
2266         for (i = 0; i < mb_block_count; i++) {
2267             if (!skip_dct[i]) {
2268                 int overflow;
2269                 s->block_last_index[i] = s->dct_quantize(s, s->block[i], i, s->qscale, &overflow);
2270                 // FIXME we could decide to change to quantizer instead of
2271                 // clipping
2272                 // JS: I don't think that would be a good idea it could lower
2273                 //     quality instead of improve it. Just INTRADC clipping
2274                 //     deserves changes in quantizer
2275                 if (overflow)
2276                     clip_coeffs(s, s->block[i], s->block_last_index[i]);
2277             } else
2278                 s->block_last_index[i] = -1;
2279         }
2280         if (s->quantizer_noise_shaping) {
2281             for (i = 0; i < mb_block_count; i++) {
2282                 if (!skip_dct[i]) {
2283                     s->block_last_index[i] =
2284                         dct_quantize_refine(s, s->block[i], weight[i],
2285                                             orig[i], i, s->qscale);
2286                 }
2287             }
2288         }
2289
2290         if (s->luma_elim_threshold && !s->mb_intra)
2291             for (i = 0; i < 4; i++)
2292                 dct_single_coeff_elimination(s, i, s->luma_elim_threshold);
2293         if (s->chroma_elim_threshold && !s->mb_intra)
2294             for (i = 4; i < mb_block_count; i++)
2295                 dct_single_coeff_elimination(s, i, s->chroma_elim_threshold);
2296
2297         if (s->mpv_flags & FF_MPV_FLAG_CBP_RD) {
2298             for (i = 0; i < mb_block_count; i++) {
2299                 if (s->block_last_index[i] == -1)
2300                     s->coded_score[i] = INT_MAX / 256;
2301             }
2302         }
2303     }
2304
2305     if ((s->avctx->flags & AV_CODEC_FLAG_GRAY) && s->mb_intra) {
2306         s->block_last_index[4] =
2307         s->block_last_index[5] = 0;
2308         s->block[4][0] =
2309         s->block[5][0] = (1024 + s->c_dc_scale / 2) / s->c_dc_scale;
2310     }
2311
2312     // non c quantize code returns incorrect block_last_index FIXME
2313     if (s->alternate_scan && s->dct_quantize != ff_dct_quantize_c) {
2314         for (i = 0; i < mb_block_count; i++) {
2315             int j;
2316             if (s->block_last_index[i] > 0) {
2317                 for (j = 63; j > 0; j--) {
2318                     if (s->block[i][s->intra_scantable.permutated[j]])
2319                         break;
2320                 }
2321                 s->block_last_index[i] = j;
2322             }
2323         }
2324     }
2325
2326     /* huffman encode */
2327     switch(s->codec_id){ //FIXME funct ptr could be slightly faster
2328     case AV_CODEC_ID_MPEG1VIDEO:
2329     case AV_CODEC_ID_MPEG2VIDEO:
2330         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
2331             ff_mpeg1_encode_mb(s, s->block, motion_x, motion_y);
2332         break;
2333     case AV_CODEC_ID_MPEG4:
2334         if (CONFIG_MPEG4_ENCODER)
2335             ff_mpeg4_encode_mb(s, s->block, motion_x, motion_y);
2336         break;
2337     case AV_CODEC_ID_MSMPEG4V2:
2338     case AV_CODEC_ID_MSMPEG4V3:
2339     case AV_CODEC_ID_WMV1:
2340         if (CONFIG_MSMPEG4_ENCODER)
2341             ff_msmpeg4_encode_mb(s, s->block, motion_x, motion_y);
2342         break;
2343     case AV_CODEC_ID_WMV2:
2344         if (CONFIG_WMV2_ENCODER)
2345             ff_wmv2_encode_mb(s, s->block, motion_x, motion_y);
2346         break;
2347     case AV_CODEC_ID_H261:
2348         if (CONFIG_H261_ENCODER)
2349             ff_h261_encode_mb(s, s->block, motion_x, motion_y);
2350         break;
2351     case AV_CODEC_ID_H263:
2352     case AV_CODEC_ID_H263P:
2353     case AV_CODEC_ID_FLV1:
2354     case AV_CODEC_ID_RV10:
2355     case AV_CODEC_ID_RV20:
2356         if (CONFIG_H263_ENCODER)
2357             ff_h263_encode_mb(s, s->block, motion_x, motion_y);
2358         break;
2359     case AV_CODEC_ID_MJPEG:
2360         if (CONFIG_MJPEG_ENCODER)
2361             ff_mjpeg_encode_mb(s, s->block);
2362         break;
2363     default:
2364         assert(0);
2365     }
2366 }
2367
2368 static av_always_inline void encode_mb(MpegEncContext *s, int motion_x, int motion_y)
2369 {
2370     if (s->chroma_format == CHROMA_420) encode_mb_internal(s, motion_x, motion_y,  8, 6);
2371     else                                encode_mb_internal(s, motion_x, motion_y, 16, 8);
2372 }
2373
2374 static inline void copy_context_before_encode(MpegEncContext *d, MpegEncContext *s, int type){
2375     int i;
2376
2377     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2378
2379     /* MPEG-1 */
2380     d->mb_skip_run= s->mb_skip_run;
2381     for(i=0; i<3; i++)
2382         d->last_dc[i] = s->last_dc[i];
2383
2384     /* statistics */
2385     d->mv_bits= s->mv_bits;
2386     d->i_tex_bits= s->i_tex_bits;
2387     d->p_tex_bits= s->p_tex_bits;
2388     d->i_count= s->i_count;
2389     d->f_count= s->f_count;
2390     d->b_count= s->b_count;
2391     d->skip_count= s->skip_count;
2392     d->misc_bits= s->misc_bits;
2393     d->last_bits= 0;
2394
2395     d->mb_skipped= 0;
2396     d->qscale= s->qscale;
2397     d->dquant= s->dquant;
2398
2399     d->esc3_level_length= s->esc3_level_length;
2400 }
2401
2402 static inline void copy_context_after_encode(MpegEncContext *d, MpegEncContext *s, int type){
2403     int i;
2404
2405     memcpy(d->mv, s->mv, 2*4*2*sizeof(int));
2406     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2407
2408     /* MPEG-1 */
2409     d->mb_skip_run= s->mb_skip_run;
2410     for(i=0; i<3; i++)
2411         d->last_dc[i] = s->last_dc[i];
2412
2413     /* statistics */
2414     d->mv_bits= s->mv_bits;
2415     d->i_tex_bits= s->i_tex_bits;
2416     d->p_tex_bits= s->p_tex_bits;
2417     d->i_count= s->i_count;
2418     d->f_count= s->f_count;
2419     d->b_count= s->b_count;
2420     d->skip_count= s->skip_count;
2421     d->misc_bits= s->misc_bits;
2422
2423     d->mb_intra= s->mb_intra;
2424     d->mb_skipped= s->mb_skipped;
2425     d->mv_type= s->mv_type;
2426     d->mv_dir= s->mv_dir;
2427     d->pb= s->pb;
2428     if(s->data_partitioning){
2429         d->pb2= s->pb2;
2430         d->tex_pb= s->tex_pb;
2431     }
2432     d->block= s->block;
2433     for(i=0; i<8; i++)
2434         d->block_last_index[i]= s->block_last_index[i];
2435     d->interlaced_dct= s->interlaced_dct;
2436     d->qscale= s->qscale;
2437
2438     d->esc3_level_length= s->esc3_level_length;
2439 }
2440
2441 static inline void encode_mb_hq(MpegEncContext *s, MpegEncContext *backup, MpegEncContext *best, int type,
2442                            PutBitContext pb[2], PutBitContext pb2[2], PutBitContext tex_pb[2],
2443                            int *dmin, int *next_block, int motion_x, int motion_y)
2444 {
2445     int score;
2446     uint8_t *dest_backup[3];
2447
2448     copy_context_before_encode(s, backup, type);
2449
2450     s->block= s->blocks[*next_block];
2451     s->pb= pb[*next_block];
2452     if(s->data_partitioning){
2453         s->pb2   = pb2   [*next_block];
2454         s->tex_pb= tex_pb[*next_block];
2455     }
2456
2457     if(*next_block){
2458         memcpy(dest_backup, s->dest, sizeof(s->dest));
2459         s->dest[0] = s->sc.rd_scratchpad;
2460         s->dest[1] = s->sc.rd_scratchpad + 16*s->linesize;
2461         s->dest[2] = s->sc.rd_scratchpad + 16*s->linesize + 8;
2462         assert(s->linesize >= 32); //FIXME
2463     }
2464
2465     encode_mb(s, motion_x, motion_y);
2466
2467     score= put_bits_count(&s->pb);
2468     if(s->data_partitioning){
2469         score+= put_bits_count(&s->pb2);
2470         score+= put_bits_count(&s->tex_pb);
2471     }
2472
2473     if(s->avctx->mb_decision == FF_MB_DECISION_RD){
2474         ff_mpv_decode_mb(s, s->block);
2475
2476         score *= s->lambda2;
2477         score += sse_mb(s) << FF_LAMBDA_SHIFT;
2478     }
2479
2480     if(*next_block){
2481         memcpy(s->dest, dest_backup, sizeof(s->dest));
2482     }
2483
2484     if(score<*dmin){
2485         *dmin= score;
2486         *next_block^=1;
2487
2488         copy_context_after_encode(best, s, type);
2489     }
2490 }
2491
2492 static int sse(MpegEncContext *s, uint8_t *src1, uint8_t *src2, int w, int h, int stride){
2493     uint32_t *sq = ff_square_tab + 256;
2494     int acc=0;
2495     int x,y;
2496
2497     if(w==16 && h==16)
2498         return s->mecc.sse[0](NULL, src1, src2, stride, 16);
2499     else if(w==8 && h==8)
2500         return s->mecc.sse[1](NULL, src1, src2, stride, 8);
2501
2502     for(y=0; y<h; y++){
2503         for(x=0; x<w; x++){
2504             acc+= sq[src1[x + y*stride] - src2[x + y*stride]];
2505         }
2506     }
2507
2508     assert(acc>=0);
2509
2510     return acc;
2511 }
2512
2513 static int sse_mb(MpegEncContext *s){
2514     int w= 16;
2515     int h= 16;
2516
2517     if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
2518     if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
2519
2520     if(w==16 && h==16)
2521       if(s->avctx->mb_cmp == FF_CMP_NSSE){
2522         return s->mecc.nsse[0](s, s->new_picture.f->data[0] + s->mb_x * 16 + s->mb_y * s->linesize   * 16, s->dest[0], s->linesize,   16) +
2523                s->mecc.nsse[1](s, s->new_picture.f->data[1] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[1], s->uvlinesize,  8) +
2524                s->mecc.nsse[1](s, s->new_picture.f->data[2] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[2], s->uvlinesize,  8);
2525       }else{
2526         return s->mecc.sse[0](NULL, s->new_picture.f->data[0] + s->mb_x * 16 + s->mb_y * s->linesize   * 16, s->dest[0], s->linesize,   16) +
2527                s->mecc.sse[1](NULL, s->new_picture.f->data[1] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[1], s->uvlinesize,  8) +
2528                s->mecc.sse[1](NULL, s->new_picture.f->data[2] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[2], s->uvlinesize,  8);
2529       }
2530     else
2531         return  sse(s, s->new_picture.f->data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], w, h, s->linesize)
2532                +sse(s, s->new_picture.f->data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], w>>1, h>>1, s->uvlinesize)
2533                +sse(s, s->new_picture.f->data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], w>>1, h>>1, s->uvlinesize);
2534 }
2535
2536 static int pre_estimate_motion_thread(AVCodecContext *c, void *arg){
2537     MpegEncContext *s= *(void**)arg;
2538
2539
2540     s->me.pre_pass=1;
2541     s->me.dia_size= s->avctx->pre_dia_size;
2542     s->first_slice_line=1;
2543     for(s->mb_y= s->end_mb_y-1; s->mb_y >= s->start_mb_y; s->mb_y--) {
2544         for(s->mb_x=s->mb_width-1; s->mb_x >=0 ;s->mb_x--) {
2545             ff_pre_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2546         }
2547         s->first_slice_line=0;
2548     }
2549
2550     s->me.pre_pass=0;
2551
2552     return 0;
2553 }
2554
2555 static int estimate_motion_thread(AVCodecContext *c, void *arg){
2556     MpegEncContext *s= *(void**)arg;
2557
2558     s->me.dia_size= s->avctx->dia_size;
2559     s->first_slice_line=1;
2560     for(s->mb_y= s->start_mb_y; s->mb_y < s->end_mb_y; s->mb_y++) {
2561         s->mb_x=0; //for block init below
2562         ff_init_block_index(s);
2563         for(s->mb_x=0; s->mb_x < s->mb_width; s->mb_x++) {
2564             s->block_index[0]+=2;
2565             s->block_index[1]+=2;
2566             s->block_index[2]+=2;
2567             s->block_index[3]+=2;
2568
2569             /* compute motion vector & mb_type and store in context */
2570             if(s->pict_type==AV_PICTURE_TYPE_B)
2571                 ff_estimate_b_frame_motion(s, s->mb_x, s->mb_y);
2572             else
2573                 ff_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2574         }
2575         s->first_slice_line=0;
2576     }
2577     return 0;
2578 }
2579
2580 static int mb_var_thread(AVCodecContext *c, void *arg){
2581     MpegEncContext *s= *(void**)arg;
2582     int mb_x, mb_y;
2583
2584     for(mb_y=s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2585         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2586             int xx = mb_x * 16;
2587             int yy = mb_y * 16;
2588             uint8_t *pix = s->new_picture.f->data[0] + (yy * s->linesize) + xx;
2589             int varc;
2590             int sum = s->mpvencdsp.pix_sum(pix, s->linesize);
2591
2592             varc = (s->mpvencdsp.pix_norm1(pix, s->linesize) -
2593                     (((unsigned) sum * sum) >> 8) + 500 + 128) >> 8;
2594
2595             s->current_picture.mb_var [s->mb_stride * mb_y + mb_x] = varc;
2596             s->current_picture.mb_mean[s->mb_stride * mb_y + mb_x] = (sum+128)>>8;
2597             s->me.mb_var_sum_temp    += varc;
2598         }
2599     }
2600     return 0;
2601 }
2602
2603 static void write_slice_end(MpegEncContext *s){
2604     if(CONFIG_MPEG4_ENCODER && s->codec_id==AV_CODEC_ID_MPEG4){
2605         if(s->partitioned_frame){
2606             ff_mpeg4_merge_partitions(s);
2607         }
2608
2609         ff_mpeg4_stuffing(&s->pb);
2610     }else if(CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG){
2611         ff_mjpeg_encode_stuffing(&s->pb);
2612     }
2613
2614     avpriv_align_put_bits(&s->pb);
2615     flush_put_bits(&s->pb);
2616
2617     if ((s->avctx->flags & AV_CODEC_FLAG_PASS1) && !s->partitioned_frame)
2618         s->misc_bits+= get_bits_diff(s);
2619 }
2620
2621 static void write_mb_info(MpegEncContext *s)
2622 {
2623     uint8_t *ptr = s->mb_info_ptr + s->mb_info_size - 12;
2624     int offset = put_bits_count(&s->pb);
2625     int mba  = s->mb_x + s->mb_width * (s->mb_y % s->gob_index);
2626     int gobn = s->mb_y / s->gob_index;
2627     int pred_x, pred_y;
2628     if (CONFIG_H263_ENCODER)
2629         ff_h263_pred_motion(s, 0, 0, &pred_x, &pred_y);
2630     bytestream_put_le32(&ptr, offset);
2631     bytestream_put_byte(&ptr, s->qscale);
2632     bytestream_put_byte(&ptr, gobn);
2633     bytestream_put_le16(&ptr, mba);
2634     bytestream_put_byte(&ptr, pred_x); /* hmv1 */
2635     bytestream_put_byte(&ptr, pred_y); /* vmv1 */
2636     /* 4MV not implemented */
2637     bytestream_put_byte(&ptr, 0); /* hmv2 */
2638     bytestream_put_byte(&ptr, 0); /* vmv2 */
2639 }
2640
2641 static void update_mb_info(MpegEncContext *s, int startcode)
2642 {
2643     if (!s->mb_info)
2644         return;
2645     if (put_bits_count(&s->pb) - s->prev_mb_info*8 >= s->mb_info*8) {
2646         s->mb_info_size += 12;
2647         s->prev_mb_info = s->last_mb_info;
2648     }
2649     if (startcode) {
2650         s->prev_mb_info = put_bits_count(&s->pb)/8;
2651         /* This might have incremented mb_info_size above, and we return without
2652          * actually writing any info into that slot yet. But in that case,
2653          * this will be called again at the start of the after writing the
2654          * start code, actually writing the mb info. */
2655         return;
2656     }
2657
2658     s->last_mb_info = put_bits_count(&s->pb)/8;
2659     if (!s->mb_info_size)
2660         s->mb_info_size += 12;
2661     write_mb_info(s);
2662 }
2663
2664 static int encode_thread(AVCodecContext *c, void *arg){
2665     MpegEncContext *s= *(void**)arg;
2666     int mb_x, mb_y;
2667     int chr_h= 16>>s->chroma_y_shift;
2668     int i, j;
2669     MpegEncContext best_s = { 0 }, backup_s;
2670     uint8_t bit_buf[2][MAX_MB_BYTES];
2671     uint8_t bit_buf2[2][MAX_MB_BYTES];
2672     uint8_t bit_buf_tex[2][MAX_MB_BYTES];
2673     PutBitContext pb[2], pb2[2], tex_pb[2];
2674
2675     for(i=0; i<2; i++){
2676         init_put_bits(&pb    [i], bit_buf    [i], MAX_MB_BYTES);
2677         init_put_bits(&pb2   [i], bit_buf2   [i], MAX_MB_BYTES);
2678         init_put_bits(&tex_pb[i], bit_buf_tex[i], MAX_MB_BYTES);
2679     }
2680
2681     s->last_bits= put_bits_count(&s->pb);
2682     s->mv_bits=0;
2683     s->misc_bits=0;
2684     s->i_tex_bits=0;
2685     s->p_tex_bits=0;
2686     s->i_count=0;
2687     s->f_count=0;
2688     s->b_count=0;
2689     s->skip_count=0;
2690
2691     for(i=0; i<3; i++){
2692         /* init last dc values */
2693         /* note: quant matrix value (8) is implied here */
2694         s->last_dc[i] = 128 << s->intra_dc_precision;
2695
2696         s->current_picture.encoding_error[i] = 0;
2697     }
2698     s->mb_skip_run = 0;
2699     memset(s->last_mv, 0, sizeof(s->last_mv));
2700
2701     s->last_mv_dir = 0;
2702
2703     switch(s->codec_id){
2704     case AV_CODEC_ID_H263:
2705     case AV_CODEC_ID_H263P:
2706     case AV_CODEC_ID_FLV1:
2707         if (CONFIG_H263_ENCODER)
2708             s->gob_index = H263_GOB_HEIGHT(s->height);
2709         break;
2710     case AV_CODEC_ID_MPEG4:
2711         if(CONFIG_MPEG4_ENCODER && s->partitioned_frame)
2712             ff_mpeg4_init_partitions(s);
2713         break;
2714     }
2715
2716     s->resync_mb_x=0;
2717     s->resync_mb_y=0;
2718     s->first_slice_line = 1;
2719     s->ptr_lastgob = s->pb.buf;
2720     for(mb_y= s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2721         s->mb_x=0;
2722         s->mb_y= mb_y;
2723
2724         ff_set_qscale(s, s->qscale);
2725         ff_init_block_index(s);
2726
2727         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2728             int xy= mb_y*s->mb_stride + mb_x; // removed const, H261 needs to adjust this
2729             int mb_type= s->mb_type[xy];
2730 //            int d;
2731             int dmin= INT_MAX;
2732             int dir;
2733
2734             if(s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < MAX_MB_BYTES){
2735                 av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2736                 return -1;
2737             }
2738             if(s->data_partitioning){
2739                 if(   s->pb2   .buf_end - s->pb2   .buf - (put_bits_count(&s->    pb2)>>3) < MAX_MB_BYTES
2740                    || s->tex_pb.buf_end - s->tex_pb.buf - (put_bits_count(&s->tex_pb )>>3) < MAX_MB_BYTES){
2741                     av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2742                     return -1;
2743                 }
2744             }
2745
2746             s->mb_x = mb_x;
2747             s->mb_y = mb_y;  // moved into loop, can get changed by H.261
2748             ff_update_block_index(s);
2749
2750             if(CONFIG_H261_ENCODER && s->codec_id == AV_CODEC_ID_H261){
2751                 ff_h261_reorder_mb_index(s);
2752                 xy= s->mb_y*s->mb_stride + s->mb_x;
2753                 mb_type= s->mb_type[xy];
2754             }
2755
2756             /* write gob / video packet header  */
2757             if(s->rtp_mode){
2758                 int current_packet_size, is_gob_start;
2759
2760                 current_packet_size= ((put_bits_count(&s->pb)+7)>>3) - (s->ptr_lastgob - s->pb.buf);
2761
2762                 is_gob_start = s->rtp_payload_size &&
2763                                current_packet_size >= s->rtp_payload_size &&
2764                                mb_y + mb_x > 0;
2765
2766                 if(s->start_mb_y == mb_y && mb_y > 0 && mb_x==0) is_gob_start=1;
2767
2768                 switch(s->codec_id){
2769                 case AV_CODEC_ID_H263:
2770                 case AV_CODEC_ID_H263P:
2771                     if(!s->h263_slice_structured)
2772                         if(s->mb_x || s->mb_y%s->gob_index) is_gob_start=0;
2773                     break;
2774                 case AV_CODEC_ID_MPEG2VIDEO:
2775                     if(s->mb_x==0 && s->mb_y!=0) is_gob_start=1;
2776                 case AV_CODEC_ID_MPEG1VIDEO:
2777                     if(s->mb_skip_run) is_gob_start=0;
2778                     break;
2779                 }
2780
2781                 if(is_gob_start){
2782                     if(s->start_mb_y != mb_y || mb_x!=0){
2783                         write_slice_end(s);
2784
2785                         if(CONFIG_MPEG4_ENCODER && s->codec_id==AV_CODEC_ID_MPEG4 && s->partitioned_frame){
2786                             ff_mpeg4_init_partitions(s);
2787                         }
2788                     }
2789
2790                     assert((put_bits_count(&s->pb)&7) == 0);
2791                     current_packet_size= put_bits_ptr(&s->pb) - s->ptr_lastgob;
2792
2793                     if (s->error_rate && s->resync_mb_x + s->resync_mb_y > 0) {
2794                         int r= put_bits_count(&s->pb)/8 + s->picture_number + 16 + s->mb_x + s->mb_y;
2795                         int d = 100 / s->error_rate;
2796                         if(r % d == 0){
2797                             current_packet_size=0;
2798                             s->pb.buf_ptr= s->ptr_lastgob;
2799                             assert(put_bits_ptr(&s->pb) == s->ptr_lastgob);
2800                         }
2801                     }
2802
2803 #if FF_API_RTP_CALLBACK
2804 FF_DISABLE_DEPRECATION_WARNINGS
2805                     if (s->avctx->rtp_callback){
2806                         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width + mb_x - s->resync_mb_x;
2807                         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, current_packet_size, number_mb);
2808                     }
2809 FF_ENABLE_DEPRECATION_WARNINGS
2810 #endif
2811                     update_mb_info(s, 1);
2812
2813                     switch(s->codec_id){
2814                     case AV_CODEC_ID_MPEG4:
2815                         if (CONFIG_MPEG4_ENCODER) {
2816                             ff_mpeg4_encode_video_packet_header(s);
2817                             ff_mpeg4_clean_buffers(s);
2818                         }
2819                     break;
2820                     case AV_CODEC_ID_MPEG1VIDEO:
2821                     case AV_CODEC_ID_MPEG2VIDEO:
2822                         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER) {
2823                             ff_mpeg1_encode_slice_header(s);
2824                             ff_mpeg1_clean_buffers(s);
2825                         }
2826                     break;
2827                     case AV_CODEC_ID_H263:
2828                     case AV_CODEC_ID_H263P:
2829                         if (CONFIG_H263_ENCODER)
2830                             ff_h263_encode_gob_header(s, mb_y);
2831                     break;
2832                     }
2833
2834                     if (s->avctx->flags & AV_CODEC_FLAG_PASS1) {
2835                         int bits= put_bits_count(&s->pb);
2836                         s->misc_bits+= bits - s->last_bits;
2837                         s->last_bits= bits;
2838                     }
2839
2840                     s->ptr_lastgob += current_packet_size;
2841                     s->first_slice_line=1;
2842                     s->resync_mb_x=mb_x;
2843                     s->resync_mb_y=mb_y;
2844                 }
2845             }
2846
2847             if(  (s->resync_mb_x   == s->mb_x)
2848                && s->resync_mb_y+1 == s->mb_y){
2849                 s->first_slice_line=0;
2850             }
2851
2852             s->mb_skipped=0;
2853             s->dquant=0; //only for QP_RD
2854
2855             update_mb_info(s, 0);
2856
2857             if (mb_type & (mb_type-1) || (s->mpv_flags & FF_MPV_FLAG_QP_RD)) { // more than 1 MB type possible or FF_MPV_FLAG_QP_RD
2858                 int next_block=0;
2859                 int pb_bits_count, pb2_bits_count, tex_pb_bits_count;
2860
2861                 copy_context_before_encode(&backup_s, s, -1);
2862                 backup_s.pb= s->pb;
2863                 best_s.data_partitioning= s->data_partitioning;
2864                 best_s.partitioned_frame= s->partitioned_frame;
2865                 if(s->data_partitioning){
2866                     backup_s.pb2= s->pb2;
2867                     backup_s.tex_pb= s->tex_pb;
2868                 }
2869
2870                 if(mb_type&CANDIDATE_MB_TYPE_INTER){
2871                     s->mv_dir = MV_DIR_FORWARD;
2872                     s->mv_type = MV_TYPE_16X16;
2873                     s->mb_intra= 0;
2874                     s->mv[0][0][0] = s->p_mv_table[xy][0];
2875                     s->mv[0][0][1] = s->p_mv_table[xy][1];
2876                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER, pb, pb2, tex_pb,
2877                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2878                 }
2879                 if(mb_type&CANDIDATE_MB_TYPE_INTER_I){
2880                     s->mv_dir = MV_DIR_FORWARD;
2881                     s->mv_type = MV_TYPE_FIELD;
2882                     s->mb_intra= 0;
2883                     for(i=0; i<2; i++){
2884                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
2885                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
2886                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
2887                     }
2888                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER_I, pb, pb2, tex_pb,
2889                                  &dmin, &next_block, 0, 0);
2890                 }
2891                 if(mb_type&CANDIDATE_MB_TYPE_SKIPPED){
2892                     s->mv_dir = MV_DIR_FORWARD;
2893                     s->mv_type = MV_TYPE_16X16;
2894                     s->mb_intra= 0;
2895                     s->mv[0][0][0] = 0;
2896                     s->mv[0][0][1] = 0;
2897                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_SKIPPED, pb, pb2, tex_pb,
2898                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2899                 }
2900                 if(mb_type&CANDIDATE_MB_TYPE_INTER4V){
2901                     s->mv_dir = MV_DIR_FORWARD;
2902                     s->mv_type = MV_TYPE_8X8;
2903                     s->mb_intra= 0;
2904                     for(i=0; i<4; i++){
2905                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
2906                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
2907                     }
2908                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER4V, pb, pb2, tex_pb,
2909                                  &dmin, &next_block, 0, 0);
2910                 }
2911                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD){
2912                     s->mv_dir = MV_DIR_FORWARD;
2913                     s->mv_type = MV_TYPE_16X16;
2914                     s->mb_intra= 0;
2915                     s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
2916                     s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
2917                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD, pb, pb2, tex_pb,
2918                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2919                 }
2920                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD){
2921                     s->mv_dir = MV_DIR_BACKWARD;
2922                     s->mv_type = MV_TYPE_16X16;
2923                     s->mb_intra= 0;
2924                     s->mv[1][0][0] = s->b_back_mv_table[xy][0];
2925                     s->mv[1][0][1] = s->b_back_mv_table[xy][1];
2926                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD, pb, pb2, tex_pb,
2927                                  &dmin, &next_block, s->mv[1][0][0], s->mv[1][0][1]);
2928                 }
2929                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR){
2930                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2931                     s->mv_type = MV_TYPE_16X16;
2932                     s->mb_intra= 0;
2933                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
2934                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
2935                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
2936                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
2937                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR, pb, pb2, tex_pb,
2938                                  &dmin, &next_block, 0, 0);
2939                 }
2940                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD_I){
2941                     s->mv_dir = MV_DIR_FORWARD;
2942                     s->mv_type = MV_TYPE_FIELD;
2943                     s->mb_intra= 0;
2944                     for(i=0; i<2; i++){
2945                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
2946                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
2947                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
2948                     }
2949                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD_I, pb, pb2, tex_pb,
2950                                  &dmin, &next_block, 0, 0);
2951                 }
2952                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD_I){
2953                     s->mv_dir = MV_DIR_BACKWARD;
2954                     s->mv_type = MV_TYPE_FIELD;
2955                     s->mb_intra= 0;
2956                     for(i=0; i<2; i++){
2957                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
2958                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
2959                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
2960                     }
2961                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD_I, pb, pb2, tex_pb,
2962                                  &dmin, &next_block, 0, 0);
2963                 }
2964                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR_I){
2965                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2966                     s->mv_type = MV_TYPE_FIELD;
2967                     s->mb_intra= 0;
2968                     for(dir=0; dir<2; dir++){
2969                         for(i=0; i<2; i++){
2970                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
2971                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
2972                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
2973                         }
2974                     }
2975                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR_I, pb, pb2, tex_pb,
2976                                  &dmin, &next_block, 0, 0);
2977                 }
2978                 if(mb_type&CANDIDATE_MB_TYPE_INTRA){
2979                     s->mv_dir = 0;
2980                     s->mv_type = MV_TYPE_16X16;
2981                     s->mb_intra= 1;
2982                     s->mv[0][0][0] = 0;
2983                     s->mv[0][0][1] = 0;
2984                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTRA, pb, pb2, tex_pb,
2985                                  &dmin, &next_block, 0, 0);
2986                     if(s->h263_pred || s->h263_aic){
2987                         if(best_s.mb_intra)
2988                             s->mbintra_table[mb_x + mb_y*s->mb_stride]=1;
2989                         else
2990                             ff_clean_intra_table_entries(s); //old mode?
2991                     }
2992                 }
2993
2994                 if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) && dmin < INT_MAX) {
2995                     if(best_s.mv_type==MV_TYPE_16X16){ //FIXME move 4mv after QPRD
2996                         const int last_qp= backup_s.qscale;
2997                         int qpi, qp, dc[6];
2998                         int16_t ac[6][16];
2999                         const int mvdir= (best_s.mv_dir&MV_DIR_BACKWARD) ? 1 : 0;
3000                         static const int dquant_tab[4]={-1,1,-2,2};
3001
3002                         assert(backup_s.dquant == 0);
3003
3004                         //FIXME intra
3005                         s->mv_dir= best_s.mv_dir;
3006                         s->mv_type = MV_TYPE_16X16;
3007                         s->mb_intra= best_s.mb_intra;
3008                         s->mv[0][0][0] = best_s.mv[0][0][0];
3009                         s->mv[0][0][1] = best_s.mv[0][0][1];
3010                         s->mv[1][0][0] = best_s.mv[1][0][0];
3011                         s->mv[1][0][1] = best_s.mv[1][0][1];
3012
3013                         qpi = s->pict_type == AV_PICTURE_TYPE_B ? 2 : 0;
3014                         for(; qpi<4; qpi++){
3015                             int dquant= dquant_tab[qpi];
3016                             qp= last_qp + dquant;
3017                             if(qp < s->avctx->qmin || qp > s->avctx->qmax)
3018                                 continue;
3019                             backup_s.dquant= dquant;
3020                             if(s->mb_intra && s->dc_val[0]){
3021                                 for(i=0; i<6; i++){
3022                                     dc[i]= s->dc_val[0][ s->block_index[i] ];
3023                                     memcpy(ac[i], s->ac_val[0][s->block_index[i]], sizeof(int16_t)*16);
3024                                 }
3025                             }
3026
3027                             encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
3028                                          &dmin, &next_block, s->mv[mvdir][0][0], s->mv[mvdir][0][1]);
3029                             if(best_s.qscale != qp){
3030                                 if(s->mb_intra && s->dc_val[0]){
3031                                     for(i=0; i<6; i++){
3032                                         s->dc_val[0][ s->block_index[i] ]= dc[i];
3033                                         memcpy(s->ac_val[0][s->block_index[i]], ac[i], sizeof(int16_t)*16);
3034                                     }
3035                                 }
3036                             }
3037                         }
3038                     }
3039                 }
3040                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT){
3041                     int mx= s->b_direct_mv_table[xy][0];
3042                     int my= s->b_direct_mv_table[xy][1];
3043
3044                     backup_s.dquant = 0;
3045                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
3046                     s->mb_intra= 0;
3047                     ff_mpeg4_set_direct_mv(s, mx, my);
3048                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
3049                                  &dmin, &next_block, mx, my);
3050                 }
3051                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT0){
3052                     backup_s.dquant = 0;
3053                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
3054                     s->mb_intra= 0;
3055                     ff_mpeg4_set_direct_mv(s, 0, 0);
3056                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
3057                                  &dmin, &next_block, 0, 0);
3058                 }
3059                 if (!best_s.mb_intra && s->mpv_flags & FF_MPV_FLAG_SKIP_RD) {
3060                     int coded=0;
3061                     for(i=0; i<6; i++)
3062                         coded |= s->block_last_index[i];
3063                     if(coded){
3064                         int mx,my;
3065                         memcpy(s->mv, best_s.mv, sizeof(s->mv));
3066                         if(CONFIG_MPEG4_ENCODER && best_s.mv_dir & MV_DIRECT){
3067                             mx=my=0; //FIXME find the one we actually used
3068                             ff_mpeg4_set_direct_mv(s, mx, my);
3069                         }else if(best_s.mv_dir&MV_DIR_BACKWARD){
3070                             mx= s->mv[1][0][0];
3071                             my= s->mv[1][0][1];
3072                         }else{
3073                             mx= s->mv[0][0][0];
3074                             my= s->mv[0][0][1];
3075                         }
3076
3077                         s->mv_dir= best_s.mv_dir;
3078                         s->mv_type = best_s.mv_type;
3079                         s->mb_intra= 0;
3080 /*                        s->mv[0][0][0] = best_s.mv[0][0][0];
3081                         s->mv[0][0][1] = best_s.mv[0][0][1];
3082                         s->mv[1][0][0] = best_s.mv[1][0][0];
3083                         s->mv[1][0][1] = best_s.mv[1][0][1];*/
3084                         backup_s.dquant= 0;
3085                         s->skipdct=1;
3086                         encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
3087                                         &dmin, &next_block, mx, my);
3088                         s->skipdct=0;
3089                     }
3090                 }
3091
3092                 s->current_picture.qscale_table[xy] = best_s.qscale;
3093
3094                 copy_context_after_encode(s, &best_s, -1);
3095
3096                 pb_bits_count= put_bits_count(&s->pb);
3097                 flush_put_bits(&s->pb);
3098                 avpriv_copy_bits(&backup_s.pb, bit_buf[next_block^1], pb_bits_count);
3099                 s->pb= backup_s.pb;
3100
3101                 if(s->data_partitioning){
3102                     pb2_bits_count= put_bits_count(&s->pb2);
3103                     flush_put_bits(&s->pb2);
3104                     avpriv_copy_bits(&backup_s.pb2, bit_buf2[next_block^1], pb2_bits_count);
3105                     s->pb2= backup_s.pb2;
3106
3107                     tex_pb_bits_count= put_bits_count(&s->tex_pb);
3108                     flush_put_bits(&s->tex_pb);
3109                     avpriv_copy_bits(&backup_s.tex_pb, bit_buf_tex[next_block^1], tex_pb_bits_count);
3110                     s->tex_pb= backup_s.tex_pb;
3111                 }
3112                 s->last_bits= put_bits_count(&s->pb);
3113
3114                 if (CONFIG_H263_ENCODER &&
3115                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
3116                     ff_h263_update_motion_val(s);
3117
3118                 if(next_block==0){ //FIXME 16 vs linesize16
3119                     s->hdsp.put_pixels_tab[0][0](s->dest[0], s->sc.rd_scratchpad                     , s->linesize  ,16);
3120                     s->hdsp.put_pixels_tab[1][0](s->dest[1], s->sc.rd_scratchpad + 16*s->linesize    , s->uvlinesize, 8);
3121                     s->hdsp.put_pixels_tab[1][0](s->dest[2], s->sc.rd_scratchpad + 16*s->linesize + 8, s->uvlinesize, 8);
3122                 }
3123
3124                 if(s->avctx->mb_decision == FF_MB_DECISION_BITS)
3125                     ff_mpv_decode_mb(s, s->block);
3126             } else {
3127                 int motion_x = 0, motion_y = 0;
3128                 s->mv_type=MV_TYPE_16X16;
3129                 // only one MB-Type possible
3130
3131                 switch(mb_type){
3132                 case CANDIDATE_MB_TYPE_INTRA:
3133                     s->mv_dir = 0;
3134                     s->mb_intra= 1;
3135                     motion_x= s->mv[0][0][0] = 0;
3136                     motion_y= s->mv[0][0][1] = 0;
3137                     break;
3138                 case CANDIDATE_MB_TYPE_INTER:
3139                     s->mv_dir = MV_DIR_FORWARD;
3140                     s->mb_intra= 0;
3141                     motion_x= s->mv[0][0][0] = s->p_mv_table[xy][0];
3142                     motion_y= s->mv[0][0][1] = s->p_mv_table[xy][1];
3143                     break;
3144                 case CANDIDATE_MB_TYPE_INTER_I:
3145                     s->mv_dir = MV_DIR_FORWARD;
3146                     s->mv_type = MV_TYPE_FIELD;
3147                     s->mb_intra= 0;
3148                     for(i=0; i<2; i++){
3149                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
3150                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
3151                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
3152                     }
3153                     break;
3154                 case CANDIDATE_MB_TYPE_INTER4V:
3155                     s->mv_dir = MV_DIR_FORWARD;
3156                     s->mv_type = MV_TYPE_8X8;
3157                     s->mb_intra= 0;
3158                     for(i=0; i<4; i++){
3159                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
3160                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
3161                     }
3162                     break;
3163                 case CANDIDATE_MB_TYPE_DIRECT:
3164                     if (CONFIG_MPEG4_ENCODER) {
3165                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
3166                         s->mb_intra= 0;
3167                         motion_x=s->b_direct_mv_table[xy][0];
3168                         motion_y=s->b_direct_mv_table[xy][1];
3169                         ff_mpeg4_set_direct_mv(s, motion_x, motion_y);
3170                     }
3171                     break;
3172                 case CANDIDATE_MB_TYPE_DIRECT0:
3173                     if (CONFIG_MPEG4_ENCODER) {
3174                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
3175                         s->mb_intra= 0;
3176                         ff_mpeg4_set_direct_mv(s, 0, 0);
3177                     }
3178                     break;
3179                 case CANDIDATE_MB_TYPE_BIDIR:
3180                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
3181                     s->mb_intra= 0;
3182                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
3183                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
3184                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
3185                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
3186                     break;
3187                 case CANDIDATE_MB_TYPE_BACKWARD:
3188                     s->mv_dir = MV_DIR_BACKWARD;
3189                     s->mb_intra= 0;
3190                     motion_x= s->mv[1][0][0] = s->b_back_mv_table[xy][0];
3191                     motion_y= s->mv[1][0][1] = s->b_back_mv_table[xy][1];
3192                     break;
3193                 case CANDIDATE_MB_TYPE_FORWARD:
3194                     s->mv_dir = MV_DIR_FORWARD;
3195                     s->mb_intra= 0;
3196                     motion_x= s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
3197                     motion_y= s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
3198                     break;
3199                 case CANDIDATE_MB_TYPE_FORWARD_I:
3200                     s->mv_dir = MV_DIR_FORWARD;
3201                     s->mv_type = MV_TYPE_FIELD;
3202                     s->mb_intra= 0;
3203                     for(i=0; i<2; i++){
3204                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
3205                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
3206                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
3207                     }
3208                     break;
3209                 case CANDIDATE_MB_TYPE_BACKWARD_I:
3210                     s->mv_dir = MV_DIR_BACKWARD;
3211                     s->mv_type = MV_TYPE_FIELD;
3212                     s->mb_intra= 0;
3213                     for(i=0; i<2; i++){
3214                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
3215                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
3216                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
3217                     }
3218                     break;
3219                 case CANDIDATE_MB_TYPE_BIDIR_I:
3220                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
3221                     s->mv_type = MV_TYPE_FIELD;
3222                     s->mb_intra= 0;
3223                     for(dir=0; dir<2; dir++){
3224                         for(i=0; i<2; i++){
3225                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
3226                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
3227                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
3228                         }
3229                     }
3230                     break;
3231                 default:
3232                     av_log(s->avctx, AV_LOG_ERROR, "illegal MB type\n");
3233                 }
3234
3235                 encode_mb(s, motion_x, motion_y);
3236
3237                 // RAL: Update last macroblock type
3238                 s->last_mv_dir = s->mv_dir;
3239
3240                 if (CONFIG_H263_ENCODER &&
3241                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
3242                     ff_h263_update_motion_val(s);
3243
3244                 ff_mpv_decode_mb(s, s->block);
3245             }
3246
3247             /* clean the MV table in IPS frames for direct mode in B-frames */
3248             if(s->mb_intra /* && I,P,S_TYPE */){
3249                 s->p_mv_table[xy][0]=0;
3250                 s->p_mv_table[xy][1]=0;
3251             }
3252
3253             if (s->avctx->flags & AV_CODEC_FLAG_PSNR) {
3254                 int w= 16;
3255                 int h= 16;
3256
3257                 if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
3258                 if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
3259
3260                 s->current_picture.encoding_error[0] += sse(
3261                     s, s->new_picture.f->data[0] + s->mb_x*16 + s->mb_y*s->linesize*16,
3262                     s->dest[0], w, h, s->linesize);
3263                 s->current_picture.encoding_error[1] += sse(
3264                     s, s->new_picture.f->data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
3265                     s->dest[1], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
3266                 s->current_picture.encoding_error[2] += sse(
3267                     s, s->new_picture.f->data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
3268                     s->dest[2], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
3269             }
3270             if(s->loop_filter){
3271                 if(CONFIG_H263_ENCODER && s->out_format == FMT_H263)
3272                     ff_h263_loop_filter(s);
3273             }
3274             ff_dlog(s->avctx, "MB %d %d bits\n",
3275                     s->mb_x + s->mb_y * s->mb_stride, put_bits_count(&s->pb));
3276         }
3277     }
3278
3279     //not beautiful here but we must write it before flushing so it has to be here
3280     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version && s->msmpeg4_version<4 && s->pict_type == AV_PICTURE_TYPE_I)
3281         ff_msmpeg4_encode_ext_header(s);
3282
3283     write_slice_end(s);
3284
3285 #if FF_API_RTP_CALLBACK
3286 FF_DISABLE_DEPRECATION_WARNINGS
3287     /* Send the last GOB if RTP */
3288     if (s->avctx->rtp_callback) {
3289         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width - s->resync_mb_x;
3290         int pdif = put_bits_ptr(&s->pb) - s->ptr_lastgob;
3291         /* Call the RTP callback to send the last GOB */
3292         emms_c();
3293         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, pdif, number_mb);
3294     }
3295 FF_ENABLE_DEPRECATION_WARNINGS
3296 #endif
3297
3298     return 0;
3299 }
3300
3301 #define MERGE(field) dst->field += src->field; src->field=0
3302 static void merge_context_after_me(MpegEncContext *dst, MpegEncContext *src){
3303     MERGE(me.scene_change_score);
3304     MERGE(me.mc_mb_var_sum_temp);
3305     MERGE(me.mb_var_sum_temp);
3306 }
3307
3308 static void merge_context_after_encode(MpegEncContext *dst, MpegEncContext *src){
3309     int i;
3310
3311     MERGE(dct_count[0]); //note, the other dct vars are not part of the context
3312     MERGE(dct_count[1]);
3313     MERGE(mv_bits);
3314     MERGE(i_tex_bits);
3315     MERGE(p_tex_bits);
3316     MERGE(i_count);
3317     MERGE(f_count);
3318     MERGE(b_count);
3319     MERGE(skip_count);
3320     MERGE(misc_bits);
3321     MERGE(er.error_count);
3322     MERGE(padding_bug_score);
3323     MERGE(current_picture.encoding_error[0]);
3324     MERGE(current_picture.encoding_error[1]);
3325     MERGE(current_picture.encoding_error[2]);
3326
3327     if (dst->noise_reduction){
3328         for(i=0; i<64; i++){
3329             MERGE(dct_error_sum[0][i]);
3330             MERGE(dct_error_sum[1][i]);
3331         }
3332     }
3333
3334     assert(put_bits_count(&src->pb) % 8 ==0);
3335     assert(put_bits_count(&dst->pb) % 8 ==0);
3336     avpriv_copy_bits(&dst->pb, src->pb.buf, put_bits_count(&src->pb));
3337     flush_put_bits(&dst->pb);
3338 }
3339
3340 static int estimate_qp(MpegEncContext *s, int dry_run){
3341     if (s->next_lambda){
3342         s->current_picture_ptr->f->quality =
3343         s->current_picture.f->quality = s->next_lambda;
3344         if(!dry_run) s->next_lambda= 0;
3345     } else if (!s->fixed_qscale) {
3346         int quality;
3347         quality = ff_rate_estimate_qscale(s, dry_run);
3348         s->current_picture_ptr->f->quality =
3349         s->current_picture.f->quality = quality;
3350         if (s->current_picture.f->quality < 0)
3351             return -1;
3352     }
3353
3354     if(s->adaptive_quant){
3355         switch(s->codec_id){
3356         case AV_CODEC_ID_MPEG4:
3357             if (CONFIG_MPEG4_ENCODER)
3358                 ff_clean_mpeg4_qscales(s);
3359             break;
3360         case AV_CODEC_ID_H263:
3361         case AV_CODEC_ID_H263P:
3362         case AV_CODEC_ID_FLV1:
3363             if (CONFIG_H263_ENCODER)
3364                 ff_clean_h263_qscales(s);
3365             break;
3366         default:
3367             ff_init_qscale_tab(s);
3368         }
3369
3370         s->lambda= s->lambda_table[0];
3371         //FIXME broken
3372     }else
3373         s->lambda = s->current_picture.f->quality;
3374     update_qscale(s);
3375     return 0;
3376 }
3377
3378 /* must be called before writing the header */
3379 static void set_frame_distances(MpegEncContext * s){
3380     assert(s->current_picture_ptr->f->pts != AV_NOPTS_VALUE);
3381     s->time = s->current_picture_ptr->f->pts * s->avctx->time_base.num;
3382
3383     if(s->pict_type==AV_PICTURE_TYPE_B){
3384         s->pb_time= s->pp_time - (s->last_non_b_time - s->time);
3385         assert(s->pb_time > 0 && s->pb_time < s->pp_time);
3386     }else{
3387         s->pp_time= s->time - s->last_non_b_time;
3388         s->last_non_b_time= s->time;
3389         assert(s->picture_number==0 || s->pp_time > 0);
3390     }
3391 }
3392
3393 static int encode_picture(MpegEncContext *s, int picture_number)
3394 {
3395     int i, ret;
3396     int bits;
3397     int context_count = s->slice_context_count;
3398
3399     s->picture_number = picture_number;
3400
3401     /* Reset the average MB variance */
3402     s->me.mb_var_sum_temp    =
3403     s->me.mc_mb_var_sum_temp = 0;
3404
3405     /* we need to initialize some time vars before we can encode B-frames */
3406     // RAL: Condition added for MPEG1VIDEO
3407     if (s->codec_id == AV_CODEC_ID_MPEG1VIDEO || s->codec_id == AV_CODEC_ID_MPEG2VIDEO || (s->h263_pred && !s->msmpeg4_version))
3408         set_frame_distances(s);
3409     if(CONFIG_MPEG4_ENCODER && s->codec_id == AV_CODEC_ID_MPEG4)
3410         ff_set_mpeg4_time(s);
3411
3412     s->me.scene_change_score=0;
3413
3414 //    s->lambda= s->current_picture_ptr->quality; //FIXME qscale / ... stuff for ME rate distortion
3415
3416     if(s->pict_type==AV_PICTURE_TYPE_I){
3417         if(s->msmpeg4_version >= 3) s->no_rounding=1;
3418         else                        s->no_rounding=0;
3419     }else if(s->pict_type!=AV_PICTURE_TYPE_B){
3420         if(s->flipflop_rounding || s->codec_id == AV_CODEC_ID_H263P || s->codec_id == AV_CODEC_ID_MPEG4)
3421             s->no_rounding ^= 1;
3422     }
3423
3424     if (s->avctx->flags & AV_CODEC_FLAG_PASS2) {
3425         if (estimate_qp(s,1) < 0)
3426             return -1;
3427         ff_get_2pass_fcode(s);
3428     } else if (!(s->avctx->flags & AV_CODEC_FLAG_QSCALE)) {
3429         if(s->pict_type==AV_PICTURE_TYPE_B)
3430             s->lambda= s->last_lambda_for[s->pict_type];
3431         else
3432             s->lambda= s->last_lambda_for[s->last_non_b_pict_type];
3433         update_qscale(s);
3434     }
3435
3436     s->mb_intra=0; //for the rate distortion & bit compare functions
3437     for(i=1; i<context_count; i++){
3438         ret = ff_update_duplicate_context(s->thread_context[i], s);
3439         if (ret < 0)
3440             return ret;
3441     }
3442
3443     if(ff_init_me(s)<0)
3444         return -1;
3445
3446     /* Estimate motion for every MB */
3447     if(s->pict_type != AV_PICTURE_TYPE_I){
3448         s->lambda  = (s->lambda  * s->me_penalty_compensation + 128) >> 8;
3449         s->lambda2 = (s->lambda2 * (int64_t) s->me_penalty_compensation + 128) >> 8;
3450         if (s->pict_type != AV_PICTURE_TYPE_B) {
3451             if ((s->me_pre && s->last_non_b_pict_type == AV_PICTURE_TYPE_I) ||
3452                 s->me_pre == 2) {
3453                 s->avctx->execute(s->avctx, pre_estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3454             }
3455         }
3456
3457         s->avctx->execute(s->avctx, estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3458     }else /* if(s->pict_type == AV_PICTURE_TYPE_I) */{
3459         /* I-Frame */
3460         for(i=0; i<s->mb_stride*s->mb_height; i++)
3461             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
3462
3463         if(!s->fixed_qscale){
3464             /* finding spatial complexity for I-frame rate control */
3465             s->avctx->execute(s->avctx, mb_var_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3466         }
3467     }
3468     for(i=1; i<context_count; i++){
3469         merge_context_after_me(s, s->thread_context[i]);
3470     }
3471     s->current_picture.mc_mb_var_sum= s->current_picture_ptr->mc_mb_var_sum= s->me.mc_mb_var_sum_temp;
3472     s->current_picture.   mb_var_sum= s->current_picture_ptr->   mb_var_sum= s->me.   mb_var_sum_temp;
3473     emms_c();
3474
3475     if (s->me.scene_change_score > s->scenechange_threshold &&
3476         s->pict_type == AV_PICTURE_TYPE_P) {
3477         s->pict_type= AV_PICTURE_TYPE_I;
3478         for(i=0; i<s->mb_stride*s->mb_height; i++)
3479             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
3480         ff_dlog(s, "Scene change detected, encoding as I Frame %d %d\n",
3481                 s->current_picture.mb_var_sum, s->current_picture.mc_mb_var_sum);
3482     }
3483
3484     if(!s->umvplus){
3485         if(s->pict_type==AV_PICTURE_TYPE_P || s->pict_type==AV_PICTURE_TYPE_S) {
3486             s->f_code= ff_get_best_fcode(s, s->p_mv_table, CANDIDATE_MB_TYPE_INTER);
3487
3488             if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_ME) {
3489                 int a,b;
3490                 a= ff_get_best_fcode(s, s->p_field_mv_table[0][0], CANDIDATE_MB_TYPE_INTER_I); //FIXME field_select
3491                 b= ff_get_best_fcode(s, s->p_field_mv_table[1][1], CANDIDATE_MB_TYPE_INTER_I);
3492                 s->f_code= FFMAX3(s->f_code, a, b);
3493             }
3494
3495             ff_fix_long_p_mvs(s);
3496             ff_fix_long_mvs(s, NULL, 0, s->p_mv_table, s->f_code, CANDIDATE_MB_TYPE_INTER, 0);
3497             if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_ME) {
3498                 int j;
3499                 for(i=0; i<2; i++){
3500                     for(j=0; j<2; j++)
3501                         ff_fix_long_mvs(s, s->p_field_select_table[i], j,
3502                                         s->p_field_mv_table[i][j], s->f_code, CANDIDATE_MB_TYPE_INTER_I, 0);
3503                 }
3504             }
3505         }
3506
3507         if(s->pict_type==AV_PICTURE_TYPE_B){
3508             int a, b;
3509
3510             a = ff_get_best_fcode(s, s->b_forw_mv_table, CANDIDATE_MB_TYPE_FORWARD);
3511             b = ff_get_best_fcode(s, s->b_bidir_forw_mv_table, CANDIDATE_MB_TYPE_BIDIR);
3512             s->f_code = FFMAX(a, b);
3513
3514             a = ff_get_best_fcode(s, s->b_back_mv_table, CANDIDATE_MB_TYPE_BACKWARD);
3515             b = ff_get_best_fcode(s, s->b_bidir_back_mv_table, CANDIDATE_MB_TYPE_BIDIR);
3516             s->b_code = FFMAX(a, b);
3517
3518             ff_fix_long_mvs(s, NULL, 0, s->b_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_FORWARD, 1);
3519             ff_fix_long_mvs(s, NULL, 0, s->b_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BACKWARD, 1);
3520             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_BIDIR, 1);
3521             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BIDIR, 1);
3522             if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_ME) {
3523                 int dir, j;
3524                 for(dir=0; dir<2; dir++){
3525                     for(i=0; i<2; i++){
3526                         for(j=0; j<2; j++){
3527                             int type= dir ? (CANDIDATE_MB_TYPE_BACKWARD_I|CANDIDATE_MB_TYPE_BIDIR_I)
3528                                           : (CANDIDATE_MB_TYPE_FORWARD_I |CANDIDATE_MB_TYPE_BIDIR_I);
3529                             ff_fix_long_mvs(s, s->b_field_select_table[dir][i], j,
3530                                             s->b_field_mv_table[dir][i][j], dir ? s->b_code : s->f_code, type, 1);
3531                         }
3532                     }
3533                 }
3534             }
3535         }
3536     }
3537
3538     if (estimate_qp(s, 0) < 0)
3539         return -1;
3540
3541     if (s->qscale < 3 && s->max_qcoeff <= 128 &&
3542         s->pict_type == AV_PICTURE_TYPE_I &&
3543         !(s->avctx->flags & AV_CODEC_FLAG_QSCALE))
3544         s->qscale= 3; //reduce clipping problems
3545
3546     if (s->out_format == FMT_MJPEG) {
3547         /* for mjpeg, we do include qscale in the matrix */
3548         for(i=1;i<64;i++){
3549             int j = s->idsp.idct_permutation[i];
3550
3551             s->intra_matrix[j] = av_clip_uint8((ff_mpeg1_default_intra_matrix[i] * s->qscale) >> 3);
3552         }
3553         s->y_dc_scale_table=
3554         s->c_dc_scale_table= ff_mpeg2_dc_scale_table[s->intra_dc_precision];
3555         s->intra_matrix[0] = ff_mpeg2_dc_scale_table[s->intra_dc_precision][8];
3556         ff_convert_matrix(s, s->q_intra_matrix, s->q_intra_matrix16,
3557                        s->intra_matrix, s->intra_quant_bias, 8, 8, 1);
3558         s->qscale= 8;
3559     }
3560
3561     //FIXME var duplication
3562     s->current_picture_ptr->f->key_frame =
3563     s->current_picture.f->key_frame = s->pict_type == AV_PICTURE_TYPE_I; //FIXME pic_ptr
3564     s->current_picture_ptr->f->pict_type =
3565     s->current_picture.f->pict_type = s->pict_type;
3566
3567     if (s->current_picture.f->key_frame)
3568         s->picture_in_gop_number=0;
3569
3570     s->last_bits= put_bits_count(&s->pb);
3571     switch(s->out_format) {
3572     case FMT_MJPEG:
3573         if (CONFIG_MJPEG_ENCODER)
3574             ff_mjpeg_encode_picture_header(s->avctx, &s->pb, &s->intra_scantable,
3575                                            s->pred, s->intra_matrix);
3576         break;
3577     case FMT_H261:
3578         if (CONFIG_H261_ENCODER)
3579             ff_h261_encode_picture_header(s, picture_number);
3580         break;
3581     case FMT_H263:
3582         if (CONFIG_WMV2_ENCODER && s->codec_id == AV_CODEC_ID_WMV2)
3583             ff_wmv2_encode_picture_header(s, picture_number);
3584         else if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
3585             ff_msmpeg4_encode_picture_header(s, picture_number);
3586         else if (CONFIG_MPEG4_ENCODER && s->h263_pred)
3587             ff_mpeg4_encode_picture_header(s, picture_number);
3588         else if (CONFIG_RV10_ENCODER && s->codec_id == AV_CODEC_ID_RV10) {
3589             ret = ff_rv10_encode_picture_header(s, picture_number);
3590             if (ret < 0)
3591                 return ret;
3592         }
3593         else if (CONFIG_RV20_ENCODER && s->codec_id == AV_CODEC_ID_RV20)
3594             ff_rv20_encode_picture_header(s, picture_number);
3595         else if (CONFIG_FLV_ENCODER && s->codec_id == AV_CODEC_ID_FLV1)
3596             ff_flv_encode_picture_header(s, picture_number);
3597         else if (CONFIG_H263_ENCODER)
3598             ff_h263_encode_picture_header(s, picture_number);
3599         break;
3600     case FMT_MPEG1:
3601         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
3602             ff_mpeg1_encode_picture_header(s, picture_number);
3603         break;
3604     default:
3605         assert(0);
3606     }
3607     bits= put_bits_count(&s->pb);
3608     s->header_bits= bits - s->last_bits;
3609
3610     for(i=1; i<context_count; i++){
3611         update_duplicate_context_after_me(s->thread_context[i], s);
3612     }
3613     s->avctx->execute(s->avctx, encode_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3614     for(i=1; i<context_count; i++){
3615         merge_context_after_encode(s, s->thread_context[i]);
3616     }
3617     emms_c();
3618     return 0;
3619 }
3620
3621 static void denoise_dct_c(MpegEncContext *s, int16_t *block){
3622     const int intra= s->mb_intra;
3623     int i;
3624
3625     s->dct_count[intra]++;
3626
3627     for(i=0; i<64; i++){
3628         int level= block[i];
3629
3630         if(level){
3631             if(level>0){
3632                 s->dct_error_sum[intra][i] += level;
3633                 level -= s->dct_offset[intra][i];
3634                 if(level<0) level=0;
3635             }else{
3636                 s->dct_error_sum[intra][i] -= level;
3637                 level += s->dct_offset[intra][i];
3638                 if(level>0) level=0;
3639             }
3640             block[i]= level;
3641         }
3642     }
3643 }
3644
3645 static int dct_quantize_trellis_c(MpegEncContext *s,
3646                                   int16_t *block, int n,
3647                                   int qscale, int *overflow){
3648     const int *qmat;
3649     const uint8_t *scantable= s->intra_scantable.scantable;
3650     const uint8_t *perm_scantable= s->intra_scantable.permutated;
3651     int max=0;
3652     unsigned int threshold1, threshold2;
3653     int bias=0;
3654     int run_tab[65];
3655     int level_tab[65];
3656     int score_tab[65];
3657     int survivor[65];
3658     int survivor_count;
3659     int last_run=0;
3660     int last_level=0;
3661     int last_score= 0;
3662     int last_i;
3663     int coeff[2][64];
3664     int coeff_count[64];
3665     int qmul, qadd, start_i, last_non_zero, i, dc;
3666     const int esc_length= s->ac_esc_length;
3667     uint8_t * length;
3668     uint8_t * last_length;
3669     const int lambda= s->lambda2 >> (FF_LAMBDA_SHIFT - 6);
3670
3671     s->fdsp.fdct(block);
3672
3673     if(s->dct_error_sum)
3674         s->denoise_dct(s, block);
3675     qmul= qscale*16;
3676     qadd= ((qscale-1)|1)*8;
3677
3678     if (s->mb_intra) {
3679         int q;
3680         if (!s->h263_aic) {
3681             if (n < 4)
3682                 q = s->y_dc_scale;
3683             else
3684                 q = s->c_dc_scale;
3685             q = q << 3;
3686         } else{
3687             /* For AIC we skip quant/dequant of INTRADC */
3688             q = 1 << 3;
3689             qadd=0;
3690         }
3691
3692         /* note: block[0] is assumed to be positive */
3693         block[0] = (block[0] + (q >> 1)) / q;
3694         start_i = 1;
3695         last_non_zero = 0;
3696         qmat = s->q_intra_matrix[qscale];
3697         if(s->mpeg_quant || s->out_format == FMT_MPEG1)
3698             bias= 1<<(QMAT_SHIFT-1);
3699         length     = s->intra_ac_vlc_length;
3700         last_length= s->intra_ac_vlc_last_length;
3701     } else {
3702         start_i = 0;
3703         last_non_zero = -1;
3704         qmat = s->q_inter_matrix[qscale];
3705         length     = s->inter_ac_vlc_length;
3706         last_length= s->inter_ac_vlc_last_length;
3707     }
3708     last_i= start_i;
3709
3710     threshold1= (1<<QMAT_SHIFT) - bias - 1;
3711     threshold2= (threshold1<<1);
3712
3713     for(i=63; i>=start_i; i--) {
3714         const int j = scantable[i];
3715         int level = block[j] * qmat[j];
3716
3717         if(((unsigned)(level+threshold1))>threshold2){
3718             last_non_zero = i;
3719             break;
3720         }
3721     }
3722
3723     for(i=start_i; i<=last_non_zero; i++) {
3724         const int j = scantable[i];
3725         int level = block[j] * qmat[j];
3726
3727 //        if(   bias+level >= (1<<(QMAT_SHIFT - 3))
3728 //           || bias-level >= (1<<(QMAT_SHIFT - 3))){
3729         if(((unsigned)(level+threshold1))>threshold2){
3730             if(level>0){
3731                 level= (bias + level)>>QMAT_SHIFT;
3732                 coeff[0][i]= level;
3733                 coeff[1][i]= level-1;
3734 //                coeff[2][k]= level-2;
3735             }else{
3736                 level= (bias - level)>>QMAT_SHIFT;
3737                 coeff[0][i]= -level;
3738                 coeff[1][i]= -level+1;
3739 //                coeff[2][k]= -level+2;
3740             }
3741             coeff_count[i]= FFMIN(level, 2);
3742             assert(coeff_count[i]);
3743             max |=level;
3744         }else{
3745             coeff[0][i]= (level>>31)|1;
3746             coeff_count[i]= 1;
3747         }
3748     }
3749
3750     *overflow= s->max_qcoeff < max; //overflow might have happened
3751
3752     if(last_non_zero < start_i){
3753         memset(block + start_i, 0, (64-start_i)*sizeof(int16_t));
3754         return last_non_zero;
3755     }
3756
3757     score_tab[start_i]= 0;
3758     survivor[0]= start_i;
3759     survivor_count= 1;
3760
3761     for(i=start_i; i<=last_non_zero; i++){
3762         int level_index, j, zero_distortion;
3763         int dct_coeff= FFABS(block[ scantable[i] ]);
3764         int best_score=256*256*256*120;
3765
3766         if (s->fdsp.fdct == ff_fdct_ifast)
3767             dct_coeff= (dct_coeff*ff_inv_aanscales[ scantable[i] ]) >> 12;
3768         zero_distortion= dct_coeff*dct_coeff;
3769
3770         for(level_index=0; level_index < coeff_count[i]; level_index++){
3771             int distortion;
3772             int level= coeff[level_index][i];
3773             const int alevel= FFABS(level);
3774             int unquant_coeff;
3775
3776             assert(level);
3777
3778             if(s->out_format == FMT_H263){
3779                 unquant_coeff= alevel*qmul + qadd;
3780             } else { // MPEG-1
3781                 j = s->idsp.idct_permutation[scantable[i]]; // FIXME: optimize
3782                 if(s->mb_intra){
3783                         unquant_coeff = (int)(  alevel  * qscale * s->intra_matrix[j]) >> 3;
3784                         unquant_coeff =   (unquant_coeff - 1) | 1;
3785                 }else{
3786                         unquant_coeff = (((  alevel  << 1) + 1) * qscale * ((int) s->inter_matrix[j])) >> 4;
3787                         unquant_coeff =   (unquant_coeff - 1) | 1;
3788                 }
3789                 unquant_coeff<<= 3;
3790             }
3791
3792             distortion= (unquant_coeff - dct_coeff) * (unquant_coeff - dct_coeff) - zero_distortion;
3793             level+=64;
3794             if((level&(~127)) == 0){
3795                 for(j=survivor_count-1; j>=0; j--){
3796                     int run= i - survivor[j];
3797                     int score= distortion + length[UNI_AC_ENC_INDEX(run, level)]*lambda;
3798                     score += score_tab[i-run];
3799
3800                     if(score < best_score){
3801                         best_score= score;
3802                         run_tab[i+1]= run;
3803                         level_tab[i+1]= level-64;
3804                     }
3805                 }
3806
3807                 if(s->out_format == FMT_H263){
3808                     for(j=survivor_count-1; j>=0; j--){
3809                         int run= i - survivor[j];
3810                         int score= distortion + last_length[UNI_AC_ENC_INDEX(run, level)]*lambda;
3811                         score += score_tab[i-run];
3812                         if(score < last_score){
3813                             last_score= score;
3814                             last_run= run;
3815                             last_level= level-64;
3816                             last_i= i+1;
3817                         }
3818                     }
3819                 }
3820             }else{
3821                 distortion += esc_length*lambda;
3822                 for(j=survivor_count-1; j>=0; j--){
3823                     int run= i - survivor[j];
3824                     int score= distortion + score_tab[i-run];
3825
3826                     if(score < best_score){
3827                         best_score= score;
3828                         run_tab[i+1]= run;
3829                         level_tab[i+1]= level-64;
3830                     }
3831                 }
3832
3833                 if(s->out_format == FMT_H263){
3834                   for(j=survivor_count-1; j>=0; j--){
3835                         int run= i - survivor[j];
3836                         int score= distortion + score_tab[i-run];
3837                         if(score < last_score){
3838                             last_score= score;
3839                             last_run= run;
3840                             last_level= level-64;
3841                             last_i= i+1;
3842                         }
3843                     }
3844                 }
3845             }
3846         }
3847
3848         score_tab[i+1]= best_score;
3849
3850         // Note: there is a vlc code in MPEG-4 which is 1 bit shorter then another one with a shorter run and the same level
3851         if(last_non_zero <= 27){
3852             for(; survivor_count; survivor_count--){
3853                 if(score_tab[ survivor[survivor_count-1] ] <= best_score)
3854                     break;
3855             }
3856         }else{
3857             for(; survivor_count; survivor_count--){
3858                 if(score_tab[ survivor[survivor_count-1] ] <= best_score + lambda)
3859                     break;
3860             }
3861         }
3862
3863         survivor[ survivor_count++ ]= i+1;
3864     }
3865
3866     if(s->out_format != FMT_H263){
3867         last_score= 256*256*256*120;
3868         for(i= survivor[0]; i<=last_non_zero + 1; i++){
3869             int score= score_tab[i];
3870             if (i)
3871                 score += lambda * 2; // FIXME more exact?
3872
3873             if(score < last_score){
3874                 last_score= score;
3875                 last_i= i;
3876                 last_level= level_tab[i];
3877                 last_run= run_tab[i];
3878             }
3879         }
3880     }
3881
3882     s->coded_score[n] = last_score;
3883
3884     dc= FFABS(block[0]);
3885     last_non_zero= last_i - 1;
3886     memset(block + start_i, 0, (64-start_i)*sizeof(int16_t));
3887
3888     if(last_non_zero < start_i)
3889         return last_non_zero;
3890
3891     if(last_non_zero == 0 && start_i == 0){
3892         int best_level= 0;
3893         int best_score= dc * dc;
3894
3895         for(i=0; i<coeff_count[0]; i++){
3896             int level= coeff[i][0];
3897             int alevel= FFABS(level);
3898             int unquant_coeff, score, distortion;
3899
3900             if(s->out_format == FMT_H263){
3901                     unquant_coeff= (alevel*qmul + qadd)>>3;
3902             } else { // MPEG-1
3903                     unquant_coeff = (((  alevel  << 1) + 1) * qscale * ((int) s->inter_matrix[0])) >> 4;
3904                     unquant_coeff =   (unquant_coeff - 1) | 1;
3905             }
3906             unquant_coeff = (unquant_coeff + 4) >> 3;
3907             unquant_coeff<<= 3 + 3;
3908
3909             distortion= (unquant_coeff - dc) * (unquant_coeff - dc);
3910             level+=64;
3911             if((level&(~127)) == 0) score= distortion + last_length[UNI_AC_ENC_INDEX(0, level)]*lambda;
3912             else                    score= distortion + esc_length*lambda;
3913
3914             if(score < best_score){
3915                 best_score= score;
3916                 best_level= level - 64;
3917             }
3918         }
3919         block[0]= best_level;
3920         s->coded_score[n] = best_score - dc*dc;
3921         if(best_level == 0) return -1;
3922         else                return last_non_zero;
3923     }
3924
3925     i= last_i;
3926     assert(last_level);
3927
3928     block[ perm_scantable[last_non_zero] ]= last_level;
3929     i -= last_run + 1;
3930
3931     for(; i>start_i; i -= run_tab[i] + 1){
3932         block[ perm_scantable[i-1] ]= level_tab[i];
3933     }
3934
3935     return last_non_zero;
3936 }
3937
3938 //#define REFINE_STATS 1
3939 static int16_t basis[64][64];
3940
3941 static void build_basis(uint8_t *perm){
3942     int i, j, x, y;
3943     emms_c();
3944     for(i=0; i<8; i++){
3945         for(j=0; j<8; j++){
3946             for(y=0; y<8; y++){
3947                 for(x=0; x<8; x++){
3948                     double s= 0.25*(1<<BASIS_SHIFT);
3949                     int index= 8*i + j;
3950                     int perm_index= perm[index];
3951                     if(i==0) s*= sqrt(0.5);
3952                     if(j==0) s*= sqrt(0.5);
3953                     basis[perm_index][8*x + y]= lrintf(s * cos((M_PI/8.0)*i*(x+0.5)) * cos((M_PI/8.0)*j*(y+0.5)));
3954                 }
3955             }
3956         }
3957     }
3958 }
3959
3960 static int dct_quantize_refine(MpegEncContext *s, //FIXME breaks denoise?
3961                         int16_t *block, int16_t *weight, int16_t *orig,
3962                         int n, int qscale){
3963     int16_t rem[64];
3964     LOCAL_ALIGNED_16(int16_t, d1, [64]);
3965     const uint8_t *scantable= s->intra_scantable.scantable;
3966     const uint8_t *perm_scantable= s->intra_scantable.permutated;
3967 //    unsigned int threshold1, threshold2;
3968 //    int bias=0;
3969     int run_tab[65];
3970     int prev_run=0;
3971     int prev_level=0;
3972     int qmul, qadd, start_i, last_non_zero, i, dc;
3973     uint8_t * length;
3974     uint8_t * last_length;
3975     int lambda;
3976     int rle_index, run, q = 1, sum; //q is only used when s->mb_intra is true
3977 #ifdef REFINE_STATS
3978 static int count=0;
3979 static int after_last=0;
3980 static int to_zero=0;
3981 static int from_zero=0;
3982 static int raise=0;
3983 static int lower=0;
3984 static int messed_sign=0;
3985 #endif
3986
3987     if(basis[0][0] == 0)
3988         build_basis(s->idsp.idct_permutation);
3989
3990     qmul= qscale*2;
3991     qadd= (qscale-1)|1;
3992     if (s->mb_intra) {
3993         if (!s->h263_aic) {
3994             if (n < 4)
3995                 q = s->y_dc_scale;
3996             else
3997                 q = s->c_dc_scale;
3998         } else{
3999             /* For AIC we skip quant/dequant of INTRADC */
4000             q = 1;
4001             qadd=0;
4002         }
4003         q <<= RECON_SHIFT-3;
4004         /* note: block[0] is assumed to be positive */
4005         dc= block[0]*q;
4006 //        block[0] = (block[0] + (q >> 1)) / q;
4007         start_i = 1;
4008 //        if(s->mpeg_quant || s->out_format == FMT_MPEG1)
4009 //            bias= 1<<(QMAT_SHIFT-1);
4010         length     = s->intra_ac_vlc_length;
4011         last_length= s->intra_ac_vlc_last_length;
4012     } else {
4013         dc= 0;
4014         start_i = 0;
4015         length     = s->inter_ac_vlc_length;
4016         last_length= s->inter_ac_vlc_last_length;
4017     }
4018     last_non_zero = s->block_last_index[n];
4019
4020 #ifdef REFINE_STATS
4021 {START_TIMER
4022 #endif
4023     dc += (1<<(RECON_SHIFT-1));
4024     for(i=0; i<64; i++){
4025         rem[i] = dc - (orig[i] << RECON_SHIFT); // FIXME use orig directly instead of copying to rem[]
4026     }
4027 #ifdef REFINE_STATS
4028 STOP_TIMER("memset rem[]")}
4029 #endif
4030     sum=0;
4031     for(i=0; i<64; i++){
4032         int one= 36;
4033         int qns=4;
4034         int w;
4035
4036         w= FFABS(weight[i]) + qns*one;
4037         w= 15 + (48*qns*one + w/2)/w; // 16 .. 63
4038
4039         weight[i] = w;
4040 //        w=weight[i] = (63*qns + (w/2)) / w;
4041
4042         assert(w>0);
4043         assert(w<(1<<6));
4044         sum += w*w;
4045     }
4046     lambda= sum*(uint64_t)s->lambda2 >> (FF_LAMBDA_SHIFT - 6 + 6 + 6 + 6);
4047 #ifdef REFINE_STATS
4048 {START_TIMER
4049 #endif
4050     run=0;
4051     rle_index=0;
4052     for(i=start_i; i<=last_non_zero; i++){
4053         int j= perm_scantable[i];
4054         const int level= block[j];
4055         int coeff;
4056
4057         if(level){
4058             if(level<0) coeff= qmul*level - qadd;
4059             else        coeff= qmul*level + qadd;
4060             run_tab[rle_index++]=run;
4061             run=0;
4062
4063             s->mpvencdsp.add_8x8basis(rem, basis[j], coeff);
4064         }else{
4065             run++;
4066         }
4067     }
4068 #ifdef REFINE_STATS
4069 if(last_non_zero>0){
4070 STOP_TIMER("init rem[]")
4071 }
4072 }
4073
4074 {START_TIMER
4075 #endif
4076     for(;;){
4077         int best_score = s->mpvencdsp.try_8x8basis(rem, weight, basis[0], 0);
4078         int best_coeff=0;
4079         int best_change=0;
4080         int run2, best_unquant_change=0, analyze_gradient;
4081 #ifdef REFINE_STATS
4082 {START_TIMER
4083 #endif
4084         analyze_gradient = last_non_zero > 2 || s->quantizer_noise_shaping >= 3;
4085
4086         if(analyze_gradient){
4087 #ifdef REFINE_STATS
4088 {START_TIMER
4089 #endif
4090             for(i=0; i<64; i++){
4091                 int w= weight[i];
4092
4093                 d1[i] = (rem[i]*w*w + (1<<(RECON_SHIFT+12-1)))>>(RECON_SHIFT+12);
4094             }
4095 #ifdef REFINE_STATS
4096 STOP_TIMER("rem*w*w")}
4097 {START_TIMER
4098 #endif
4099             s->fdsp.fdct(d1);
4100 #ifdef REFINE_STATS
4101 STOP_TIMER("dct")}
4102 #endif
4103         }
4104
4105         if(start_i){
4106             const int level= block[0];
4107             int change, old_coeff;
4108
4109             assert(s->mb_intra);
4110
4111             old_coeff= q*level;
4112
4113             for(change=-1; change<=1; change+=2){
4114                 int new_level= level + change;
4115                 int score, new_coeff;
4116
4117                 new_coeff= q*new_level;
4118                 if(new_coeff >= 2048 || new_coeff < 0)
4119                     continue;
4120
4121                 score = s->mpvencdsp.try_8x8basis(rem, weight, basis[0],
4122                                                   new_coeff - old_coeff);
4123                 if(score<best_score){
4124                     best_score= score;
4125                     best_coeff= 0;
4126                     best_change= change;
4127                     best_unquant_change= new_coeff - old_coeff;
4128                 }
4129             }
4130         }
4131
4132         run=0;
4133         rle_index=0;
4134         run2= run_tab[rle_index++];
4135         prev_level=0;
4136         prev_run=0;
4137
4138         for(i=start_i; i<64; i++){
4139             int j= perm_scantable[i];
4140             const int level= block[j];
4141             int change, old_coeff;
4142
4143             if(s->quantizer_noise_shaping < 3 && i > last_non_zero + 1)
4144                 break;
4145
4146             if(level){
4147                 if(level<0) old_coeff= qmul*level - qadd;
4148                 else        old_coeff= qmul*level + qadd;
4149                 run2= run_tab[rle_index++]; //FIXME ! maybe after last
4150             }else{
4151                 old_coeff=0;
4152                 run2--;
4153                 assert(run2>=0 || i >= last_non_zero );
4154             }
4155
4156             for(change=-1; change<=1; change+=2){
4157                 int new_level= level + change;
4158                 int score, new_coeff, unquant_change;
4159
4160                 score=0;
4161                 if(s->quantizer_noise_shaping < 2 && FFABS(new_level) > FFABS(level))
4162                    continue;
4163
4164                 if(new_level){
4165                     if(new_level<0) new_coeff= qmul*new_level - qadd;
4166                     else            new_coeff= qmul*new_level + qadd;
4167                     if(new_coeff >= 2048 || new_coeff <= -2048)
4168                         continue;
4169                     //FIXME check for overflow
4170
4171                     if(level){
4172                         if(level < 63 && level > -63){
4173                             if(i < last_non_zero)
4174                                 score +=   length[UNI_AC_ENC_INDEX(run, new_level+64)]
4175                                          - length[UNI_AC_ENC_INDEX(run, level+64)];
4176                             else
4177                                 score +=   last_length[UNI_AC_ENC_INDEX(run, new_level+64)]
4178                                          - last_length[UNI_AC_ENC_INDEX(run, level+64)];
4179                         }
4180                     }else{
4181                         assert(FFABS(new_level)==1);
4182
4183                         if(analyze_gradient){
4184                             int g= d1[ scantable[i] ];
4185                             if(g && (g^new_level) >= 0)
4186                                 continue;
4187                         }
4188
4189                         if(i < last_non_zero){
4190                             int next_i= i + run2 + 1;
4191                             int next_level= block[ perm_scantable[next_i] ] + 64;
4192
4193                             if(next_level&(~127))
4194                                 next_level= 0;
4195
4196                             if(next_i < last_non_zero)
4197                                 score +=   length[UNI_AC_ENC_INDEX(run, 65)]
4198                                          + length[UNI_AC_ENC_INDEX(run2, next_level)]
4199                                          - length[UNI_AC_ENC_INDEX(run + run2 + 1, next_level)];
4200                             else
4201                                 score +=  length[UNI_AC_ENC_INDEX(run, 65)]
4202                                         + last_length[UNI_AC_ENC_INDEX(run2, next_level)]
4203                                         - last_length[UNI_AC_ENC_INDEX(run + run2 + 1, next_level)];
4204                         }else{
4205                             score += last_length[UNI_AC_ENC_INDEX(run, 65)];
4206                             if(prev_level){
4207                                 score +=  length[UNI_AC_ENC_INDEX(prev_run, prev_level)]
4208                                         - last_length[UNI_AC_ENC_INDEX(prev_run, prev_level)];
4209                             }
4210                         }
4211                     }
4212                 }else{
4213                     new_coeff=0;
4214                     assert(FFABS(level)==1);
4215
4216                     if(i < last_non_zero){
4217                         int next_i= i + run2 + 1;
4218                         int next_level= block[ perm_scantable[next_i] ] + 64;
4219
4220                         if(next_level&(~127))
4221                             next_level= 0;
4222
4223                         if(next_i < last_non_zero)
4224                             score +=   length[UNI_AC_ENC_INDEX(run + run2 + 1, next_level)]
4225                                      - length[UNI_AC_ENC_INDEX(run2, next_level)]
4226                                      - length[UNI_AC_ENC_INDEX(run, 65)];
4227                         else
4228                             score +=   last_length[UNI_AC_ENC_INDEX(run + run2 + 1, next_level)]
4229                                      - last_length[UNI_AC_ENC_INDEX(run2, next_level)]
4230                                      - length[UNI_AC_ENC_INDEX(run, 65)];
4231                     }else{
4232                         score += -last_length[UNI_AC_ENC_INDEX(run, 65)];
4233                         if(prev_level){
4234                             score +=  last_length[UNI_AC_ENC_INDEX(prev_run, prev_level)]
4235                                     - length[UNI_AC_ENC_INDEX(prev_run, prev_level)];
4236                         }
4237                     }
4238                 }
4239
4240                 score *= lambda;
4241
4242                 unquant_change= new_coeff - old_coeff;
4243                 assert((score < 100*lambda && score > -100*lambda) || lambda==0);
4244
4245                 score += s->mpvencdsp.try_8x8basis(rem, weight, basis[j],
4246                                                    unquant_change);
4247                 if(score<best_score){
4248                     best_score= score;
4249                     best_coeff= i;
4250                     best_change= change;
4251                     best_unquant_change= unquant_change;
4252                 }
4253             }
4254             if(level){
4255                 prev_level= level + 64;
4256                 if(prev_level&(~127))
4257                     prev_level= 0;
4258                 prev_run= run;
4259                 run=0;
4260             }else{
4261                 run++;
4262             }
4263         }
4264 #ifdef REFINE_STATS
4265 STOP_TIMER("iterative step")}
4266 #endif
4267
4268         if(best_change){
4269             int j= perm_scantable[ best_coeff ];
4270
4271             block[j] += best_change;
4272
4273             if(best_coeff > last_non_zero){
4274                 last_non_zero= best_coeff;
4275                 assert(block[j]);
4276 #ifdef REFINE_STATS
4277 after_last++;
4278 #endif
4279             }else{
4280 #ifdef REFINE_STATS
4281 if(block[j]){
4282     if(block[j] - best_change){
4283         if(FFABS(block[j]) > FFABS(block[j] - best_change)){
4284             raise++;
4285         }else{
4286             lower++;
4287         }
4288     }else{
4289         from_zero++;
4290     }
4291 }else{
4292     to_zero++;
4293 }
4294 #endif
4295                 for(; last_non_zero>=start_i; last_non_zero--){
4296                     if(block[perm_scantable[last_non_zero]])
4297                         break;
4298                 }
4299             }
4300 #ifdef REFINE_STATS
4301 count++;
4302 if(256*256*256*64 % count == 0){
4303     printf("after_last:%d to_zero:%d from_zero:%d raise:%d lower:%d sign:%d xyp:%d/%d/%d\n", after_last, to_zero, from_zero, raise, lower, messed_sign, s->mb_x, s->mb_y, s->picture_number);
4304 }
4305 #endif
4306             run=0;
4307             rle_index=0;
4308             for(i=start_i; i<=last_non_zero; i++){
4309                 int j= perm_scantable[i];
4310                 const int level= block[j];
4311
4312                  if(level){
4313                      run_tab[rle_index++]=run;
4314                      run=0;
4315                  }else{
4316                      run++;
4317                  }
4318             }
4319
4320             s->mpvencdsp.add_8x8basis(rem, basis[j], best_unquant_change);
4321         }else{
4322             break;
4323         }
4324     }
4325 #ifdef REFINE_STATS
4326 if(last_non_zero>0){
4327 STOP_TIMER("iterative search")
4328 }
4329 }
4330 #endif
4331
4332     return last_non_zero;
4333 }
4334
4335 /**
4336  * Permute an 8x8 block according to permutation.
4337  * @param block the block which will be permuted according to
4338  *              the given permutation vector
4339  * @param permutation the permutation vector
4340  * @param last the last non zero coefficient in scantable order, used to
4341  *             speed the permutation up
4342  * @param scantable the used scantable, this is only used to speed the
4343  *                  permutation up, the block is not (inverse) permutated
4344  *                  to scantable order!
4345  */
4346 static void block_permute(int16_t *block, uint8_t *permutation,
4347                           const uint8_t *scantable, int last)
4348 {
4349     int i;
4350     int16_t temp[64];
4351
4352     if (last <= 0)
4353         return;
4354     //FIXME it is ok but not clean and might fail for some permutations
4355     // if (permutation[1] == 1)
4356     // return;
4357
4358     for (i = 0; i <= last; i++) {
4359         const int j = scantable[i];
4360         temp[j] = block[j];
4361         block[j] = 0;
4362     }
4363
4364     for (i = 0; i <= last; i++) {
4365         const int j = scantable[i];
4366         const int perm_j = permutation[j];
4367         block[perm_j] = temp[j];
4368     }
4369 }
4370
4371 int ff_dct_quantize_c(MpegEncContext *s,
4372                         int16_t *block, int n,
4373                         int qscale, int *overflow)
4374 {
4375     int i, j, level, last_non_zero, q, start_i;
4376     const int *qmat;
4377     const uint8_t *scantable= s->intra_scantable.scantable;
4378     int bias;
4379     int max=0;
4380     unsigned int threshold1, threshold2;
4381
4382     s->fdsp.fdct(block);
4383
4384     if(s->dct_error_sum)
4385         s->denoise_dct(s, block);
4386
4387     if (s->mb_intra) {
4388         if (!s->h263_aic) {
4389             if (n < 4)
4390                 q = s->y_dc_scale;
4391             else
4392                 q = s->c_dc_scale;
4393             q = q << 3;
4394         } else
4395             /* For AIC we skip quant/dequant of INTRADC */
4396             q = 1 << 3;
4397
4398         /* note: block[0] is assumed to be positive */
4399         block[0] = (block[0] + (q >> 1)) / q;
4400         start_i = 1;
4401         last_non_zero = 0;
4402         qmat = s->q_intra_matrix[qscale];
4403         bias= s->intra_quant_bias<<(QMAT_SHIFT - QUANT_BIAS_SHIFT);
4404     } else {
4405         start_i = 0;
4406         last_non_zero = -1;
4407         qmat = s->q_inter_matrix[qscale];
4408         bias= s->inter_quant_bias<<(QMAT_SHIFT - QUANT_BIAS_SHIFT);
4409     }
4410     threshold1= (1<<QMAT_SHIFT) - bias - 1;
4411     threshold2= (threshold1<<1);
4412     for(i=63;i>=start_i;i--) {
4413         j = scantable[i];
4414         level = block[j] * qmat[j];
4415
4416         if(((unsigned)(level+threshold1))>threshold2){
4417             last_non_zero = i;
4418             break;
4419         }else{
4420             block[j]=0;
4421         }
4422     }
4423     for(i=start_i; i<=last_non_zero; i++) {
4424         j = scantable[i];
4425         level = block[j] * qmat[j];
4426
4427 //        if(   bias+level >= (1<<QMAT_SHIFT)
4428 //           || bias-level >= (1<<QMAT_SHIFT)){
4429         if(((unsigned)(level+threshold1))>threshold2){
4430             if(level>0){
4431                 level= (bias + level)>>QMAT_SHIFT;
4432                 block[j]= level;
4433             }else{
4434                 level= (bias - level)>>QMAT_SHIFT;
4435                 block[j]= -level;
4436             }
4437             max |=level;
4438         }else{
4439             block[j]=0;
4440         }
4441     }
4442     *overflow= s->max_qcoeff < max; //overflow might have happened
4443
4444     /* we need this permutation so that we correct the IDCT, we only permute the !=0 elements */
4445     if (s->idsp.perm_type != FF_IDCT_PERM_NONE)
4446         block_permute(block, s->idsp.idct_permutation,
4447                       scantable, last_non_zero);
4448
4449     return last_non_zero;
4450 }
4451
4452 #define OFFSET(x) offsetof(MpegEncContext, x)
4453 #define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM
4454 static const AVOption h263_options[] = {
4455     { "obmc",         "use overlapped block motion compensation.", OFFSET(obmc), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, VE },
4456     { "structured_slices","Write slice start position at every GOB header instead of just GOB number.", OFFSET(h263_slice_structured), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, VE},
4457     { "mb_info",      "emit macroblock info for RFC 2190 packetization, the parameter value is the maximum payload size", OFFSET(mb_info), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, VE },
4458     FF_MPV_COMMON_OPTS
4459     { NULL },
4460 };
4461
4462 static const AVClass h263_class = {
4463     .class_name = "H.263 encoder",
4464     .item_name  = av_default_item_name,
4465     .option     = h263_options,
4466     .version    = LIBAVUTIL_VERSION_INT,
4467 };
4468
4469 AVCodec ff_h263_encoder = {
4470     .name           = "h263",
4471     .long_name      = NULL_IF_CONFIG_SMALL("H.263 / H.263-1996"),
4472     .type           = AVMEDIA_TYPE_VIDEO,
4473     .id             = AV_CODEC_ID_H263,
4474     .priv_data_size = sizeof(MpegEncContext),
4475     .init           = ff_mpv_encode_init,
4476     .encode2        = ff_mpv_encode_picture,
4477     .close          = ff_mpv_encode_end,
4478     .pix_fmts= (const enum AVPixelFormat[]){AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE},
4479     .priv_class     = &h263_class,
4480 };
4481
4482 static const AVOption h263p_options[] = {
4483     { "umv",        "Use unlimited motion vectors.",    OFFSET(umvplus), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, VE },
4484     { "aiv",        "Use alternative inter VLC.",       OFFSET(alt_inter_vlc), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, VE },
4485     { "obmc",       "use overlapped block motion compensation.", OFFSET(obmc), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, VE },
4486     { "structured_slices", "Write slice start position at every GOB header instead of just GOB number.", OFFSET(h263_slice_structured), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, VE},
4487     FF_MPV_COMMON_OPTS
4488     { NULL },
4489 };
4490 static const AVClass h263p_class = {
4491     .class_name = "H.263p encoder",
4492     .item_name  = av_default_item_name,
4493     .option     = h263p_options,
4494     .version    = LIBAVUTIL_VERSION_INT,
4495 };
4496
4497 AVCodec ff_h263p_encoder = {
4498     .name           = "h263p",
4499     .long_name      = NULL_IF_CONFIG_SMALL("H.263+ / H.263-1998 / H.263 version 2"),
4500     .type           = AVMEDIA_TYPE_VIDEO,
4501     .id             = AV_CODEC_ID_H263P,
4502     .priv_data_size = sizeof(MpegEncContext),
4503     .init           = ff_mpv_encode_init,
4504     .encode2        = ff_mpv_encode_picture,
4505     .close          = ff_mpv_encode_end,
4506     .capabilities   = AV_CODEC_CAP_SLICE_THREADS,
4507     .pix_fmts       = (const enum AVPixelFormat[]){ AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE },
4508     .priv_class     = &h263p_class,
4509 };
4510
4511 static const AVClass msmpeg4v2_class = {
4512     .class_name = "msmpeg4v2 encoder",
4513     .item_name  = av_default_item_name,
4514     .option     = ff_mpv_generic_options,
4515     .version    = LIBAVUTIL_VERSION_INT,
4516 };
4517
4518 AVCodec ff_msmpeg4v2_encoder = {
4519     .name           = "msmpeg4v2",
4520     .long_name      = NULL_IF_CONFIG_SMALL("MPEG-4 part 2 Microsoft variant version 2"),
4521     .type           = AVMEDIA_TYPE_VIDEO,
4522     .id             = AV_CODEC_ID_MSMPEG4V2,
4523     .priv_data_size = sizeof(MpegEncContext),
4524     .init           = ff_mpv_encode_init,
4525     .encode2        = ff_mpv_encode_picture,
4526     .close          = ff_mpv_encode_end,
4527     .pix_fmts       = (const enum AVPixelFormat[]){ AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE },
4528     .priv_class     = &msmpeg4v2_class,
4529 };
4530
4531 static const AVClass msmpeg4v3_class = {
4532     .class_name = "msmpeg4v3 encoder",
4533     .item_name  = av_default_item_name,
4534     .option     = ff_mpv_generic_options,
4535     .version    = LIBAVUTIL_VERSION_INT,
4536 };
4537
4538 AVCodec ff_msmpeg4v3_encoder = {
4539     .name           = "msmpeg4",
4540     .long_name      = NULL_IF_CONFIG_SMALL("MPEG-4 part 2 Microsoft variant version 3"),
4541     .type           = AVMEDIA_TYPE_VIDEO,
4542     .id             = AV_CODEC_ID_MSMPEG4V3,
4543     .priv_data_size = sizeof(MpegEncContext),
4544     .init           = ff_mpv_encode_init,
4545     .encode2        = ff_mpv_encode_picture,
4546     .close          = ff_mpv_encode_end,
4547     .pix_fmts       = (const enum AVPixelFormat[]){ AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE },
4548     .priv_class     = &msmpeg4v3_class,
4549 };
4550
4551 static const AVClass wmv1_class = {
4552     .class_name = "wmv1 encoder",
4553     .item_name  = av_default_item_name,
4554     .option     = ff_mpv_generic_options,
4555     .version    = LIBAVUTIL_VERSION_INT,
4556 };
4557
4558 AVCodec ff_wmv1_encoder = {
4559     .name           = "wmv1",
4560     .long_name      = NULL_IF_CONFIG_SMALL("Windows Media Video 7"),
4561     .type           = AVMEDIA_TYPE_VIDEO,
4562     .id             = AV_CODEC_ID_WMV1,
4563     .priv_data_size = sizeof(MpegEncContext),
4564     .init           = ff_mpv_encode_init,
4565     .encode2        = ff_mpv_encode_picture,
4566     .close          = ff_mpv_encode_end,
4567     .pix_fmts       = (const enum AVPixelFormat[]){ AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE },
4568     .priv_class     = &wmv1_class,
4569 };