git.sesse.net Git - ffmpeg/blob - libavcodec/mpegvideo_enc.c

   1 /*
   2  * The simplest mpeg encoder (well, it was the simplest!)
   3  * Copyright (c) 2000,2001 Fabrice Bellard
   4  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
   5  *
   6  * 4MV & hq & B-frame encoding stuff by Michael Niedermayer <michaelni@gmx.at>
   7  *
   8  * This file is part of Libav.
   9  *
  10  * Libav is free software; you can redistribute it and/or
  11  * modify it under the terms of the GNU Lesser General Public
  12  * License as published by the Free Software Foundation; either
  13  * version 2.1 of the License, or (at your option) any later version.
  14  *
  15  * Libav is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18  * Lesser General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU Lesser General Public
  21  * License along with Libav; if not, write to the Free Software
  22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  23  */
  24
  25 /**
  26  * @file
  27  * The simplest mpeg encoder (well, it was the simplest!).
  28  */
  29
  30 #include <stdint.h>
  31
  32 #include "libavutil/internal.h"
  33 #include "libavutil/intmath.h"
  34 #include "libavutil/mathematics.h"
  35 #include "libavutil/pixdesc.h"
  36 #include "libavutil/opt.h"
  37 #include "libavutil/timer.h"
  38 #include "avcodec.h"
  39 #include "dct.h"
  40 #include "idctdsp.h"
  41 #include "mpeg12.h"
  42 #include "mpegvideo.h"
  43 #include "mpegvideodata.h"
  44 #include "h261.h"
  45 #include "h263.h"
  46 #include "h263data.h"
  47 #include "mjpegenc_common.h"
  48 #include "mathops.h"
  49 #include "mpegutils.h"
  50 #include "mjpegenc.h"
  51 #include "msmpeg4.h"
  52 #include "pixblockdsp.h"
  53 #include "qpeldsp.h"
  54 #include "faandct.h"
  55 #include "thread.h"
  56 #include "aandcttab.h"
  57 #include "flv.h"
  58 #include "mpeg4video.h"
  59 #include "internal.h"
  60 #include "bytestream.h"
  61 #include "wmv2.h"
  62 #include "rv10.h"
  63 #include <limits.h>
  64
  65 #define QUANT_BIAS_SHIFT 8
  66
  67 #define QMAT_SHIFT_MMX 16
  68 #define QMAT_SHIFT 22
  69
  70 static int encode_picture(MpegEncContext *s, int picture_number);
  71 static int dct_quantize_refine(MpegEncContext *s, int16_t *block, int16_t *weight, int16_t *orig, int n, int qscale);
  72 static int sse_mb(MpegEncContext *s);
  73 static void denoise_dct_c(MpegEncContext *s, int16_t *block);
  74 static int dct_quantize_trellis_c(MpegEncContext *s, int16_t *block, int n, int qscale, int *overflow);
  75
  76 static uint8_t default_mv_penalty[MAX_FCODE + 1][MAX_MV * 2 + 1];
  77 static uint8_t default_fcode_tab[MAX_MV * 2 + 1];
  78
  79 const AVOption ff_mpv_generic_options[] = {
  80     FF_MPV_COMMON_OPTS
  81     { NULL },
  82 };
  83
  84 void ff_convert_matrix(MpegEncContext *s, int (*qmat)[64],
  85                        uint16_t (*qmat16)[2][64],
  86                        const uint16_t *quant_matrix,
  87                        int bias, int qmin, int qmax, int intra)
  88 {
  89     FDCTDSPContext *fdsp = &s->fdsp;
  90     int qscale;
  91     int shift = 0;
  92
  93     for (qscale = qmin; qscale <= qmax; qscale++) {
  94         int i;
  95         if (fdsp->fdct == ff_jpeg_fdct_islow_8  ||
  96 #if CONFIG_FAANDCT
  97             fdsp->fdct == ff_faandct            ||
  98 #endif /* CONFIG_FAANDCT */
  99             fdsp->fdct == ff_jpeg_fdct_islow_10) {
 100             for (i = 0; i < 64; i++) {
 101                 const int j = s->idsp.idct_permutation[i];
 102                 int64_t den = (int64_t) qscale * quant_matrix[j];
 103                 /* 16 <= qscale * quant_matrix[i] <= 7905
 104                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
 105                  *             19952 <=              x  <= 249205026
 106                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
 107                  *           3444240 >= (1 << 36) / (x) >= 275 */
 108
 109                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) / den);
 110             }
 111         } else if (fdsp->fdct == ff_fdct_ifast) {
 112             for (i = 0; i < 64; i++) {
 113                 const int j = s->idsp.idct_permutation[i];
 114                 int64_t den = ff_aanscales[i] * (int64_t) qscale * quant_matrix[j];
 115                 /* 16 <= qscale * quant_matrix[i] <= 7905
 116                  * Assume x = ff_aanscales[i] * qscale * quant_matrix[i]
 117                  *             19952 <=              x  <= 249205026
 118                  * (1 << 36) / 19952 >= (1 << 36) / (x) >= (1 << 36) / 249205026
 119                  *           3444240 >= (1 << 36) / (x) >= 275 */
 120
 121                 qmat[qscale][i] = (int)((UINT64_C(1) << (QMAT_SHIFT + 14)) / den);
 122             }
 123         } else {
 124             for (i = 0; i < 64; i++) {
 125                 const int j = s->idsp.idct_permutation[i];
 126                 int64_t den = (int64_t) qscale * quant_matrix[j];
 127                 /* We can safely suppose that 16 <= quant_matrix[i] <= 255
 128                  * Assume x = qscale * quant_matrix[i]
 129                  * So             16 <=              x  <= 7905
 130                  * so (1 << 19) / 16 >= (1 << 19) / (x) >= (1 << 19) / 7905
 131                  * so          32768 >= (1 << 19) / (x) >= 67 */
 132                 qmat[qscale][i] = (int)((UINT64_C(1) << QMAT_SHIFT) / den);
 133                 //qmat  [qscale][i] = (1 << QMAT_SHIFT_MMX) /
 134                 //                    (qscale * quant_matrix[i]);
 135                 qmat16[qscale][0][i] = (1 << QMAT_SHIFT_MMX) / den;
 136
 137                 if (qmat16[qscale][0][i] == 0 ||
 138                     qmat16[qscale][0][i] == 128 * 256)
 139                     qmat16[qscale][0][i] = 128 * 256 - 1;
 140                 qmat16[qscale][1][i] =
 141                     ROUNDED_DIV(bias << (16 - QUANT_BIAS_SHIFT),
 142                                 qmat16[qscale][0][i]);
 143             }
 144         }
 145
 146         for (i = intra; i < 64; i++) {
 147             int64_t max = 8191;
 148             if (fdsp->fdct == ff_fdct_ifast) {
 149                 max = (8191LL * ff_aanscales[i]) >> 14;
 150             }
 151             while (((max * qmat[qscale][i]) >> shift) > INT_MAX) {
 152                 shift++;
 153             }
 154         }
 155     }
 156     if (shift) {
 157         av_log(NULL, AV_LOG_INFO,
 158                "Warning, QMAT_SHIFT is larger than %d, overflows possible\n",
 159                QMAT_SHIFT - shift);
 160     }
 161 }
 162
 163 static inline void update_qscale(MpegEncContext *s)
 164 {
 165     s->qscale = (s->lambda * 139 + FF_LAMBDA_SCALE * 64) >>
 166                 (FF_LAMBDA_SHIFT + 7);
 167     s->qscale = av_clip(s->qscale, s->avctx->qmin, s->avctx->qmax);
 168
 169     s->lambda2 = (s->lambda * s->lambda + FF_LAMBDA_SCALE / 2) >>
 170                  FF_LAMBDA_SHIFT;
 171 }
 172
 173 void ff_write_quant_matrix(PutBitContext *pb, uint16_t *matrix)
 174 {
 175     int i;
 176
 177     if (matrix) {
 178         put_bits(pb, 1, 1);
 179         for (i = 0; i < 64; i++) {
 180             put_bits(pb, 8, matrix[ff_zigzag_direct[i]]);
 181         }
 182     } else
 183         put_bits(pb, 1, 0);
 184 }
 185
 186 /**
 187  * init s->current_picture.qscale_table from s->lambda_table
 188  */
 189 void ff_init_qscale_tab(MpegEncContext *s)
 190 {
 191     int8_t * const qscale_table = s->current_picture.qscale_table;
 192     int i;
 193
 194     for (i = 0; i < s->mb_num; i++) {
 195         unsigned int lam = s->lambda_table[s->mb_index2xy[i]];
 196         int qp = (lam * 139 + FF_LAMBDA_SCALE * 64) >> (FF_LAMBDA_SHIFT + 7);
 197         qscale_table[s->mb_index2xy[i]] = av_clip(qp, s->avctx->qmin,
 198                                                   s->avctx->qmax);
 199     }
 200 }
 201
 202 static void update_duplicate_context_after_me(MpegEncContext *dst,
 203                                               MpegEncContext *src)
 204 {
 205 #define COPY(a) dst->a= src->a
 206     COPY(pict_type);
 207     COPY(current_picture);
 208     COPY(f_code);
 209     COPY(b_code);
 210     COPY(qscale);
 211     COPY(lambda);
 212     COPY(lambda2);
 213     COPY(picture_in_gop_number);
 214     COPY(gop_picture_number);
 215     COPY(frame_pred_frame_dct); // FIXME don't set in encode_header
 216     COPY(progressive_frame);    // FIXME don't set in encode_header
 217     COPY(partitioned_frame);    // FIXME don't set in encode_header
 218 #undef COPY
 219 }
 220
 221 /**
 222  * Set the given MpegEncContext to defaults for encoding.
 223  * the changed fields will not depend upon the prior state of the MpegEncContext.
 224  */
 225 static void mpv_encode_defaults(MpegEncContext *s)
 226 {
 227     int i;
 228     ff_mpv_common_defaults(s);
 229
 230     for (i = -16; i < 16; i++) {
 231         default_fcode_tab[i + MAX_MV] = 1;
 232     }
 233     s->me.mv_penalty = default_mv_penalty;
 234     s->fcode_tab     = default_fcode_tab;
 235
 236     s->input_picture_number  = 0;
 237     s->picture_in_gop_number = 0;
 238 }
 239
 240 /* init video encoder */
 241 av_cold int ff_mpv_encode_init(AVCodecContext *avctx)
 242 {
 243     MpegEncContext *s = avctx->priv_data;
 244     AVCPBProperties *cpb_props;
 245     int i, ret, format_supported;
 246
 247     mpv_encode_defaults(s);
 248
 249     switch (avctx->codec_id) {
 250     case AV_CODEC_ID_MPEG2VIDEO:
 251         if (avctx->pix_fmt != AV_PIX_FMT_YUV420P &&
 252             avctx->pix_fmt != AV_PIX_FMT_YUV422P) {
 253             av_log(avctx, AV_LOG_ERROR,
 254                    "only YUV420 and YUV422 are supported\n");
 255             return -1;
 256         }
 257         break;
 258     case AV_CODEC_ID_MJPEG:
 259         format_supported = 0;
 260         /* JPEG color space */
 261         if (avctx->pix_fmt == AV_PIX_FMT_YUVJ420P ||
 262             avctx->pix_fmt == AV_PIX_FMT_YUVJ422P ||
 263             (avctx->color_range == AVCOL_RANGE_JPEG &&
 264              (avctx->pix_fmt == AV_PIX_FMT_YUV420P ||
 265               avctx->pix_fmt == AV_PIX_FMT_YUV422P)))
 266             format_supported = 1;
 267         /* MPEG color space */
 268         else if (avctx->strict_std_compliance <= FF_COMPLIANCE_UNOFFICIAL &&
 269                  (avctx->pix_fmt == AV_PIX_FMT_YUV420P ||
 270                   avctx->pix_fmt == AV_PIX_FMT_YUV422P))
 271             format_supported = 1;
 272
 273         if (!format_supported) {
 274             av_log(avctx, AV_LOG_ERROR, "colorspace not supported in jpeg\n");
 275             return -1;
 276         }
 277         break;
 278     default:
 279         if (avctx->pix_fmt != AV_PIX_FMT_YUV420P) {
 280             av_log(avctx, AV_LOG_ERROR, "only YUV420 is supported\n");
 281             return -1;
 282         }
 283     }
 284
 285     switch (avctx->pix_fmt) {
 286     case AV_PIX_FMT_YUVJ422P:
 287     case AV_PIX_FMT_YUV422P:
 288         s->chroma_format = CHROMA_422;
 289         break;
 290     case AV_PIX_FMT_YUVJ420P:
 291     case AV_PIX_FMT_YUV420P:
 292     default:
 293         s->chroma_format = CHROMA_420;
 294         break;
 295     }
 296
 297     s->bit_rate = avctx->bit_rate;
 298     s->width    = avctx->width;
 299     s->height   = avctx->height;
 300     if (avctx->gop_size > 600 &&
 301         avctx->strict_std_compliance > FF_COMPLIANCE_EXPERIMENTAL) {
 302         av_log(avctx, AV_LOG_ERROR,
 303                "Warning keyframe interval too large! reducing it ...\n");
 304         avctx->gop_size = 600;
 305     }
 306     s->gop_size     = avctx->gop_size;
 307     s->avctx        = avctx;
 308     if (avctx->max_b_frames > MAX_B_FRAMES) {
 309         av_log(avctx, AV_LOG_ERROR, "Too many B-frames requested, maximum "
 310                "is %d.\n", MAX_B_FRAMES);
 311     }
 312     s->max_b_frames = avctx->max_b_frames;
 313     s->codec_id     = avctx->codec->id;
 314     s->strict_std_compliance = avctx->strict_std_compliance;
 315     s->quarter_sample     = (avctx->flags & AV_CODEC_FLAG_QPEL) != 0;
 316     s->mpeg_quant         = avctx->mpeg_quant;
 317     s->rtp_mode           = !!avctx->rtp_payload_size;
 318     s->intra_dc_precision = avctx->intra_dc_precision;
 319     s->user_specified_pts = AV_NOPTS_VALUE;
 320
 321     if (s->gop_size <= 1) {
 322         s->intra_only = 1;
 323         s->gop_size   = 12;
 324     } else {
 325         s->intra_only = 0;
 326     }
 327
 328 #if FF_API_MOTION_EST
 329 FF_DISABLE_DEPRECATION_WARNINGS
 330     s->me_method = avctx->me_method;
 331 FF_ENABLE_DEPRECATION_WARNINGS
 332 #endif
 333
 334     /* Fixed QSCALE */
 335     s->fixed_qscale = !!(avctx->flags & AV_CODEC_FLAG_QSCALE);
 336
 337 #if FF_API_MPV_OPT
 338     FF_DISABLE_DEPRECATION_WARNINGS
 339     if (avctx->border_masking != 0.0)
 340         s->border_masking = avctx->border_masking;
 341     FF_ENABLE_DEPRECATION_WARNINGS
 342 #endif
 343
 344     s->adaptive_quant = (s->avctx->lumi_masking ||
 345                          s->avctx->dark_masking ||
 346                          s->avctx->temporal_cplx_masking ||
 347                          s->avctx->spatial_cplx_masking  ||
 348                          s->avctx->p_masking      ||
 349                          s->border_masking ||
 350                          (s->mpv_flags & FF_MPV_FLAG_QP_RD)) &&
 351                         !s->fixed_qscale;
 352
 353     s->loop_filter = !!(s->avctx->flags & AV_CODEC_FLAG_LOOP_FILTER);
 354
 355     if (avctx->rc_max_rate && !avctx->rc_buffer_size) {
 356         av_log(avctx, AV_LOG_ERROR,
 357                "a vbv buffer size is needed, "
 358                "for encoding with a maximum bitrate\n");
 359         return -1;
 360     }
 361
 362     if (avctx->rc_min_rate && avctx->rc_max_rate != avctx->rc_min_rate) {
 363         av_log(avctx, AV_LOG_INFO,
 364                "Warning min_rate > 0 but min_rate != max_rate isn't recommended!\n");
 365     }
 366
 367     if (avctx->rc_min_rate && avctx->rc_min_rate > avctx->bit_rate) {
 368         av_log(avctx, AV_LOG_ERROR, "bitrate below min bitrate\n");
 369         return -1;
 370     }
 371
 372     if (avctx->rc_max_rate && avctx->rc_max_rate < avctx->bit_rate) {
 373         av_log(avctx, AV_LOG_INFO, "bitrate above max bitrate\n");
 374         return -1;
 375     }
 376
 377     if (avctx->rc_max_rate &&
 378         avctx->rc_max_rate == avctx->bit_rate &&
 379         avctx->rc_max_rate != avctx->rc_min_rate) {
 380         av_log(avctx, AV_LOG_INFO,
 381                "impossible bitrate constraints, this will fail\n");
 382     }
 383
 384     if (avctx->rc_buffer_size &&
 385         avctx->bit_rate * (int64_t)avctx->time_base.num >
 386             avctx->rc_buffer_size * (int64_t)avctx->time_base.den) {
 387         av_log(avctx, AV_LOG_ERROR, "VBV buffer too small for bitrate\n");
 388         return -1;
 389     }
 390
 391     if (!s->fixed_qscale &&
 392         avctx->bit_rate * av_q2d(avctx->time_base) >
 393             avctx->bit_rate_tolerance) {
 394         av_log(avctx, AV_LOG_ERROR,
 395                "bitrate tolerance too small for bitrate\n");
 396         return -1;
 397     }
 398
 399     if (s->avctx->rc_max_rate &&
 400         s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
 401         (s->codec_id == AV_CODEC_ID_MPEG1VIDEO ||
 402          s->codec_id == AV_CODEC_ID_MPEG2VIDEO) &&
 403         90000LL * (avctx->rc_buffer_size - 1) >
 404             s->avctx->rc_max_rate * 0xFFFFLL) {
 405         av_log(avctx, AV_LOG_INFO,
 406                "Warning vbv_delay will be set to 0xFFFF (=VBR) as the "
 407                "specified vbv buffer is too large for the given bitrate!\n");
 408     }
 409
 410     if ((s->avctx->flags & AV_CODEC_FLAG_4MV) && s->codec_id != AV_CODEC_ID_MPEG4 &&
 411         s->codec_id != AV_CODEC_ID_H263 && s->codec_id != AV_CODEC_ID_H263P &&
 412         s->codec_id != AV_CODEC_ID_FLV1) {
 413         av_log(avctx, AV_LOG_ERROR, "4MV not supported by codec\n");
 414         return -1;
 415     }
 416
 417     if (s->obmc && s->avctx->mb_decision != FF_MB_DECISION_SIMPLE) {
 418         av_log(avctx, AV_LOG_ERROR,
 419                "OBMC is only supported with simple mb decision\n");
 420         return -1;
 421     }
 422
 423     if (s->quarter_sample && s->codec_id != AV_CODEC_ID_MPEG4) {
 424         av_log(avctx, AV_LOG_ERROR, "qpel not supported by codec\n");
 425         return -1;
 426     }
 427
 428     if (s->max_b_frames                    &&
 429         s->codec_id != AV_CODEC_ID_MPEG4      &&
 430         s->codec_id != AV_CODEC_ID_MPEG1VIDEO &&
 431         s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
 432         av_log(avctx, AV_LOG_ERROR, "b frames not supported by codec\n");
 433         return -1;
 434     }
 435
 436     if ((s->codec_id == AV_CODEC_ID_MPEG4 ||
 437          s->codec_id == AV_CODEC_ID_H263  ||
 438          s->codec_id == AV_CODEC_ID_H263P) &&
 439         (avctx->sample_aspect_ratio.num > 255 ||
 440          avctx->sample_aspect_ratio.den > 255)) {
 441         av_log(avctx, AV_LOG_ERROR,
 442                "Invalid pixel aspect ratio %i/%i, limit is 255/255\n",
 443                avctx->sample_aspect_ratio.num, avctx->sample_aspect_ratio.den);
 444         return -1;
 445     }
 446
 447     if ((s->avctx->flags & (AV_CODEC_FLAG_INTERLACED_DCT | AV_CODEC_FLAG_INTERLACED_ME)) &&
 448         s->codec_id != AV_CODEC_ID_MPEG4 && s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
 449         av_log(avctx, AV_LOG_ERROR, "interlacing not supported by codec\n");
 450         return -1;
 451     }
 452
 453     // FIXME mpeg2 uses that too
 454     if (s->mpeg_quant && s->codec_id != AV_CODEC_ID_MPEG4) {
 455         av_log(avctx, AV_LOG_ERROR,
 456                "mpeg2 style quantization not supported by codec\n");
 457         return -1;
 458     }
 459
 460     if ((s->mpv_flags & FF_MPV_FLAG_CBP_RD) && !avctx->trellis) {
 461         av_log(avctx, AV_LOG_ERROR, "CBP RD needs trellis quant\n");
 462         return -1;
 463     }
 464
 465     if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) &&
 466         s->avctx->mb_decision != FF_MB_DECISION_RD) {
 467         av_log(avctx, AV_LOG_ERROR, "QP RD needs mbd=2\n");
 468         return -1;
 469     }
 470
 471 #if FF_API_PRIVATE_OPT
 472 FF_DISABLE_DEPRECATION_WARNINGS
 473     if (avctx->scenechange_threshold)
 474         s->scenechange_threshold = avctx->scenechange_threshold;
 475 FF_ENABLE_DEPRECATION_WARNINGS
 476 #endif
 477
 478     if (s->scenechange_threshold < 1000000000 &&
 479         (s->avctx->flags & AV_CODEC_FLAG_CLOSED_GOP)) {
 480         av_log(avctx, AV_LOG_ERROR,
 481                "closed gop with scene change detection are not supported yet, "
 482                "set threshold to 1000000000\n");
 483         return -1;
 484     }
 485
 486     if (s->avctx->flags & AV_CODEC_FLAG_LOW_DELAY) {
 487         if (s->codec_id != AV_CODEC_ID_MPEG2VIDEO) {
 488             av_log(avctx, AV_LOG_ERROR,
 489                   "low delay forcing is only available for mpeg2\n");
 490             return -1;
 491         }
 492         if (s->max_b_frames != 0) {
 493             av_log(avctx, AV_LOG_ERROR,
 494                    "b frames cannot be used with low delay\n");
 495             return -1;
 496         }
 497     }
 498
 499     if (s->q_scale_type == 1) {
 500         if (avctx->qmax > 12) {
 501             av_log(avctx, AV_LOG_ERROR,
 502                    "non linear quant only supports qmax <= 12 currently\n");
 503             return -1;
 504         }
 505     }
 506
 507     if (avctx->slices > 1 &&
 508         (avctx->codec_id == AV_CODEC_ID_FLV1 || avctx->codec_id == AV_CODEC_ID_H261)) {
 509         av_log(avctx, AV_LOG_ERROR, "Multiple slices are not supported by this codec\n");
 510         return AVERROR(EINVAL);
 511     }
 512
 513     if (s->avctx->thread_count > 1         &&
 514         s->codec_id != AV_CODEC_ID_MPEG4      &&
 515         s->codec_id != AV_CODEC_ID_MPEG1VIDEO &&
 516         s->codec_id != AV_CODEC_ID_MPEG2VIDEO &&
 517         (s->codec_id != AV_CODEC_ID_H263P)) {
 518         av_log(avctx, AV_LOG_ERROR,
 519                "multi threaded encoding not supported by codec\n");
 520         return -1;
 521     }
 522
 523     if (s->avctx->thread_count < 1) {
 524         av_log(avctx, AV_LOG_ERROR,
 525                "automatic thread number detection not supported by codec,"
 526                "patch welcome\n");
 527         return -1;
 528     }
 529
 530     if (!avctx->time_base.den || !avctx->time_base.num) {
 531         av_log(avctx, AV_LOG_ERROR, "framerate not set\n");
 532         return -1;
 533     }
 534
 535 #if FF_API_PRIVATE_OPT
 536 FF_DISABLE_DEPRECATION_WARNINGS
 537     if (avctx->b_frame_strategy)
 538         s->b_frame_strategy = avctx->b_frame_strategy;
 539     if (avctx->b_sensitivity != 40)
 540         s->b_sensitivity = avctx->b_sensitivity;
 541 FF_ENABLE_DEPRECATION_WARNINGS
 542 #endif
 543
 544     if (s->b_frame_strategy && (avctx->flags & AV_CODEC_FLAG_PASS2)) {
 545         av_log(avctx, AV_LOG_INFO,
 546                "notice: b_frame_strategy only affects the first pass\n");
 547         s->b_frame_strategy = 0;
 548     }
 549
 550     i = av_gcd(avctx->time_base.den, avctx->time_base.num);
 551     if (i > 1) {
 552         av_log(avctx, AV_LOG_INFO, "removing common factors from framerate\n");
 553         avctx->time_base.den /= i;
 554         avctx->time_base.num /= i;
 555         //return -1;
 556     }
 557
 558     if (s->mpeg_quant || s->codec_id == AV_CODEC_ID_MPEG1VIDEO ||
 559         s->codec_id == AV_CODEC_ID_MPEG2VIDEO || s->codec_id == AV_CODEC_ID_MJPEG) {
 560         // (a + x * 3 / 8) / x
 561         s->intra_quant_bias = 3 << (QUANT_BIAS_SHIFT - 3);
 562         s->inter_quant_bias = 0;
 563     } else {
 564         s->intra_quant_bias = 0;
 565         // (a - x / 4) / x
 566         s->inter_quant_bias = -(1 << (QUANT_BIAS_SHIFT - 2));
 567     }
 568
 569 #if FF_API_QUANT_BIAS
 570 FF_DISABLE_DEPRECATION_WARNINGS
 571     if (avctx->intra_quant_bias != FF_DEFAULT_QUANT_BIAS)
 572         s->intra_quant_bias = avctx->intra_quant_bias;
 573     if (avctx->inter_quant_bias != FF_DEFAULT_QUANT_BIAS)
 574         s->inter_quant_bias = avctx->inter_quant_bias;
 575 FF_ENABLE_DEPRECATION_WARNINGS
 576 #endif
 577
 578     if (avctx->codec_id == AV_CODEC_ID_MPEG4 &&
 579         s->avctx->time_base.den > (1 << 16) - 1) {
 580         av_log(avctx, AV_LOG_ERROR,
 581                "timebase %d/%d not supported by MPEG 4 standard, "
 582                "the maximum admitted value for the timebase denominator "
 583                "is %d\n", s->avctx->time_base.num, s->avctx->time_base.den,
 584                (1 << 16) - 1);
 585         return -1;
 586     }
 587     s->time_increment_bits = av_log2(s->avctx->time_base.den - 1) + 1;
 588
 589     switch (avctx->codec->id) {
 590     case AV_CODEC_ID_MPEG1VIDEO:
 591         s->out_format = FMT_MPEG1;
 592         s->low_delay  = !!(s->avctx->flags & AV_CODEC_FLAG_LOW_DELAY);
 593         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
 594         break;
 595     case AV_CODEC_ID_MPEG2VIDEO:
 596         s->out_format = FMT_MPEG1;
 597         s->low_delay  = !!(s->avctx->flags & AV_CODEC_FLAG_LOW_DELAY);
 598         avctx->delay  = s->low_delay ? 0 : (s->max_b_frames + 1);
 599         s->rtp_mode   = 1;
 600         break;
 601     case AV_CODEC_ID_MJPEG:
 602         s->out_format = FMT_MJPEG;
 603         s->intra_only = 1; /* force intra only for jpeg */
 604         if (!CONFIG_MJPEG_ENCODER ||
 605             ff_mjpeg_encode_init(s) < 0)
 606             return -1;
 607         avctx->delay = 0;
 608         s->low_delay = 1;
 609         break;
 610     case AV_CODEC_ID_H261:
 611         if (!CONFIG_H261_ENCODER)
 612             return -1;
 613         if (ff_h261_get_picture_format(s->width, s->height) < 0) {
 614             av_log(avctx, AV_LOG_ERROR,
 615                    "The specified picture size of %dx%d is not valid for the "
 616                    "H.261 codec.\nValid sizes are 176x144, 352x288\n",
 617                     s->width, s->height);
 618             return -1;
 619         }
 620         s->out_format = FMT_H261;
 621         avctx->delay  = 0;
 622         s->low_delay  = 1;
 623         s->rtp_mode   = 0; /* Sliced encoding not supported */
 624         break;
 625     case AV_CODEC_ID_H263:
 626         if (!CONFIG_H263_ENCODER)
 627         return -1;
 628         if (ff_match_2uint16(ff_h263_format, FF_ARRAY_ELEMS(ff_h263_format),
 629                              s->width, s->height) == 8) {
 630             av_log(avctx, AV_LOG_INFO,
 631                    "The specified picture size of %dx%d is not valid for "
 632                    "the H.263 codec.\nValid sizes are 128x96, 176x144, "
 633                    "352x288, 704x576, and 1408x1152."
 634                    "Try H.263+.\n", s->width, s->height);
 635             return -1;
 636         }
 637         s->out_format = FMT_H263;
 638         avctx->delay  = 0;
 639         s->low_delay  = 1;
 640         break;
 641     case AV_CODEC_ID_H263P:
 642         s->out_format = FMT_H263;
 643         s->h263_plus  = 1;
 644         /* Fx */
 645         s->h263_aic        = (avctx->flags & AV_CODEC_FLAG_AC_PRED) ? 1 : 0;
 646         s->modified_quant  = s->h263_aic;
 647         s->loop_filter     = (avctx->flags & AV_CODEC_FLAG_LOOP_FILTER) ? 1 : 0;
 648         s->unrestricted_mv = s->obmc || s->loop_filter || s->umvplus;
 649
 650         /* /Fx */
 651         /* These are just to be sure */
 652         avctx->delay = 0;
 653         s->low_delay = 1;
 654         break;
 655     case AV_CODEC_ID_FLV1:
 656         s->out_format      = FMT_H263;
 657         s->h263_flv        = 2; /* format = 1; 11-bit codes */
 658         s->unrestricted_mv = 1;
 659         s->rtp_mode  = 0; /* don't allow GOB */
 660         avctx->delay = 0;
 661         s->low_delay = 1;
 662         break;
 663     case AV_CODEC_ID_RV10:
 664         s->out_format = FMT_H263;
 665         avctx->delay  = 0;
 666         s->low_delay  = 1;
 667         break;
 668     case AV_CODEC_ID_RV20:
 669         s->out_format      = FMT_H263;
 670         avctx->delay       = 0;
 671         s->low_delay       = 1;
 672         s->modified_quant  = 1;
 673         s->h263_aic        = 1;
 674         s->h263_plus       = 1;
 675         s->loop_filter     = 1;
 676         s->unrestricted_mv = 0;
 677         break;
 678     case AV_CODEC_ID_MPEG4:
 679         s->out_format      = FMT_H263;
 680         s->h263_pred       = 1;
 681         s->unrestricted_mv = 1;
 682         s->low_delay       = s->max_b_frames ? 0 : 1;
 683         avctx->delay       = s->low_delay ? 0 : (s->max_b_frames + 1);
 684         break;
 685     case AV_CODEC_ID_MSMPEG4V2:
 686         s->out_format      = FMT_H263;
 687         s->h263_pred       = 1;
 688         s->unrestricted_mv = 1;
 689         s->msmpeg4_version = 2;
 690         avctx->delay       = 0;
 691         s->low_delay       = 1;
 692         break;
 693     case AV_CODEC_ID_MSMPEG4V3:
 694         s->out_format        = FMT_H263;
 695         s->h263_pred         = 1;
 696         s->unrestricted_mv   = 1;
 697         s->msmpeg4_version   = 3;
 698         s->flipflop_rounding = 1;
 699         avctx->delay         = 0;
 700         s->low_delay         = 1;
 701         break;
 702     case AV_CODEC_ID_WMV1:
 703         s->out_format        = FMT_H263;
 704         s->h263_pred         = 1;
 705         s->unrestricted_mv   = 1;
 706         s->msmpeg4_version   = 4;
 707         s->flipflop_rounding = 1;
 708         avctx->delay         = 0;
 709         s->low_delay         = 1;
 710         break;
 711     case AV_CODEC_ID_WMV2:
 712         s->out_format        = FMT_H263;
 713         s->h263_pred         = 1;
 714         s->unrestricted_mv   = 1;
 715         s->msmpeg4_version   = 5;
 716         s->flipflop_rounding = 1;
 717         avctx->delay         = 0;
 718         s->low_delay         = 1;
 719         break;
 720     default:
 721         return -1;
 722     }
 723
 724     avctx->has_b_frames = !s->low_delay;
 725
 726     s->encoding = 1;
 727
 728     s->progressive_frame    =
 729     s->progressive_sequence = !(avctx->flags & (AV_CODEC_FLAG_INTERLACED_DCT |
 730                                                 AV_CODEC_FLAG_INTERLACED_ME) ||
 731                                 s->alternate_scan);
 732
 733     /* init */
 734     ff_mpv_idct_init(s);
 735     if (ff_mpv_common_init(s) < 0)
 736         return -1;
 737
 738     if (ARCH_X86)
 739         ff_mpv_encode_init_x86(s);
 740
 741     ff_fdctdsp_init(&s->fdsp, avctx);
 742     ff_me_cmp_init(&s->mecc, avctx);
 743     ff_mpegvideoencdsp_init(&s->mpvencdsp, avctx);
 744     ff_pixblockdsp_init(&s->pdsp, avctx);
 745     ff_qpeldsp_init(&s->qdsp);
 746
 747     if (s->msmpeg4_version) {
 748         FF_ALLOCZ_OR_GOTO(s->avctx, s->ac_stats,
 749                           2 * 2 * (MAX_LEVEL + 1) *
 750                           (MAX_RUN + 1) * 2 * sizeof(int), fail);
 751     }
 752     FF_ALLOCZ_OR_GOTO(s->avctx, s->avctx->stats_out, 256, fail);
 753
 754     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_intra_matrix,   64 * 32 * sizeof(int), fail);
 755     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_inter_matrix,   64 * 32 * sizeof(int), fail);
 756     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_intra_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
 757     FF_ALLOCZ_OR_GOTO(s->avctx, s->q_inter_matrix16, 64 * 32 * 2 * sizeof(uint16_t), fail);
 758     FF_ALLOCZ_OR_GOTO(s->avctx, s->input_picture,
 759                       MAX_PICTURE_COUNT * sizeof(Picture *), fail);
 760     FF_ALLOCZ_OR_GOTO(s->avctx, s->reordered_input_picture,
 761                       MAX_PICTURE_COUNT * sizeof(Picture *), fail);
 762
 763     if (s->avctx->noise_reduction) {
 764         FF_ALLOCZ_OR_GOTO(s->avctx, s->dct_offset,
 765                           2 * 64 * sizeof(uint16_t), fail);
 766     }
 767
 768     if (CONFIG_H263_ENCODER)
 769         ff_h263dsp_init(&s->h263dsp);
 770     if (!s->dct_quantize)
 771         s->dct_quantize = ff_dct_quantize_c;
 772     if (!s->denoise_dct)
 773         s->denoise_dct  = denoise_dct_c;
 774     s->fast_dct_quantize = s->dct_quantize;
 775     if (avctx->trellis)
 776         s->dct_quantize  = dct_quantize_trellis_c;
 777
 778     if ((CONFIG_H263P_ENCODER || CONFIG_RV20_ENCODER) && s->modified_quant)
 779         s->chroma_qscale_table = ff_h263_chroma_qscale_table;
 780
 781     if (s->slice_context_count > 1) {
 782         s->rtp_mode = 1;
 783
 784         if (avctx->codec_id == AV_CODEC_ID_H263 || avctx->codec_id == AV_CODEC_ID_H263P)
 785             s->h263_slice_structured = 1;
 786     }
 787
 788     s->quant_precision = 5;
 789
 790 #if FF_API_PRIVATE_OPT
 791 FF_DISABLE_DEPRECATION_WARNINGS
 792     if (avctx->frame_skip_threshold)
 793         s->frame_skip_threshold = avctx->frame_skip_threshold;
 794     if (avctx->frame_skip_factor)
 795         s->frame_skip_factor = avctx->frame_skip_factor;
 796     if (avctx->frame_skip_exp)
 797         s->frame_skip_exp = avctx->frame_skip_exp;
 798     if (avctx->frame_skip_cmp != FF_CMP_DCTMAX)
 799         s->frame_skip_cmp = avctx->frame_skip_cmp;
 800 FF_ENABLE_DEPRECATION_WARNINGS
 801 #endif
 802
 803     ff_set_cmp(&s->mecc, s->mecc.ildct_cmp,      s->avctx->ildct_cmp);
 804     ff_set_cmp(&s->mecc, s->mecc.frame_skip_cmp, s->frame_skip_cmp);
 805
 806     if (CONFIG_H261_ENCODER && s->out_format == FMT_H261)
 807         ff_h261_encode_init(s);
 808     if (CONFIG_H263_ENCODER && s->out_format == FMT_H263)
 809         ff_h263_encode_init(s);
 810     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
 811         if ((ret = ff_msmpeg4_encode_init(s)) < 0)
 812             return ret;
 813     if ((CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
 814         && s->out_format == FMT_MPEG1)
 815         ff_mpeg1_encode_init(s);
 816
 817     /* init q matrix */
 818     for (i = 0; i < 64; i++) {
 819         int j = s->idsp.idct_permutation[i];
 820         if (CONFIG_MPEG4_ENCODER && s->codec_id == AV_CODEC_ID_MPEG4 &&
 821             s->mpeg_quant) {
 822             s->intra_matrix[j] = ff_mpeg4_default_intra_matrix[i];
 823             s->inter_matrix[j] = ff_mpeg4_default_non_intra_matrix[i];
 824         } else if (s->out_format == FMT_H263 || s->out_format == FMT_H261) {
 825             s->intra_matrix[j] =
 826             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
 827         } else {
 828             /* mpeg1/2 */
 829             s->intra_matrix[j] = ff_mpeg1_default_intra_matrix[i];
 830             s->inter_matrix[j] = ff_mpeg1_default_non_intra_matrix[i];
 831         }
 832         if (s->avctx->intra_matrix)
 833             s->intra_matrix[j] = s->avctx->intra_matrix[i];
 834         if (s->avctx->inter_matrix)
 835             s->inter_matrix[j] = s->avctx->inter_matrix[i];
 836     }
 837
 838     /* precompute matrix */
 839     /* for mjpeg, we do include qscale in the matrix */
 840     if (s->out_format != FMT_MJPEG) {
 841         ff_convert_matrix(s, s->q_intra_matrix, s->q_intra_matrix16,
 842                           s->intra_matrix, s->intra_quant_bias, avctx->qmin,
 843                           31, 1);
 844         ff_convert_matrix(s, s->q_inter_matrix, s->q_inter_matrix16,
 845                           s->inter_matrix, s->inter_quant_bias, avctx->qmin,
 846                           31, 0);
 847     }
 848
 849     if (ff_rate_control_init(s) < 0)
 850         return -1;
 851
 852 #if FF_API_ERROR_RATE
 853     FF_DISABLE_DEPRECATION_WARNINGS
 854     if (avctx->error_rate)
 855         s->error_rate = avctx->error_rate;
 856     FF_ENABLE_DEPRECATION_WARNINGS;
 857 #endif
 858
 859 #if FF_API_NORMALIZE_AQP
 860     FF_DISABLE_DEPRECATION_WARNINGS
 861     if (avctx->flags & CODEC_FLAG_NORMALIZE_AQP)
 862         s->mpv_flags |= FF_MPV_FLAG_NAQ;
 863     FF_ENABLE_DEPRECATION_WARNINGS;
 864 #endif
 865
 866 #if FF_API_MV0
 867     FF_DISABLE_DEPRECATION_WARNINGS
 868     if (avctx->flags & CODEC_FLAG_MV0)
 869         s->mpv_flags |= FF_MPV_FLAG_MV0;
 870     FF_ENABLE_DEPRECATION_WARNINGS
 871 #endif
 872
 873 #if FF_API_MPV_OPT
 874     FF_DISABLE_DEPRECATION_WARNINGS
 875     if (avctx->rc_qsquish != 0.0)
 876         s->rc_qsquish = avctx->rc_qsquish;
 877     if (avctx->rc_qmod_amp != 0.0)
 878         s->rc_qmod_amp = avctx->rc_qmod_amp;
 879     if (avctx->rc_qmod_freq)
 880         s->rc_qmod_freq = avctx->rc_qmod_freq;
 881     if (avctx->rc_buffer_aggressivity != 1.0)
 882         s->rc_buffer_aggressivity = avctx->rc_buffer_aggressivity;
 883     if (avctx->rc_initial_cplx != 0.0)
 884         s->rc_initial_cplx = avctx->rc_initial_cplx;
 885     if (avctx->lmin)
 886         s->lmin = avctx->lmin;
 887     if (avctx->lmax)
 888         s->lmax = avctx->lmax;
 889
 890     if (avctx->rc_eq) {
 891         av_freep(&s->rc_eq);
 892         s->rc_eq = av_strdup(avctx->rc_eq);
 893         if (!s->rc_eq)
 894             return AVERROR(ENOMEM);
 895     }
 896     FF_ENABLE_DEPRECATION_WARNINGS
 897 #endif
 898
 899 #if FF_API_PRIVATE_OPT
 900     FF_DISABLE_DEPRECATION_WARNINGS
 901     if (avctx->brd_scale)
 902         s->brd_scale = avctx->brd_scale;
 903     FF_ENABLE_DEPRECATION_WARNINGS
 904 #endif
 905
 906     if (s->b_frame_strategy == 2) {
 907         for (i = 0; i < s->max_b_frames + 2; i++) {
 908             s->tmp_frames[i] = av_frame_alloc();
 909             if (!s->tmp_frames[i])
 910                 return AVERROR(ENOMEM);
 911
 912             s->tmp_frames[i]->format = AV_PIX_FMT_YUV420P;
 913             s->tmp_frames[i]->width  = s->width  >> s->brd_scale;
 914             s->tmp_frames[i]->height = s->height >> s->brd_scale;
 915
 916             ret = av_frame_get_buffer(s->tmp_frames[i], 32);
 917             if (ret < 0)
 918                 return ret;
 919         }
 920     }
 921
 922     cpb_props = ff_add_cpb_side_data(avctx);
 923     if (!cpb_props)
 924         return AVERROR(ENOMEM);
 925     cpb_props->max_bitrate = avctx->rc_max_rate;
 926     cpb_props->min_bitrate = avctx->rc_min_rate;
 927     cpb_props->avg_bitrate = avctx->bit_rate;
 928     cpb_props->buffer_size = avctx->rc_buffer_size;
 929
 930     return 0;
 931 fail:
 932     ff_mpv_encode_end(avctx);
 933     return AVERROR_UNKNOWN;
 934 }
 935
 936 av_cold int ff_mpv_encode_end(AVCodecContext *avctx)
 937 {
 938     MpegEncContext *s = avctx->priv_data;
 939     int i;
 940
 941     ff_rate_control_uninit(s);
 942
 943     ff_mpv_common_end(s);
 944     if (CONFIG_MJPEG_ENCODER &&
 945         s->out_format == FMT_MJPEG)
 946         ff_mjpeg_encode_close(s);
 947
 948     av_freep(&avctx->extradata);
 949
 950     for (i = 0; i < FF_ARRAY_ELEMS(s->tmp_frames); i++)
 951         av_frame_free(&s->tmp_frames[i]);
 952
 953     ff_free_picture_tables(&s->new_picture);
 954     ff_mpeg_unref_picture(s->avctx, &s->new_picture);
 955
 956     av_freep(&s->avctx->stats_out);
 957     av_freep(&s->ac_stats);
 958
 959     av_freep(&s->q_intra_matrix);
 960     av_freep(&s->q_inter_matrix);
 961     av_freep(&s->q_intra_matrix16);
 962     av_freep(&s->q_inter_matrix16);
 963     av_freep(&s->input_picture);
 964     av_freep(&s->reordered_input_picture);
 965     av_freep(&s->dct_offset);
 966
 967     return 0;
 968 }
 969
 970 static int get_sae(uint8_t *src, int ref, int stride)
 971 {
 972     int x,y;
 973     int acc = 0;
 974
 975     for (y = 0; y < 16; y++) {
 976         for (x = 0; x < 16; x++) {
 977             acc += FFABS(src[x + y * stride] - ref);
 978         }
 979     }
 980
 981     return acc;
 982 }
 983
 984 static int get_intra_count(MpegEncContext *s, uint8_t *src,
 985                            uint8_t *ref, int stride)
 986 {
 987     int x, y, w, h;
 988     int acc = 0;
 989
 990     w = s->width  & ~15;
 991     h = s->height & ~15;
 992
 993     for (y = 0; y < h; y += 16) {
 994         for (x = 0; x < w; x += 16) {
 995             int offset = x + y * stride;
 996             int sad  = s->mecc.sad[0](NULL, src + offset, ref + offset,
 997                                       stride, 16);
 998             int mean = (s->mpvencdsp.pix_sum(src + offset, stride) + 128) >> 8;
 999             int sae  = get_sae(src + offset, mean, stride);
1000
1001             acc += sae + 500 < sad;
1002         }
1003     }
1004     return acc;
1005 }
1006
1007 static int alloc_picture(MpegEncContext *s, Picture *pic, int shared)
1008 {
1009     return ff_alloc_picture(s->avctx, pic, &s->me, &s->sc, shared, 1,
1010                             s->chroma_x_shift, s->chroma_y_shift, s->out_format,
1011                             s->mb_stride, s->mb_height, s->b8_stride,
1012                             &s->linesize, &s->uvlinesize);
1013 }
1014
1015 static int load_input_picture(MpegEncContext *s, const AVFrame *pic_arg)
1016 {
1017     Picture *pic = NULL;
1018     int64_t pts;
1019     int i, display_picture_number = 0, ret;
1020     int encoding_delay = s->max_b_frames ? s->max_b_frames
1021                                          : (s->low_delay ? 0 : 1);
1022     int flush_offset = 1;
1023     int direct = 1;
1024
1025     if (pic_arg) {
1026         pts = pic_arg->pts;
1027         display_picture_number = s->input_picture_number++;
1028
1029         if (pts != AV_NOPTS_VALUE) {
1030             if (s->user_specified_pts != AV_NOPTS_VALUE) {
1031                 int64_t time = pts;
1032                 int64_t last = s->user_specified_pts;
1033
1034                 if (time <= last) {
1035                     av_log(s->avctx, AV_LOG_ERROR,
1036                            "Error, Invalid timestamp=%"PRId64", "
1037                            "last=%"PRId64"\n", pts, s->user_specified_pts);
1038                     return -1;
1039                 }
1040
1041                 if (!s->low_delay && display_picture_number == 1)
1042                     s->dts_delta = time - last;
1043             }
1044             s->user_specified_pts = pts;
1045         } else {
1046             if (s->user_specified_pts != AV_NOPTS_VALUE) {
1047                 s->user_specified_pts =
1048                 pts = s->user_specified_pts + 1;
1049                 av_log(s->avctx, AV_LOG_INFO,
1050                        "Warning: AVFrame.pts=? trying to guess (%"PRId64")\n",
1051                        pts);
1052             } else {
1053                 pts = display_picture_number;
1054             }
1055         }
1056
1057         if (!pic_arg->buf[0] ||
1058             pic_arg->linesize[0] != s->linesize ||
1059             pic_arg->linesize[1] != s->uvlinesize ||
1060             pic_arg->linesize[2] != s->uvlinesize)
1061             direct = 0;
1062         if ((s->width & 15) || (s->height & 15))
1063             direct = 0;
1064
1065         ff_dlog(s->avctx, "%d %d %td %td\n", pic_arg->linesize[0],
1066                 pic_arg->linesize[1], s->linesize, s->uvlinesize);
1067
1068         i = ff_find_unused_picture(s->avctx, s->picture, direct);
1069         if (i < 0)
1070             return i;
1071
1072         pic = &s->picture[i];
1073         pic->reference = 3;
1074
1075         if (direct) {
1076             if ((ret = av_frame_ref(pic->f, pic_arg)) < 0)
1077                 return ret;
1078         }
1079         ret = alloc_picture(s, pic, direct);
1080         if (ret < 0)
1081             return ret;
1082
1083         if (!direct) {
1084             if (pic->f->data[0] + INPLACE_OFFSET == pic_arg->data[0] &&
1085                 pic->f->data[1] + INPLACE_OFFSET == pic_arg->data[1] &&
1086                 pic->f->data[2] + INPLACE_OFFSET == pic_arg->data[2]) {
1087                 // empty
1088             } else {
1089                 int h_chroma_shift, v_chroma_shift;
1090                 av_pix_fmt_get_chroma_sub_sample(s->avctx->pix_fmt,
1091                                                  &h_chroma_shift,
1092                                                  &v_chroma_shift);
1093
1094                 for (i = 0; i < 3; i++) {
1095                     int src_stride = pic_arg->linesize[i];
1096                     int dst_stride = i ? s->uvlinesize : s->linesize;
1097                     int h_shift = i ? h_chroma_shift : 0;
1098                     int v_shift = i ? v_chroma_shift : 0;
1099                     int w = s->width  >> h_shift;
1100                     int h = s->height >> v_shift;
1101                     uint8_t *src = pic_arg->data[i];
1102                     uint8_t *dst = pic->f->data[i];
1103
1104                     if (!s->avctx->rc_buffer_size)
1105                         dst += INPLACE_OFFSET;
1106
1107                     if (src_stride == dst_stride)
1108                         memcpy(dst, src, src_stride * h);
1109                     else {
1110                         int h2 = h;
1111                         uint8_t *dst2 = dst;
1112                         while (h2--) {
1113                             memcpy(dst2, src, w);
1114                             dst2 += dst_stride;
1115                             src += src_stride;
1116                         }
1117                     }
1118                     if ((s->width & 15) || (s->height & 15)) {
1119                         s->mpvencdsp.draw_edges(dst, dst_stride,
1120                                                 w, h,
1121                                                 16 >> h_shift,
1122                                                 16 >> v_shift,
1123                                                 EDGE_BOTTOM);
1124                     }
1125                 }
1126             }
1127         }
1128         ret = av_frame_copy_props(pic->f, pic_arg);
1129         if (ret < 0)
1130             return ret;
1131
1132         pic->f->display_picture_number = display_picture_number;
1133         pic->f->pts = pts; // we set this here to avoid modifiying pic_arg
1134     } else {
1135         /* Flushing: When we have not received enough input frames,
1136          * ensure s->input_picture[0] contains the first picture */
1137         for (flush_offset = 0; flush_offset < encoding_delay + 1; flush_offset++)
1138             if (s->input_picture[flush_offset])
1139                 break;
1140
1141         if (flush_offset <= 1)
1142             flush_offset = 1;
1143         else
1144             encoding_delay = encoding_delay - flush_offset + 1;
1145     }
1146
1147     /* shift buffer entries */
1148     for (i = flush_offset; i < MAX_PICTURE_COUNT /*s->encoding_delay + 1*/; i++)
1149         s->input_picture[i - flush_offset] = s->input_picture[i];
1150
1151     s->input_picture[encoding_delay] = (Picture*) pic;
1152
1153     return 0;
1154 }
1155
1156 static int skip_check(MpegEncContext *s, Picture *p, Picture *ref)
1157 {
1158     int x, y, plane;
1159     int score = 0;
1160     int64_t score64 = 0;
1161
1162     for (plane = 0; plane < 3; plane++) {
1163         const int stride = p->f->linesize[plane];
1164         const int bw = plane ? 1 : 2;
1165         for (y = 0; y < s->mb_height * bw; y++) {
1166             for (x = 0; x < s->mb_width * bw; x++) {
1167                 int off = p->shared ? 0 : 16;
1168                 uint8_t *dptr = p->f->data[plane] + 8 * (x + y * stride) + off;
1169                 uint8_t *rptr = ref->f->data[plane] + 8 * (x + y * stride);
1170                 int v = s->mecc.frame_skip_cmp[1](s, dptr, rptr, stride, 8);
1171
1172                 switch (s->frame_skip_exp) {
1173                 case 0: score    =  FFMAX(score, v);          break;
1174                 case 1: score   += FFABS(v);                  break;
1175                 case 2: score   += v * v;                     break;
1176                 case 3: score64 += FFABS(v * v * (int64_t)v); break;
1177                 case 4: score64 += v * v * (int64_t)(v * v);  break;
1178                 }
1179             }
1180         }
1181     }
1182
1183     if (score)
1184         score64 = score;
1185
1186     if (score64 < s->frame_skip_threshold)
1187         return 1;
1188     if (score64 < ((s->frame_skip_factor * (int64_t) s->lambda) >> 8))
1189         return 1;
1190     return 0;
1191 }
1192
1193 static int encode_frame(AVCodecContext *c, AVFrame *frame)
1194 {
1195     AVPacket pkt = { 0 };
1196     int ret, got_output;
1197
1198     av_init_packet(&pkt);
1199     ret = avcodec_encode_video2(c, &pkt, frame, &got_output);
1200     if (ret < 0)
1201         return ret;
1202
1203     ret = pkt.size;
1204     av_packet_unref(&pkt);
1205     return ret;
1206 }
1207
1208 static int estimate_best_b_count(MpegEncContext *s)
1209 {
1210     AVCodec *codec    = avcodec_find_encoder(s->avctx->codec_id);
1211     AVCodecContext *c = avcodec_alloc_context3(NULL);
1212     const int scale = s->brd_scale;
1213     int i, j, out_size, p_lambda, b_lambda, lambda2;
1214     int64_t best_rd  = INT64_MAX;
1215     int best_b_count = -1;
1216
1217     if (!c)
1218         return AVERROR(ENOMEM);
1219     assert(scale >= 0 && scale <= 3);
1220
1221     //emms_c();
1222     //s->next_picture_ptr->quality;
1223     p_lambda = s->last_lambda_for[AV_PICTURE_TYPE_P];
1224     //p_lambda * FFABS(s->avctx->b_quant_factor) + s->avctx->b_quant_offset;
1225     b_lambda = s->last_lambda_for[AV_PICTURE_TYPE_B];
1226     if (!b_lambda) // FIXME we should do this somewhere else
1227         b_lambda = p_lambda;
1228     lambda2  = (b_lambda * b_lambda + (1 << FF_LAMBDA_SHIFT) / 2) >>
1229                FF_LAMBDA_SHIFT;
1230
1231     c->width        = s->width  >> scale;
1232     c->height       = s->height >> scale;
1233     c->flags        = AV_CODEC_FLAG_QSCALE | AV_CODEC_FLAG_PSNR;
1234     c->flags       |= s->avctx->flags & AV_CODEC_FLAG_QPEL;
1235     c->mb_decision  = s->avctx->mb_decision;
1236     c->me_cmp       = s->avctx->me_cmp;
1237     c->mb_cmp       = s->avctx->mb_cmp;
1238     c->me_sub_cmp   = s->avctx->me_sub_cmp;
1239     c->pix_fmt      = AV_PIX_FMT_YUV420P;
1240     c->time_base    = s->avctx->time_base;
1241     c->max_b_frames = s->max_b_frames;
1242
1243     if (avcodec_open2(c, codec, NULL) < 0)
1244         return -1;
1245
1246     for (i = 0; i < s->max_b_frames + 2; i++) {
1247         Picture pre_input, *pre_input_ptr = i ? s->input_picture[i - 1] :
1248                                                 s->next_picture_ptr;
1249
1250         if (pre_input_ptr && (!i || s->input_picture[i - 1])) {
1251             pre_input = *pre_input_ptr;
1252
1253             if (!pre_input.shared && i) {
1254                 pre_input.f->data[0] += INPLACE_OFFSET;
1255                 pre_input.f->data[1] += INPLACE_OFFSET;
1256                 pre_input.f->data[2] += INPLACE_OFFSET;
1257             }
1258
1259             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[0],
1260                                        s->tmp_frames[i]->linesize[0],
1261                                        pre_input.f->data[0],
1262                                        pre_input.f->linesize[0],
1263                                        c->width, c->height);
1264             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[1],
1265                                        s->tmp_frames[i]->linesize[1],
1266                                        pre_input.f->data[1],
1267                                        pre_input.f->linesize[1],
1268                                        c->width >> 1, c->height >> 1);
1269             s->mpvencdsp.shrink[scale](s->tmp_frames[i]->data[2],
1270                                        s->tmp_frames[i]->linesize[2],
1271                                        pre_input.f->data[2],
1272                                        pre_input.f->linesize[2],
1273                                        c->width >> 1, c->height >> 1);
1274         }
1275     }
1276
1277     for (j = 0; j < s->max_b_frames + 1; j++) {
1278         int64_t rd = 0;
1279
1280         if (!s->input_picture[j])
1281             break;
1282
1283         c->error[0] = c->error[1] = c->error[2] = 0;
1284
1285         s->tmp_frames[0]->pict_type = AV_PICTURE_TYPE_I;
1286         s->tmp_frames[0]->quality   = 1 * FF_QP2LAMBDA;
1287
1288         out_size = encode_frame(c, s->tmp_frames[0]);
1289
1290         //rd += (out_size * lambda2) >> FF_LAMBDA_SHIFT;
1291
1292         for (i = 0; i < s->max_b_frames + 1; i++) {
1293             int is_p = i % (j + 1) == j || i == s->max_b_frames;
1294
1295             s->tmp_frames[i + 1]->pict_type = is_p ?
1296                                      AV_PICTURE_TYPE_P : AV_PICTURE_TYPE_B;
1297             s->tmp_frames[i + 1]->quality   = is_p ? p_lambda : b_lambda;
1298
1299             out_size = encode_frame(c, s->tmp_frames[i + 1]);
1300
1301             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1302         }
1303
1304         /* get the delayed frames */
1305         while (out_size) {
1306             out_size = encode_frame(c, NULL);
1307             rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3);
1308         }
1309
1310         rd += c->error[0] + c->error[1] + c->error[2];
1311
1312         if (rd < best_rd) {
1313             best_rd = rd;
1314             best_b_count = j;
1315         }
1316     }
1317
1318     avcodec_close(c);
1319     av_freep(&c);
1320
1321     return best_b_count;
1322 }
1323
1324 static int select_input_picture(MpegEncContext *s)
1325 {
1326     int i, ret;
1327
1328     for (i = 1; i < MAX_PICTURE_COUNT; i++)
1329         s->reordered_input_picture[i - 1] = s->reordered_input_picture[i];
1330     s->reordered_input_picture[MAX_PICTURE_COUNT - 1] = NULL;
1331
1332     /* set next picture type & ordering */
1333     if (!s->reordered_input_picture[0] && s->input_picture[0]) {
1334         if (/*s->picture_in_gop_number >= s->gop_size ||*/
1335             !s->next_picture_ptr || s->intra_only) {
1336             s->reordered_input_picture[0] = s->input_picture[0];
1337             s->reordered_input_picture[0]->f->pict_type = AV_PICTURE_TYPE_I;
1338             s->reordered_input_picture[0]->f->coded_picture_number =
1339                 s->coded_picture_number++;
1340         } else {
1341             int b_frames = 0;
1342
1343             if (s->frame_skip_threshold || s->frame_skip_factor) {
1344                 if (s->picture_in_gop_number < s->gop_size &&
1345                     skip_check(s, s->input_picture[0], s->next_picture_ptr)) {
1346                     // FIXME check that te gop check above is +-1 correct
1347                     av_frame_unref(s->input_picture[0]->f);
1348
1349                     emms_c();
1350                     ff_vbv_update(s, 0);
1351
1352                     goto no_output_pic;
1353                 }
1354             }
1355
1356             if (s->avctx->flags & AV_CODEC_FLAG_PASS2) {
1357                 for (i = 0; i < s->max_b_frames + 1; i++) {
1358                     int pict_num = s->input_picture[0]->f->display_picture_number + i;
1359
1360                     if (pict_num >= s->rc_context.num_entries)
1361                         break;
1362                     if (!s->input_picture[i]) {
1363                         s->rc_context.entry[pict_num - 1].new_pict_type = AV_PICTURE_TYPE_P;
1364                         break;
1365                     }
1366
1367                     s->input_picture[i]->f->pict_type =
1368                         s->rc_context.entry[pict_num].new_pict_type;
1369                 }
1370             }
1371
1372             if (s->b_frame_strategy == 0) {
1373                 b_frames = s->max_b_frames;
1374                 while (b_frames && !s->input_picture[b_frames])
1375                     b_frames--;
1376             } else if (s->b_frame_strategy == 1) {
1377                 for (i = 1; i < s->max_b_frames + 1; i++) {
1378                     if (s->input_picture[i] &&
1379                         s->input_picture[i]->b_frame_score == 0) {
1380                         s->input_picture[i]->b_frame_score =
1381                             get_intra_count(s,
1382                                             s->input_picture[i    ]->f->data[0],
1383                                             s->input_picture[i - 1]->f->data[0],
1384                                             s->linesize) + 1;
1385                     }
1386                 }
1387                 for (i = 0; i < s->max_b_frames + 1; i++) {
1388                     if (!s->input_picture[i] ||
1389                         s->input_picture[i]->b_frame_score - 1 >
1390                             s->mb_num / s->b_sensitivity)
1391                         break;
1392                 }
1393
1394                 b_frames = FFMAX(0, i - 1);
1395
1396                 /* reset scores */
1397                 for (i = 0; i < b_frames + 1; i++) {
1398                     s->input_picture[i]->b_frame_score = 0;
1399                 }
1400             } else if (s->b_frame_strategy == 2) {
1401                 b_frames = estimate_best_b_count(s);
1402             }
1403
1404             emms_c();
1405
1406             for (i = b_frames - 1; i >= 0; i--) {
1407                 int type = s->input_picture[i]->f->pict_type;
1408                 if (type && type != AV_PICTURE_TYPE_B)
1409                     b_frames = i;
1410             }
1411             if (s->input_picture[b_frames]->f->pict_type == AV_PICTURE_TYPE_B &&
1412                 b_frames == s->max_b_frames) {
1413                 av_log(s->avctx, AV_LOG_ERROR,
1414                        "warning, too many b frames in a row\n");
1415             }
1416
1417             if (s->picture_in_gop_number + b_frames >= s->gop_size) {
1418                 if ((s->mpv_flags & FF_MPV_FLAG_STRICT_GOP) &&
1419                     s->gop_size > s->picture_in_gop_number) {
1420                     b_frames = s->gop_size - s->picture_in_gop_number - 1;
1421                 } else {
1422                     if (s->avctx->flags & AV_CODEC_FLAG_CLOSED_GOP)
1423                         b_frames = 0;
1424                     s->input_picture[b_frames]->f->pict_type = AV_PICTURE_TYPE_I;
1425                 }
1426             }
1427
1428             if ((s->avctx->flags & AV_CODEC_FLAG_CLOSED_GOP) && b_frames &&
1429                 s->input_picture[b_frames]->f->pict_type == AV_PICTURE_TYPE_I)
1430                 b_frames--;
1431
1432             s->reordered_input_picture[0] = s->input_picture[b_frames];
1433             if (s->reordered_input_picture[0]->f->pict_type != AV_PICTURE_TYPE_I)
1434                 s->reordered_input_picture[0]->f->pict_type = AV_PICTURE_TYPE_P;
1435             s->reordered_input_picture[0]->f->coded_picture_number =
1436                 s->coded_picture_number++;
1437             for (i = 0; i < b_frames; i++) {
1438                 s->reordered_input_picture[i + 1] = s->input_picture[i];
1439                 s->reordered_input_picture[i + 1]->f->pict_type =
1440                     AV_PICTURE_TYPE_B;
1441                 s->reordered_input_picture[i + 1]->f->coded_picture_number =
1442                     s->coded_picture_number++;
1443             }
1444         }
1445     }
1446 no_output_pic:
1447     ff_mpeg_unref_picture(s->avctx, &s->new_picture);
1448
1449     if (s->reordered_input_picture[0]) {
1450         s->reordered_input_picture[0]->reference =
1451            s->reordered_input_picture[0]->f->pict_type !=
1452                AV_PICTURE_TYPE_B ? 3 : 0;
1453
1454         if ((ret = ff_mpeg_ref_picture(s->avctx, &s->new_picture, s->reordered_input_picture[0])))
1455             return ret;
1456
1457         if (s->reordered_input_picture[0]->shared || s->avctx->rc_buffer_size) {
1458             // input is a shared pix, so we can't modifiy it -> alloc a new
1459             // one & ensure that the shared one is reuseable
1460
1461             Picture *pic;
1462             int i = ff_find_unused_picture(s->avctx, s->picture, 0);
1463             if (i < 0)
1464                 return i;
1465             pic = &s->picture[i];
1466
1467             pic->reference = s->reordered_input_picture[0]->reference;
1468             if (alloc_picture(s, pic, 0) < 0) {
1469                 return -1;
1470             }
1471
1472             ret = av_frame_copy_props(pic->f, s->reordered_input_picture[0]->f);
1473             if (ret < 0)
1474                 return ret;
1475
1476             /* mark us unused / free shared pic */
1477             av_frame_unref(s->reordered_input_picture[0]->f);
1478             s->reordered_input_picture[0]->shared = 0;
1479
1480             s->current_picture_ptr = pic;
1481         } else {
1482             // input is not a shared pix -> reuse buffer for current_pix
1483             s->current_picture_ptr = s->reordered_input_picture[0];
1484             for (i = 0; i < 4; i++) {
1485                 s->new_picture.f->data[i] += INPLACE_OFFSET;
1486             }
1487         }
1488         ff_mpeg_unref_picture(s->avctx, &s->current_picture);
1489         if ((ret = ff_mpeg_ref_picture(s->avctx, &s->current_picture,
1490                                        s->current_picture_ptr)) < 0)
1491             return ret;
1492
1493         s->picture_number = s->new_picture.f->display_picture_number;
1494     }
1495     return 0;
1496 }
1497
1498 static void frame_end(MpegEncContext *s)
1499 {
1500     int i;
1501
1502     if (s->unrestricted_mv &&
1503         s->current_picture.reference &&
1504         !s->intra_only) {
1505         const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(s->avctx->pix_fmt);
1506         int hshift = desc->log2_chroma_w;
1507         int vshift = desc->log2_chroma_h;
1508         s->mpvencdsp.draw_edges(s->current_picture.f->data[0], s->linesize,
1509                                 s->h_edge_pos, s->v_edge_pos,
1510                                 EDGE_WIDTH, EDGE_WIDTH,
1511                                 EDGE_TOP | EDGE_BOTTOM);
1512         s->mpvencdsp.draw_edges(s->current_picture.f->data[1], s->uvlinesize,
1513                                 s->h_edge_pos >> hshift,
1514                                 s->v_edge_pos >> vshift,
1515                                 EDGE_WIDTH >> hshift,
1516                                 EDGE_WIDTH >> vshift,
1517                                 EDGE_TOP | EDGE_BOTTOM);
1518         s->mpvencdsp.draw_edges(s->current_picture.f->data[2], s->uvlinesize,
1519                                 s->h_edge_pos >> hshift,
1520                                 s->v_edge_pos >> vshift,
1521                                 EDGE_WIDTH >> hshift,
1522                                 EDGE_WIDTH >> vshift,
1523                                 EDGE_TOP | EDGE_BOTTOM);
1524     }
1525
1526     emms_c();
1527
1528     s->last_pict_type                 = s->pict_type;
1529     s->last_lambda_for [s->pict_type] = s->current_picture_ptr->f->quality;
1530     if (s->pict_type!= AV_PICTURE_TYPE_B)
1531         s->last_non_b_pict_type = s->pict_type;
1532
1533     if (s->encoding) {
1534         /* release non-reference frames */
1535         for (i = 0; i < MAX_PICTURE_COUNT; i++) {
1536             if (!s->picture[i].reference)
1537                 ff_mpeg_unref_picture(s->avctx, &s->picture[i]);
1538         }
1539     }
1540
1541 #if FF_API_CODED_FRAME
1542 FF_DISABLE_DEPRECATION_WARNINGS
1543     av_frame_copy_props(s->avctx->coded_frame, s->current_picture.f);
1544 FF_ENABLE_DEPRECATION_WARNINGS
1545 #endif
1546 #if FF_API_ERROR_FRAME
1547 FF_DISABLE_DEPRECATION_WARNINGS
1548     memcpy(s->current_picture.f->error, s->current_picture.encoding_error,
1549            sizeof(s->current_picture.encoding_error));
1550 FF_ENABLE_DEPRECATION_WARNINGS
1551 #endif
1552 }
1553
1554 static void update_noise_reduction(MpegEncContext *s)
1555 {
1556     int intra, i;
1557
1558     for (intra = 0; intra < 2; intra++) {
1559         if (s->dct_count[intra] > (1 << 16)) {
1560             for (i = 0; i < 64; i++) {
1561                 s->dct_error_sum[intra][i] >>= 1;
1562             }
1563             s->dct_count[intra] >>= 1;
1564         }
1565
1566         for (i = 0; i < 64; i++) {
1567             s->dct_offset[intra][i] = (s->avctx->noise_reduction *
1568                                        s->dct_count[intra] +
1569                                        s->dct_error_sum[intra][i] / 2) /
1570                                       (s->dct_error_sum[intra][i] + 1);
1571         }
1572     }
1573 }
1574
1575 static int frame_start(MpegEncContext *s)
1576 {
1577     int ret;
1578
1579     /* mark & release old frames */
1580     if (s->pict_type != AV_PICTURE_TYPE_B && s->last_picture_ptr &&
1581         s->last_picture_ptr != s->next_picture_ptr &&
1582         s->last_picture_ptr->f->buf[0]) {
1583         ff_mpeg_unref_picture(s->avctx, s->last_picture_ptr);
1584     }
1585
1586     s->current_picture_ptr->f->pict_type = s->pict_type;
1587     s->current_picture_ptr->f->key_frame = s->pict_type == AV_PICTURE_TYPE_I;
1588
1589     ff_mpeg_unref_picture(s->avctx, &s->current_picture);
1590     if ((ret = ff_mpeg_ref_picture(s->avctx, &s->current_picture,
1591                                    s->current_picture_ptr)) < 0)
1592         return ret;
1593
1594     if (s->pict_type != AV_PICTURE_TYPE_B) {
1595         s->last_picture_ptr = s->next_picture_ptr;
1596         if (!s->droppable)
1597             s->next_picture_ptr = s->current_picture_ptr;
1598     }
1599
1600     if (s->last_picture_ptr) {
1601         ff_mpeg_unref_picture(s->avctx, &s->last_picture);
1602         if (s->last_picture_ptr->f->buf[0] &&
1603             (ret = ff_mpeg_ref_picture(s->avctx, &s->last_picture,
1604                                        s->last_picture_ptr)) < 0)
1605             return ret;
1606     }
1607     if (s->next_picture_ptr) {
1608         ff_mpeg_unref_picture(s->avctx, &s->next_picture);
1609         if (s->next_picture_ptr->f->buf[0] &&
1610             (ret = ff_mpeg_ref_picture(s->avctx, &s->next_picture,
1611                                        s->next_picture_ptr)) < 0)
1612             return ret;
1613     }
1614
1615     if (s->picture_structure!= PICT_FRAME) {
1616         int i;
1617         for (i = 0; i < 4; i++) {
1618             if (s->picture_structure == PICT_BOTTOM_FIELD) {
1619                 s->current_picture.f->data[i] +=
1620                     s->current_picture.f->linesize[i];
1621             }
1622             s->current_picture.f->linesize[i] *= 2;
1623             s->last_picture.f->linesize[i]    *= 2;
1624             s->next_picture.f->linesize[i]    *= 2;
1625         }
1626     }
1627
1628     if (s->mpeg_quant || s->codec_id == AV_CODEC_ID_MPEG2VIDEO) {
1629         s->dct_unquantize_intra = s->dct_unquantize_mpeg2_intra;
1630         s->dct_unquantize_inter = s->dct_unquantize_mpeg2_inter;
1631     } else if (s->out_format == FMT_H263 || s->out_format == FMT_H261) {
1632         s->dct_unquantize_intra = s->dct_unquantize_h263_intra;
1633         s->dct_unquantize_inter = s->dct_unquantize_h263_inter;
1634     } else {
1635         s->dct_unquantize_intra = s->dct_unquantize_mpeg1_intra;
1636         s->dct_unquantize_inter = s->dct_unquantize_mpeg1_inter;
1637     }
1638
1639     if (s->dct_error_sum) {
1640         assert(s->avctx->noise_reduction && s->encoding);
1641         update_noise_reduction(s);
1642     }
1643
1644     return 0;
1645 }
1646
1647 int ff_mpv_encode_picture(AVCodecContext *avctx, AVPacket *pkt,
1648                           const AVFrame *pic_arg, int *got_packet)
1649 {
1650     MpegEncContext *s = avctx->priv_data;
1651     int i, stuffing_count, ret;
1652     int context_count = s->slice_context_count;
1653
1654     s->picture_in_gop_number++;
1655
1656     if (load_input_picture(s, pic_arg) < 0)
1657         return -1;
1658
1659     if (select_input_picture(s) < 0) {
1660         return -1;
1661     }
1662
1663     /* output? */
1664     if (s->new_picture.f->data[0]) {
1665         uint8_t *sd;
1666         if (!pkt->data &&
1667             (ret = ff_alloc_packet(pkt, s->mb_width*s->mb_height*MAX_MB_BYTES)) < 0)
1668             return ret;
1669         if (s->mb_info) {
1670             s->mb_info_ptr = av_packet_new_side_data(pkt,
1671                                  AV_PKT_DATA_H263_MB_INFO,
1672                                  s->mb_width*s->mb_height*12);
1673             s->prev_mb_info = s->last_mb_info = s->mb_info_size = 0;
1674         }
1675
1676         for (i = 0; i < context_count; i++) {
1677             int start_y = s->thread_context[i]->start_mb_y;
1678             int   end_y = s->thread_context[i]->  end_mb_y;
1679             int h       = s->mb_height;
1680             uint8_t *start = pkt->data + (size_t)(((int64_t) pkt->size) * start_y / h);
1681             uint8_t *end   = pkt->data + (size_t)(((int64_t) pkt->size) *   end_y / h);
1682
1683             init_put_bits(&s->thread_context[i]->pb, start, end - start);
1684         }
1685
1686         s->pict_type = s->new_picture.f->pict_type;
1687         //emms_c();
1688         ret = frame_start(s);
1689         if (ret < 0)
1690             return ret;
1691 vbv_retry:
1692         if (encode_picture(s, s->picture_number) < 0)
1693             return -1;
1694
1695 #if FF_API_STAT_BITS
1696 FF_DISABLE_DEPRECATION_WARNINGS
1697         avctx->header_bits = s->header_bits;
1698         avctx->mv_bits     = s->mv_bits;
1699         avctx->misc_bits   = s->misc_bits;
1700         avctx->i_tex_bits  = s->i_tex_bits;
1701         avctx->p_tex_bits  = s->p_tex_bits;
1702         avctx->i_count     = s->i_count;
1703         // FIXME f/b_count in avctx
1704         avctx->p_count     = s->mb_num - s->i_count - s->skip_count;
1705         avctx->skip_count  = s->skip_count;
1706 FF_ENABLE_DEPRECATION_WARNINGS
1707 #endif
1708
1709         frame_end(s);
1710
1711         sd = av_packet_new_side_data(pkt, AV_PKT_DATA_QUALITY_FACTOR,
1712                                      sizeof(int));
1713         if (!sd)
1714             return AVERROR(ENOMEM);
1715         *(int *)sd = s->current_picture.f->quality;
1716
1717         if (CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG)
1718             ff_mjpeg_encode_picture_trailer(&s->pb, s->header_bits);
1719
1720         if (avctx->rc_buffer_size) {
1721             RateControlContext *rcc = &s->rc_context;
1722             int max_size = rcc->buffer_index * avctx->rc_max_available_vbv_use;
1723
1724             if (put_bits_count(&s->pb) > max_size &&
1725                 s->lambda < s->lmax) {
1726                 s->next_lambda = FFMAX(s->lambda + 1, s->lambda *
1727                                        (s->qscale + 1) / s->qscale);
1728                 if (s->adaptive_quant) {
1729                     int i;
1730                     for (i = 0; i < s->mb_height * s->mb_stride; i++)
1731                         s->lambda_table[i] =
1732                             FFMAX(s->lambda_table[i] + 1,
1733                                   s->lambda_table[i] * (s->qscale + 1) /
1734                                   s->qscale);
1735                 }
1736                 s->mb_skipped = 0;        // done in frame_start()
1737                 // done in encode_picture() so we must undo it
1738                 if (s->pict_type == AV_PICTURE_TYPE_P) {
1739                     if (s->flipflop_rounding          ||
1740                         s->codec_id == AV_CODEC_ID_H263P ||
1741                         s->codec_id == AV_CODEC_ID_MPEG4)
1742                         s->no_rounding ^= 1;
1743                 }
1744                 if (s->pict_type != AV_PICTURE_TYPE_B) {
1745                     s->time_base       = s->last_time_base;
1746                     s->last_non_b_time = s->time - s->pp_time;
1747                 }
1748                 for (i = 0; i < context_count; i++) {
1749                     PutBitContext *pb = &s->thread_context[i]->pb;
1750                     init_put_bits(pb, pb->buf, pb->buf_end - pb->buf);
1751                 }
1752                 goto vbv_retry;
1753             }
1754
1755             assert(s->avctx->rc_max_rate);
1756         }
1757
1758         if (s->avctx->flags & AV_CODEC_FLAG_PASS1)
1759             ff_write_pass1_stats(s);
1760
1761         for (i = 0; i < 4; i++) {
1762             s->current_picture_ptr->encoding_error[i] = s->current_picture.encoding_error[i];
1763             avctx->error[i] += s->current_picture_ptr->encoding_error[i];
1764         }
1765
1766         if (s->avctx->flags & AV_CODEC_FLAG_PASS1)
1767             assert(put_bits_count(&s->pb) == s->header_bits + s->mv_bits +
1768                                              s->misc_bits + s->i_tex_bits +
1769                                              s->p_tex_bits);
1770         flush_put_bits(&s->pb);
1771         s->frame_bits  = put_bits_count(&s->pb);
1772
1773         stuffing_count = ff_vbv_update(s, s->frame_bits);
1774         if (stuffing_count) {
1775             if (s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb) >> 3) <
1776                     stuffing_count + 50) {
1777                 av_log(s->avctx, AV_LOG_ERROR, "stuffing too large\n");
1778                 return -1;
1779             }
1780
1781             switch (s->codec_id) {
1782             case AV_CODEC_ID_MPEG1VIDEO:
1783             case AV_CODEC_ID_MPEG2VIDEO:
1784                 while (stuffing_count--) {
1785                     put_bits(&s->pb, 8, 0);
1786                 }
1787             break;
1788             case AV_CODEC_ID_MPEG4:
1789                 put_bits(&s->pb, 16, 0);
1790                 put_bits(&s->pb, 16, 0x1C3);
1791                 stuffing_count -= 4;
1792                 while (stuffing_count--) {
1793                     put_bits(&s->pb, 8, 0xFF);
1794                 }
1795             break;
1796             default:
1797                 av_log(s->avctx, AV_LOG_ERROR, "vbv buffer overflow\n");
1798             }
1799             flush_put_bits(&s->pb);
1800             s->frame_bits  = put_bits_count(&s->pb);
1801         }
1802
1803         /* update mpeg1/2 vbv_delay for CBR */
1804         if (s->avctx->rc_max_rate                          &&
1805             s->avctx->rc_min_rate == s->avctx->rc_max_rate &&
1806             s->out_format == FMT_MPEG1                     &&
1807             90000LL * (avctx->rc_buffer_size - 1) <=
1808                 s->avctx->rc_max_rate * 0xFFFFLL) {
1809             AVCPBProperties *props;
1810             size_t props_size;
1811
1812             int vbv_delay, min_delay;
1813             double inbits  = s->avctx->rc_max_rate *
1814                              av_q2d(s->avctx->time_base);
1815             int    minbits = s->frame_bits - 8 *
1816                              (s->vbv_delay_ptr - s->pb.buf - 1);
1817             double bits    = s->rc_context.buffer_index + minbits - inbits;
1818
1819             if (bits < 0)
1820                 av_log(s->avctx, AV_LOG_ERROR,
1821                        "Internal error, negative bits\n");
1822
1823             assert(s->repeat_first_field == 0);
1824
1825             vbv_delay = bits * 90000 / s->avctx->rc_max_rate;
1826             min_delay = (minbits * 90000LL + s->avctx->rc_max_rate - 1) /
1827                         s->avctx->rc_max_rate;
1828
1829             vbv_delay = FFMAX(vbv_delay, min_delay);
1830
1831             assert(vbv_delay < 0xFFFF);
1832
1833             s->vbv_delay_ptr[0] &= 0xF8;
1834             s->vbv_delay_ptr[0] |= vbv_delay >> 13;
1835             s->vbv_delay_ptr[1]  = vbv_delay >> 5;
1836             s->vbv_delay_ptr[2] &= 0x07;
1837             s->vbv_delay_ptr[2] |= vbv_delay << 3;
1838
1839             props = av_cpb_properties_alloc(&props_size);
1840             if (!props)
1841                 return AVERROR(ENOMEM);
1842             props->vbv_delay = vbv_delay * 300;
1843
1844             ret = av_packet_add_side_data(pkt, AV_PKT_DATA_CPB_PROPERTIES,
1845                                           (uint8_t*)props, props_size);
1846             if (ret < 0) {
1847                 av_freep(&props);
1848                 return ret;
1849             }
1850
1851 #if FF_API_VBV_DELAY
1852 FF_DISABLE_DEPRECATION_WARNINGS
1853             avctx->vbv_delay     = vbv_delay * 300;
1854 FF_ENABLE_DEPRECATION_WARNINGS
1855 #endif
1856         }
1857         s->total_bits     += s->frame_bits;
1858 #if FF_API_STAT_BITS
1859 FF_DISABLE_DEPRECATION_WARNINGS
1860         avctx->frame_bits  = s->frame_bits;
1861 FF_ENABLE_DEPRECATION_WARNINGS
1862 #endif
1863
1864
1865         pkt->pts = s->current_picture.f->pts;
1866         if (!s->low_delay && s->pict_type != AV_PICTURE_TYPE_B) {
1867             if (!s->current_picture.f->coded_picture_number)
1868                 pkt->dts = pkt->pts - s->dts_delta;
1869             else
1870                 pkt->dts = s->reordered_pts;
1871             s->reordered_pts = pkt->pts;
1872         } else
1873             pkt->dts = pkt->pts;
1874         if (s->current_picture.f->key_frame)
1875             pkt->flags |= AV_PKT_FLAG_KEY;
1876         if (s->mb_info)
1877             av_packet_shrink_side_data(pkt, AV_PKT_DATA_H263_MB_INFO, s->mb_info_size);
1878     } else {
1879         s->frame_bits = 0;
1880     }
1881     assert((s->frame_bits & 7) == 0);
1882
1883     pkt->size = s->frame_bits / 8;
1884     *got_packet = !!pkt->size;
1885     return 0;
1886 }
1887
1888 static inline void dct_single_coeff_elimination(MpegEncContext *s,
1889                                                 int n, int threshold)
1890 {
1891     static const char tab[64] = {
1892         3, 2, 2, 1, 1, 1, 1, 1,
1893         1, 1, 1, 1, 1, 1, 1, 1,
1894         1, 1, 1, 1, 1, 1, 1, 1,
1895         0, 0, 0, 0, 0, 0, 0, 0,
1896         0, 0, 0, 0, 0, 0, 0, 0,
1897         0, 0, 0, 0, 0, 0, 0, 0,
1898         0, 0, 0, 0, 0, 0, 0, 0,
1899         0, 0, 0, 0, 0, 0, 0, 0
1900     };
1901     int score = 0;
1902     int run = 0;
1903     int i;
1904     int16_t *block = s->block[n];
1905     const int last_index = s->block_last_index[n];
1906     int skip_dc;
1907
1908     if (threshold < 0) {
1909         skip_dc = 0;
1910         threshold = -threshold;
1911     } else
1912         skip_dc = 1;
1913
1914     /* Are all we could set to zero already zero? */
1915     if (last_index <= skip_dc - 1)
1916         return;
1917
1918     for (i = 0; i <= last_index; i++) {
1919         const int j = s->intra_scantable.permutated[i];
1920         const int level = FFABS(block[j]);
1921         if (level == 1) {
1922             if (skip_dc && i == 0)
1923                 continue;
1924             score += tab[run];
1925             run = 0;
1926         } else if (level > 1) {
1927             return;
1928         } else {
1929             run++;
1930         }
1931     }
1932     if (score >= threshold)
1933         return;
1934     for (i = skip_dc; i <= last_index; i++) {
1935         const int j = s->intra_scantable.permutated[i];
1936         block[j] = 0;
1937     }
1938     if (block[0])
1939         s->block_last_index[n] = 0;
1940     else
1941         s->block_last_index[n] = -1;
1942 }
1943
1944 static inline void clip_coeffs(MpegEncContext *s, int16_t *block,
1945                                int last_index)
1946 {
1947     int i;
1948     const int maxlevel = s->max_qcoeff;
1949     const int minlevel = s->min_qcoeff;
1950     int overflow = 0;
1951
1952     if (s->mb_intra) {
1953         i = 1; // skip clipping of intra dc
1954     } else
1955         i = 0;
1956
1957     for (; i <= last_index; i++) {
1958         const int j = s->intra_scantable.permutated[i];
1959         int level = block[j];
1960
1961         if (level > maxlevel) {
1962             level = maxlevel;
1963             overflow++;
1964         } else if (level < minlevel) {
1965             level = minlevel;
1966             overflow++;
1967         }
1968
1969         block[j] = level;
1970     }
1971
1972     if (overflow && s->avctx->mb_decision == FF_MB_DECISION_SIMPLE)
1973         av_log(s->avctx, AV_LOG_INFO,
1974                "warning, clipping %d dct coefficients to %d..%d\n",
1975                overflow, minlevel, maxlevel);
1976 }
1977
1978 static void get_visual_weight(int16_t *weight, uint8_t *ptr, int stride)
1979 {
1980     int x, y;
1981     // FIXME optimize
1982     for (y = 0; y < 8; y++) {
1983         for (x = 0; x < 8; x++) {
1984             int x2, y2;
1985             int sum = 0;
1986             int sqr = 0;
1987             int count = 0;
1988
1989             for (y2 = FFMAX(y - 1, 0); y2 < FFMIN(8, y + 2); y2++) {
1990                 for (x2= FFMAX(x - 1, 0); x2 < FFMIN(8, x + 2); x2++) {
1991                     int v = ptr[x2 + y2 * stride];
1992                     sum += v;
1993                     sqr += v * v;
1994                     count++;
1995                 }
1996             }
1997             weight[x + 8 * y]= (36 * ff_sqrt(count * sqr - sum * sum)) / count;
1998         }
1999     }
2000 }
2001
2002 static av_always_inline void encode_mb_internal(MpegEncContext *s,
2003                                                 int motion_x, int motion_y,
2004                                                 int mb_block_height,
2005                                                 int mb_block_count)
2006 {
2007     int16_t weight[8][64];
2008     int16_t orig[8][64];
2009     const int mb_x = s->mb_x;
2010     const int mb_y = s->mb_y;
2011     int i;
2012     int skip_dct[8];
2013     int dct_offset = s->linesize * 8; // default for progressive frames
2014     uint8_t *ptr_y, *ptr_cb, *ptr_cr;
2015     ptrdiff_t wrap_y, wrap_c;
2016
2017     for (i = 0; i < mb_block_count; i++)
2018         skip_dct[i] = s->skipdct;
2019
2020     if (s->adaptive_quant) {
2021         const int last_qp = s->qscale;
2022         const int mb_xy = mb_x + mb_y * s->mb_stride;
2023
2024         s->lambda = s->lambda_table[mb_xy];
2025         update_qscale(s);
2026
2027         if (!(s->mpv_flags & FF_MPV_FLAG_QP_RD)) {
2028             s->qscale = s->current_picture_ptr->qscale_table[mb_xy];
2029             s->dquant = s->qscale - last_qp;
2030
2031             if (s->out_format == FMT_H263) {
2032                 s->dquant = av_clip(s->dquant, -2, 2);
2033
2034                 if (s->codec_id == AV_CODEC_ID_MPEG4) {
2035                     if (!s->mb_intra) {
2036                         if (s->pict_type == AV_PICTURE_TYPE_B) {
2037                             if (s->dquant & 1 || s->mv_dir & MV_DIRECT)
2038                                 s->dquant = 0;
2039                         }
2040                         if (s->mv_type == MV_TYPE_8X8)
2041                             s->dquant = 0;
2042                     }
2043                 }
2044             }
2045         }
2046         ff_set_qscale(s, last_qp + s->dquant);
2047     } else if (s->mpv_flags & FF_MPV_FLAG_QP_RD)
2048         ff_set_qscale(s, s->qscale + s->dquant);
2049
2050     wrap_y = s->linesize;
2051     wrap_c = s->uvlinesize;
2052     ptr_y  = s->new_picture.f->data[0] +
2053              (mb_y * 16 * wrap_y)              + mb_x * 16;
2054     ptr_cb = s->new_picture.f->data[1] +
2055              (mb_y * mb_block_height * wrap_c) + mb_x * 8;
2056     ptr_cr = s->new_picture.f->data[2] +
2057              (mb_y * mb_block_height * wrap_c) + mb_x * 8;
2058
2059     if (mb_x * 16 + 16 > s->width || mb_y * 16 + 16 > s->height) {
2060         uint8_t *ebuf = s->sc.edge_emu_buffer + 32;
2061         s->vdsp.emulated_edge_mc(ebuf, ptr_y,
2062                                  wrap_y, wrap_y,
2063                                  16, 16, mb_x * 16, mb_y * 16,
2064                                  s->width, s->height);
2065         ptr_y = ebuf;
2066         s->vdsp.emulated_edge_mc(ebuf + 18 * wrap_y, ptr_cb,
2067                                  wrap_c, wrap_c,
2068                                  8, mb_block_height, mb_x * 8, mb_y * 8,
2069                                  s->width >> 1, s->height >> 1);
2070         ptr_cb = ebuf + 18 * wrap_y;
2071         s->vdsp.emulated_edge_mc(ebuf + 18 * wrap_y + 8, ptr_cr,
2072                                  wrap_c, wrap_c,
2073                                  8, mb_block_height, mb_x * 8, mb_y * 8,
2074                                  s->width >> 1, s->height >> 1);
2075         ptr_cr = ebuf + 18 * wrap_y + 8;
2076     }
2077
2078     if (s->mb_intra) {
2079         if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_DCT) {
2080             int progressive_score, interlaced_score;
2081
2082             s->interlaced_dct = 0;
2083             progressive_score = s->mecc.ildct_cmp[4](s, ptr_y, NULL, wrap_y, 8) +
2084                                 s->mecc.ildct_cmp[4](s, ptr_y + wrap_y * 8,
2085                                                      NULL, wrap_y, 8) - 400;
2086
2087             if (progressive_score > 0) {
2088                 interlaced_score = s->mecc.ildct_cmp[4](s, ptr_y,
2089                                                         NULL, wrap_y * 2, 8) +
2090                                    s->mecc.ildct_cmp[4](s, ptr_y + wrap_y,
2091                                                         NULL, wrap_y * 2, 8);
2092                 if (progressive_score > interlaced_score) {
2093                     s->interlaced_dct = 1;
2094
2095                     dct_offset = wrap_y;
2096                     wrap_y <<= 1;
2097                     if (s->chroma_format == CHROMA_422)
2098                         wrap_c <<= 1;
2099                 }
2100             }
2101         }
2102
2103         s->pdsp.get_pixels(s->block[0], ptr_y,                  wrap_y);
2104         s->pdsp.get_pixels(s->block[1], ptr_y + 8,              wrap_y);
2105         s->pdsp.get_pixels(s->block[2], ptr_y + dct_offset,     wrap_y);
2106         s->pdsp.get_pixels(s->block[3], ptr_y + dct_offset + 8, wrap_y);
2107
2108         if (s->avctx->flags & AV_CODEC_FLAG_GRAY) {
2109             skip_dct[4] = 1;
2110             skip_dct[5] = 1;
2111         } else {
2112             s->pdsp.get_pixels(s->block[4], ptr_cb, wrap_c);
2113             s->pdsp.get_pixels(s->block[5], ptr_cr, wrap_c);
2114             if (!s->chroma_y_shift) { /* 422 */
2115                 s->pdsp.get_pixels(s->block[6],
2116                                    ptr_cb + (dct_offset >> 1), wrap_c);
2117                 s->pdsp.get_pixels(s->block[7],
2118                                    ptr_cr + (dct_offset >> 1), wrap_c);
2119             }
2120         }
2121     } else {
2122         op_pixels_func (*op_pix)[4];
2123         qpel_mc_func (*op_qpix)[16];
2124         uint8_t *dest_y, *dest_cb, *dest_cr;
2125
2126         dest_y  = s->dest[0];
2127         dest_cb = s->dest[1];
2128         dest_cr = s->dest[2];
2129
2130         if ((!s->no_rounding) || s->pict_type == AV_PICTURE_TYPE_B) {
2131             op_pix  = s->hdsp.put_pixels_tab;
2132             op_qpix = s->qdsp.put_qpel_pixels_tab;
2133         } else {
2134             op_pix  = s->hdsp.put_no_rnd_pixels_tab;
2135             op_qpix = s->qdsp.put_no_rnd_qpel_pixels_tab;
2136         }
2137
2138         if (s->mv_dir & MV_DIR_FORWARD) {
2139             ff_mpv_motion(s, dest_y, dest_cb, dest_cr, 0,
2140                           s->last_picture.f->data,
2141                           op_pix, op_qpix);
2142             op_pix  = s->hdsp.avg_pixels_tab;
2143             op_qpix = s->qdsp.avg_qpel_pixels_tab;
2144         }
2145         if (s->mv_dir & MV_DIR_BACKWARD) {
2146             ff_mpv_motion(s, dest_y, dest_cb, dest_cr, 1,
2147                           s->next_picture.f->data,
2148                           op_pix, op_qpix);
2149         }
2150
2151         if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_DCT) {
2152             int progressive_score, interlaced_score;
2153
2154             s->interlaced_dct = 0;
2155             progressive_score = s->mecc.ildct_cmp[0](s, dest_y, ptr_y, wrap_y, 8) +
2156                                 s->mecc.ildct_cmp[0](s, dest_y + wrap_y * 8,
2157                                                      ptr_y + wrap_y * 8,
2158                                                      wrap_y, 8) - 400;
2159
2160             if (s->avctx->ildct_cmp == FF_CMP_VSSE)
2161                 progressive_score -= 400;
2162
2163             if (progressive_score > 0) {
2164                 interlaced_score = s->mecc.ildct_cmp[0](s, dest_y, ptr_y,
2165                                                         wrap_y * 2, 8) +
2166                                    s->mecc.ildct_cmp[0](s, dest_y + wrap_y,
2167                                                         ptr_y + wrap_y,
2168                                                         wrap_y * 2, 8);
2169
2170                 if (progressive_score > interlaced_score) {
2171                     s->interlaced_dct = 1;
2172
2173                     dct_offset = wrap_y;
2174                     wrap_y <<= 1;
2175                     if (s->chroma_format == CHROMA_422)
2176                         wrap_c <<= 1;
2177                 }
2178             }
2179         }
2180
2181         s->pdsp.diff_pixels(s->block[0], ptr_y, dest_y, wrap_y);
2182         s->pdsp.diff_pixels(s->block[1], ptr_y + 8, dest_y + 8, wrap_y);
2183         s->pdsp.diff_pixels(s->block[2], ptr_y + dct_offset,
2184                             dest_y + dct_offset, wrap_y);
2185         s->pdsp.diff_pixels(s->block[3], ptr_y + dct_offset + 8,
2186                             dest_y + dct_offset + 8, wrap_y);
2187
2188         if (s->avctx->flags & AV_CODEC_FLAG_GRAY) {
2189             skip_dct[4] = 1;
2190             skip_dct[5] = 1;
2191         } else {
2192             s->pdsp.diff_pixels(s->block[4], ptr_cb, dest_cb, wrap_c);
2193             s->pdsp.diff_pixels(s->block[5], ptr_cr, dest_cr, wrap_c);
2194             if (!s->chroma_y_shift) { /* 422 */
2195                 s->pdsp.diff_pixels(s->block[6], ptr_cb + (dct_offset >> 1),
2196                                     dest_cb + (dct_offset >> 1), wrap_c);
2197                 s->pdsp.diff_pixels(s->block[7], ptr_cr + (dct_offset >> 1),
2198                                     dest_cr + (dct_offset >> 1), wrap_c);
2199             }
2200         }
2201         /* pre quantization */
2202         if (s->current_picture.mc_mb_var[s->mb_stride * mb_y + mb_x] <
2203                 2 * s->qscale * s->qscale) {
2204             // FIXME optimize
2205             if (s->mecc.sad[1](NULL, ptr_y, dest_y, wrap_y, 8) < 20 * s->qscale)
2206                 skip_dct[0] = 1;
2207             if (s->mecc.sad[1](NULL, ptr_y + 8, dest_y + 8, wrap_y, 8) < 20 * s->qscale)
2208                 skip_dct[1] = 1;
2209             if (s->mecc.sad[1](NULL, ptr_y + dct_offset, dest_y + dct_offset,
2210                                wrap_y, 8) < 20 * s->qscale)
2211                 skip_dct[2] = 1;
2212             if (s->mecc.sad[1](NULL, ptr_y + dct_offset + 8, dest_y + dct_offset + 8,
2213                                wrap_y, 8) < 20 * s->qscale)
2214                 skip_dct[3] = 1;
2215             if (s->mecc.sad[1](NULL, ptr_cb, dest_cb, wrap_c, 8) < 20 * s->qscale)
2216                 skip_dct[4] = 1;
2217             if (s->mecc.sad[1](NULL, ptr_cr, dest_cr, wrap_c, 8) < 20 * s->qscale)
2218                 skip_dct[5] = 1;
2219             if (!s->chroma_y_shift) { /* 422 */
2220                 if (s->mecc.sad[1](NULL, ptr_cb + (dct_offset >> 1),
2221                                    dest_cb + (dct_offset >> 1),
2222                                    wrap_c, 8) < 20 * s->qscale)
2223                     skip_dct[6] = 1;
2224                 if (s->mecc.sad[1](NULL, ptr_cr + (dct_offset >> 1),
2225                                    dest_cr + (dct_offset >> 1),
2226                                    wrap_c, 8) < 20 * s->qscale)
2227                     skip_dct[7] = 1;
2228             }
2229         }
2230     }
2231
2232     if (s->quantizer_noise_shaping) {
2233         if (!skip_dct[0])
2234             get_visual_weight(weight[0], ptr_y                 , wrap_y);
2235         if (!skip_dct[1])
2236             get_visual_weight(weight[1], ptr_y              + 8, wrap_y);
2237         if (!skip_dct[2])
2238             get_visual_weight(weight[2], ptr_y + dct_offset    , wrap_y);
2239         if (!skip_dct[3])
2240             get_visual_weight(weight[3], ptr_y + dct_offset + 8, wrap_y);
2241         if (!skip_dct[4])
2242             get_visual_weight(weight[4], ptr_cb                , wrap_c);
2243         if (!skip_dct[5])
2244             get_visual_weight(weight[5], ptr_cr                , wrap_c);
2245         if (!s->chroma_y_shift) { /* 422 */
2246             if (!skip_dct[6])
2247                 get_visual_weight(weight[6], ptr_cb + (dct_offset >> 1),
2248                                   wrap_c);
2249             if (!skip_dct[7])
2250                 get_visual_weight(weight[7], ptr_cr + (dct_offset >> 1),
2251                                   wrap_c);
2252         }
2253         memcpy(orig[0], s->block[0], sizeof(int16_t) * 64 * mb_block_count);
2254     }
2255
2256     /* DCT & quantize */
2257     assert(s->out_format != FMT_MJPEG || s->qscale == 8);
2258     {
2259         for (i = 0; i < mb_block_count; i++) {
2260             if (!skip_dct[i]) {
2261                 int overflow;
2262                 s->block_last_index[i] = s->dct_quantize(s, s->block[i], i, s->qscale, &overflow);
2263                 // FIXME we could decide to change to quantizer instead of
2264                 // clipping
2265                 // JS: I don't think that would be a good idea it could lower
2266                 //     quality instead of improve it. Just INTRADC clipping
2267                 //     deserves changes in quantizer
2268                 if (overflow)
2269                     clip_coeffs(s, s->block[i], s->block_last_index[i]);
2270             } else
2271                 s->block_last_index[i] = -1;
2272         }
2273         if (s->quantizer_noise_shaping) {
2274             for (i = 0; i < mb_block_count; i++) {
2275                 if (!skip_dct[i]) {
2276                     s->block_last_index[i] =
2277                         dct_quantize_refine(s, s->block[i], weight[i],
2278                                             orig[i], i, s->qscale);
2279                 }
2280             }
2281         }
2282
2283         if (s->luma_elim_threshold && !s->mb_intra)
2284             for (i = 0; i < 4; i++)
2285                 dct_single_coeff_elimination(s, i, s->luma_elim_threshold);
2286         if (s->chroma_elim_threshold && !s->mb_intra)
2287             for (i = 4; i < mb_block_count; i++)
2288                 dct_single_coeff_elimination(s, i, s->chroma_elim_threshold);
2289
2290         if (s->mpv_flags & FF_MPV_FLAG_CBP_RD) {
2291             for (i = 0; i < mb_block_count; i++) {
2292                 if (s->block_last_index[i] == -1)
2293                     s->coded_score[i] = INT_MAX / 256;
2294             }
2295         }
2296     }
2297
2298     if ((s->avctx->flags & AV_CODEC_FLAG_GRAY) && s->mb_intra) {
2299         s->block_last_index[4] =
2300         s->block_last_index[5] = 0;
2301         s->block[4][0] =
2302         s->block[5][0] = (1024 + s->c_dc_scale / 2) / s->c_dc_scale;
2303     }
2304
2305     // non c quantize code returns incorrect block_last_index FIXME
2306     if (s->alternate_scan && s->dct_quantize != ff_dct_quantize_c) {
2307         for (i = 0; i < mb_block_count; i++) {
2308             int j;
2309             if (s->block_last_index[i] > 0) {
2310                 for (j = 63; j > 0; j--) {
2311                     if (s->block[i][s->intra_scantable.permutated[j]])
2312                         break;
2313                 }
2314                 s->block_last_index[i] = j;
2315             }
2316         }
2317     }
2318
2319     /* huffman encode */
2320     switch(s->codec_id){ //FIXME funct ptr could be slightly faster
2321     case AV_CODEC_ID_MPEG1VIDEO:
2322     case AV_CODEC_ID_MPEG2VIDEO:
2323         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
2324             ff_mpeg1_encode_mb(s, s->block, motion_x, motion_y);
2325         break;
2326     case AV_CODEC_ID_MPEG4:
2327         if (CONFIG_MPEG4_ENCODER)
2328             ff_mpeg4_encode_mb(s, s->block, motion_x, motion_y);
2329         break;
2330     case AV_CODEC_ID_MSMPEG4V2:
2331     case AV_CODEC_ID_MSMPEG4V3:
2332     case AV_CODEC_ID_WMV1:
2333         if (CONFIG_MSMPEG4_ENCODER)
2334             ff_msmpeg4_encode_mb(s, s->block, motion_x, motion_y);
2335         break;
2336     case AV_CODEC_ID_WMV2:
2337         if (CONFIG_WMV2_ENCODER)
2338             ff_wmv2_encode_mb(s, s->block, motion_x, motion_y);
2339         break;
2340     case AV_CODEC_ID_H261:
2341         if (CONFIG_H261_ENCODER)
2342             ff_h261_encode_mb(s, s->block, motion_x, motion_y);
2343         break;
2344     case AV_CODEC_ID_H263:
2345     case AV_CODEC_ID_H263P:
2346     case AV_CODEC_ID_FLV1:
2347     case AV_CODEC_ID_RV10:
2348     case AV_CODEC_ID_RV20:
2349         if (CONFIG_H263_ENCODER)
2350             ff_h263_encode_mb(s, s->block, motion_x, motion_y);
2351         break;
2352     case AV_CODEC_ID_MJPEG:
2353         if (CONFIG_MJPEG_ENCODER)
2354             ff_mjpeg_encode_mb(s, s->block);
2355         break;
2356     default:
2357         assert(0);
2358     }
2359 }
2360
2361 static av_always_inline void encode_mb(MpegEncContext *s, int motion_x, int motion_y)
2362 {
2363     if (s->chroma_format == CHROMA_420) encode_mb_internal(s, motion_x, motion_y,  8, 6);
2364     else                                encode_mb_internal(s, motion_x, motion_y, 16, 8);
2365 }
2366
2367 static inline void copy_context_before_encode(MpegEncContext *d, MpegEncContext *s, int type){
2368     int i;
2369
2370     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2371
2372     /* mpeg1 */
2373     d->mb_skip_run= s->mb_skip_run;
2374     for(i=0; i<3; i++)
2375         d->last_dc[i] = s->last_dc[i];
2376
2377     /* statistics */
2378     d->mv_bits= s->mv_bits;
2379     d->i_tex_bits= s->i_tex_bits;
2380     d->p_tex_bits= s->p_tex_bits;
2381     d->i_count= s->i_count;
2382     d->f_count= s->f_count;
2383     d->b_count= s->b_count;
2384     d->skip_count= s->skip_count;
2385     d->misc_bits= s->misc_bits;
2386     d->last_bits= 0;
2387
2388     d->mb_skipped= 0;
2389     d->qscale= s->qscale;
2390     d->dquant= s->dquant;
2391
2392     d->esc3_level_length= s->esc3_level_length;
2393 }
2394
2395 static inline void copy_context_after_encode(MpegEncContext *d, MpegEncContext *s, int type){
2396     int i;
2397
2398     memcpy(d->mv, s->mv, 2*4*2*sizeof(int));
2399     memcpy(d->last_mv, s->last_mv, 2*2*2*sizeof(int)); //FIXME is memcpy faster than a loop?
2400
2401     /* mpeg1 */
2402     d->mb_skip_run= s->mb_skip_run;
2403     for(i=0; i<3; i++)
2404         d->last_dc[i] = s->last_dc[i];
2405
2406     /* statistics */
2407     d->mv_bits= s->mv_bits;
2408     d->i_tex_bits= s->i_tex_bits;
2409     d->p_tex_bits= s->p_tex_bits;
2410     d->i_count= s->i_count;
2411     d->f_count= s->f_count;
2412     d->b_count= s->b_count;
2413     d->skip_count= s->skip_count;
2414     d->misc_bits= s->misc_bits;
2415
2416     d->mb_intra= s->mb_intra;
2417     d->mb_skipped= s->mb_skipped;
2418     d->mv_type= s->mv_type;
2419     d->mv_dir= s->mv_dir;
2420     d->pb= s->pb;
2421     if(s->data_partitioning){
2422         d->pb2= s->pb2;
2423         d->tex_pb= s->tex_pb;
2424     }
2425     d->block= s->block;
2426     for(i=0; i<8; i++)
2427         d->block_last_index[i]= s->block_last_index[i];
2428     d->interlaced_dct= s->interlaced_dct;
2429     d->qscale= s->qscale;
2430
2431     d->esc3_level_length= s->esc3_level_length;
2432 }
2433
2434 static inline void encode_mb_hq(MpegEncContext *s, MpegEncContext *backup, MpegEncContext *best, int type,
2435                            PutBitContext pb[2], PutBitContext pb2[2], PutBitContext tex_pb[2],
2436                            int *dmin, int *next_block, int motion_x, int motion_y)
2437 {
2438     int score;
2439     uint8_t *dest_backup[3];
2440
2441     copy_context_before_encode(s, backup, type);
2442
2443     s->block= s->blocks[*next_block];
2444     s->pb= pb[*next_block];
2445     if(s->data_partitioning){
2446         s->pb2   = pb2   [*next_block];
2447         s->tex_pb= tex_pb[*next_block];
2448     }
2449
2450     if(*next_block){
2451         memcpy(dest_backup, s->dest, sizeof(s->dest));
2452         s->dest[0] = s->sc.rd_scratchpad;
2453         s->dest[1] = s->sc.rd_scratchpad + 16*s->linesize;
2454         s->dest[2] = s->sc.rd_scratchpad + 16*s->linesize + 8;
2455         assert(s->linesize >= 32); //FIXME
2456     }
2457
2458     encode_mb(s, motion_x, motion_y);
2459
2460     score= put_bits_count(&s->pb);
2461     if(s->data_partitioning){
2462         score+= put_bits_count(&s->pb2);
2463         score+= put_bits_count(&s->tex_pb);
2464     }
2465
2466     if(s->avctx->mb_decision == FF_MB_DECISION_RD){
2467         ff_mpv_decode_mb(s, s->block);
2468
2469         score *= s->lambda2;
2470         score += sse_mb(s) << FF_LAMBDA_SHIFT;
2471     }
2472
2473     if(*next_block){
2474         memcpy(s->dest, dest_backup, sizeof(s->dest));
2475     }
2476
2477     if(score<*dmin){
2478         *dmin= score;
2479         *next_block^=1;
2480
2481         copy_context_after_encode(best, s, type);
2482     }
2483 }
2484
2485 static int sse(MpegEncContext *s, uint8_t *src1, uint8_t *src2, int w, int h, int stride){
2486     uint32_t *sq = ff_square_tab + 256;
2487     int acc=0;
2488     int x,y;
2489
2490     if(w==16 && h==16)
2491         return s->mecc.sse[0](NULL, src1, src2, stride, 16);
2492     else if(w==8 && h==8)
2493         return s->mecc.sse[1](NULL, src1, src2, stride, 8);
2494
2495     for(y=0; y<h; y++){
2496         for(x=0; x<w; x++){
2497             acc+= sq[src1[x + y*stride] - src2[x + y*stride]];
2498         }
2499     }
2500
2501     assert(acc>=0);
2502
2503     return acc;
2504 }
2505
2506 static int sse_mb(MpegEncContext *s){
2507     int w= 16;
2508     int h= 16;
2509
2510     if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
2511     if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
2512
2513     if(w==16 && h==16)
2514       if(s->avctx->mb_cmp == FF_CMP_NSSE){
2515         return s->mecc.nsse[0](s, s->new_picture.f->data[0] + s->mb_x * 16 + s->mb_y * s->linesize   * 16, s->dest[0], s->linesize,   16) +
2516                s->mecc.nsse[1](s, s->new_picture.f->data[1] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[1], s->uvlinesize,  8) +
2517                s->mecc.nsse[1](s, s->new_picture.f->data[2] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[2], s->uvlinesize,  8);
2518       }else{
2519         return s->mecc.sse[0](NULL, s->new_picture.f->data[0] + s->mb_x * 16 + s->mb_y * s->linesize   * 16, s->dest[0], s->linesize,   16) +
2520                s->mecc.sse[1](NULL, s->new_picture.f->data[1] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[1], s->uvlinesize,  8) +
2521                s->mecc.sse[1](NULL, s->new_picture.f->data[2] + s->mb_x *  8 + s->mb_y * s->uvlinesize *  8, s->dest[2], s->uvlinesize,  8);
2522       }
2523     else
2524         return  sse(s, s->new_picture.f->data[0] + s->mb_x*16 + s->mb_y*s->linesize*16, s->dest[0], w, h, s->linesize)
2525                +sse(s, s->new_picture.f->data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[1], w>>1, h>>1, s->uvlinesize)
2526                +sse(s, s->new_picture.f->data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*8,s->dest[2], w>>1, h>>1, s->uvlinesize);
2527 }
2528
2529 static int pre_estimate_motion_thread(AVCodecContext *c, void *arg){
2530     MpegEncContext *s= *(void**)arg;
2531
2532
2533     s->me.pre_pass=1;
2534     s->me.dia_size= s->avctx->pre_dia_size;
2535     s->first_slice_line=1;
2536     for(s->mb_y= s->end_mb_y-1; s->mb_y >= s->start_mb_y; s->mb_y--) {
2537         for(s->mb_x=s->mb_width-1; s->mb_x >=0 ;s->mb_x--) {
2538             ff_pre_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2539         }
2540         s->first_slice_line=0;
2541     }
2542
2543     s->me.pre_pass=0;
2544
2545     return 0;
2546 }
2547
2548 static int estimate_motion_thread(AVCodecContext *c, void *arg){
2549     MpegEncContext *s= *(void**)arg;
2550
2551     s->me.dia_size= s->avctx->dia_size;
2552     s->first_slice_line=1;
2553     for(s->mb_y= s->start_mb_y; s->mb_y < s->end_mb_y; s->mb_y++) {
2554         s->mb_x=0; //for block init below
2555         ff_init_block_index(s);
2556         for(s->mb_x=0; s->mb_x < s->mb_width; s->mb_x++) {
2557             s->block_index[0]+=2;
2558             s->block_index[1]+=2;
2559             s->block_index[2]+=2;
2560             s->block_index[3]+=2;
2561
2562             /* compute motion vector & mb_type and store in context */
2563             if(s->pict_type==AV_PICTURE_TYPE_B)
2564                 ff_estimate_b_frame_motion(s, s->mb_x, s->mb_y);
2565             else
2566                 ff_estimate_p_frame_motion(s, s->mb_x, s->mb_y);
2567         }
2568         s->first_slice_line=0;
2569     }
2570     return 0;
2571 }
2572
2573 static int mb_var_thread(AVCodecContext *c, void *arg){
2574     MpegEncContext *s= *(void**)arg;
2575     int mb_x, mb_y;
2576
2577     for(mb_y=s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2578         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2579             int xx = mb_x * 16;
2580             int yy = mb_y * 16;
2581             uint8_t *pix = s->new_picture.f->data[0] + (yy * s->linesize) + xx;
2582             int varc;
2583             int sum = s->mpvencdsp.pix_sum(pix, s->linesize);
2584
2585             varc = (s->mpvencdsp.pix_norm1(pix, s->linesize) -
2586                     (((unsigned) sum * sum) >> 8) + 500 + 128) >> 8;
2587
2588             s->current_picture.mb_var [s->mb_stride * mb_y + mb_x] = varc;
2589             s->current_picture.mb_mean[s->mb_stride * mb_y + mb_x] = (sum+128)>>8;
2590             s->me.mb_var_sum_temp    += varc;
2591         }
2592     }
2593     return 0;
2594 }
2595
2596 static void write_slice_end(MpegEncContext *s){
2597     if(CONFIG_MPEG4_ENCODER && s->codec_id==AV_CODEC_ID_MPEG4){
2598         if(s->partitioned_frame){
2599             ff_mpeg4_merge_partitions(s);
2600         }
2601
2602         ff_mpeg4_stuffing(&s->pb);
2603     }else if(CONFIG_MJPEG_ENCODER && s->out_format == FMT_MJPEG){
2604         ff_mjpeg_encode_stuffing(&s->pb);
2605     }
2606
2607     avpriv_align_put_bits(&s->pb);
2608     flush_put_bits(&s->pb);
2609
2610     if ((s->avctx->flags & AV_CODEC_FLAG_PASS1) && !s->partitioned_frame)
2611         s->misc_bits+= get_bits_diff(s);
2612 }
2613
2614 static void write_mb_info(MpegEncContext *s)
2615 {
2616     uint8_t *ptr = s->mb_info_ptr + s->mb_info_size - 12;
2617     int offset = put_bits_count(&s->pb);
2618     int mba  = s->mb_x + s->mb_width * (s->mb_y % s->gob_index);
2619     int gobn = s->mb_y / s->gob_index;
2620     int pred_x, pred_y;
2621     if (CONFIG_H263_ENCODER)
2622         ff_h263_pred_motion(s, 0, 0, &pred_x, &pred_y);
2623     bytestream_put_le32(&ptr, offset);
2624     bytestream_put_byte(&ptr, s->qscale);
2625     bytestream_put_byte(&ptr, gobn);
2626     bytestream_put_le16(&ptr, mba);
2627     bytestream_put_byte(&ptr, pred_x); /* hmv1 */
2628     bytestream_put_byte(&ptr, pred_y); /* vmv1 */
2629     /* 4MV not implemented */
2630     bytestream_put_byte(&ptr, 0); /* hmv2 */
2631     bytestream_put_byte(&ptr, 0); /* vmv2 */
2632 }
2633
2634 static void update_mb_info(MpegEncContext *s, int startcode)
2635 {
2636     if (!s->mb_info)
2637         return;
2638     if (put_bits_count(&s->pb) - s->prev_mb_info*8 >= s->mb_info*8) {
2639         s->mb_info_size += 12;
2640         s->prev_mb_info = s->last_mb_info;
2641     }
2642     if (startcode) {
2643         s->prev_mb_info = put_bits_count(&s->pb)/8;
2644         /* This might have incremented mb_info_size above, and we return without
2645          * actually writing any info into that slot yet. But in that case,
2646          * this will be called again at the start of the after writing the
2647          * start code, actually writing the mb info. */
2648         return;
2649     }
2650
2651     s->last_mb_info = put_bits_count(&s->pb)/8;
2652     if (!s->mb_info_size)
2653         s->mb_info_size += 12;
2654     write_mb_info(s);
2655 }
2656
2657 static int encode_thread(AVCodecContext *c, void *arg){
2658     MpegEncContext *s= *(void**)arg;
2659     int mb_x, mb_y, pdif = 0;
2660     int chr_h= 16>>s->chroma_y_shift;
2661     int i, j;
2662     MpegEncContext best_s = { 0 }, backup_s;
2663     uint8_t bit_buf[2][MAX_MB_BYTES];
2664     uint8_t bit_buf2[2][MAX_MB_BYTES];
2665     uint8_t bit_buf_tex[2][MAX_MB_BYTES];
2666     PutBitContext pb[2], pb2[2], tex_pb[2];
2667
2668     for(i=0; i<2; i++){
2669         init_put_bits(&pb    [i], bit_buf    [i], MAX_MB_BYTES);
2670         init_put_bits(&pb2   [i], bit_buf2   [i], MAX_MB_BYTES);
2671         init_put_bits(&tex_pb[i], bit_buf_tex[i], MAX_MB_BYTES);
2672     }
2673
2674     s->last_bits= put_bits_count(&s->pb);
2675     s->mv_bits=0;
2676     s->misc_bits=0;
2677     s->i_tex_bits=0;
2678     s->p_tex_bits=0;
2679     s->i_count=0;
2680     s->f_count=0;
2681     s->b_count=0;
2682     s->skip_count=0;
2683
2684     for(i=0; i<3; i++){
2685         /* init last dc values */
2686         /* note: quant matrix value (8) is implied here */
2687         s->last_dc[i] = 128 << s->intra_dc_precision;
2688
2689         s->current_picture.encoding_error[i] = 0;
2690     }
2691     s->mb_skip_run = 0;
2692     memset(s->last_mv, 0, sizeof(s->last_mv));
2693
2694     s->last_mv_dir = 0;
2695
2696     switch(s->codec_id){
2697     case AV_CODEC_ID_H263:
2698     case AV_CODEC_ID_H263P:
2699     case AV_CODEC_ID_FLV1:
2700         if (CONFIG_H263_ENCODER)
2701             s->gob_index = H263_GOB_HEIGHT(s->height);
2702         break;
2703     case AV_CODEC_ID_MPEG4:
2704         if(CONFIG_MPEG4_ENCODER && s->partitioned_frame)
2705             ff_mpeg4_init_partitions(s);
2706         break;
2707     }
2708
2709     s->resync_mb_x=0;
2710     s->resync_mb_y=0;
2711     s->first_slice_line = 1;
2712     s->ptr_lastgob = s->pb.buf;
2713     for(mb_y= s->start_mb_y; mb_y < s->end_mb_y; mb_y++) {
2714         s->mb_x=0;
2715         s->mb_y= mb_y;
2716
2717         ff_set_qscale(s, s->qscale);
2718         ff_init_block_index(s);
2719
2720         for(mb_x=0; mb_x < s->mb_width; mb_x++) {
2721             int xy= mb_y*s->mb_stride + mb_x; // removed const, H261 needs to adjust this
2722             int mb_type= s->mb_type[xy];
2723 //            int d;
2724             int dmin= INT_MAX;
2725             int dir;
2726
2727             if(s->pb.buf_end - s->pb.buf - (put_bits_count(&s->pb)>>3) < MAX_MB_BYTES){
2728                 av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2729                 return -1;
2730             }
2731             if(s->data_partitioning){
2732                 if(   s->pb2   .buf_end - s->pb2   .buf - (put_bits_count(&s->    pb2)>>3) < MAX_MB_BYTES
2733                    || s->tex_pb.buf_end - s->tex_pb.buf - (put_bits_count(&s->tex_pb )>>3) < MAX_MB_BYTES){
2734                     av_log(s->avctx, AV_LOG_ERROR, "encoded frame too large\n");
2735                     return -1;
2736                 }
2737             }
2738
2739             s->mb_x = mb_x;
2740             s->mb_y = mb_y;  // moved into loop, can get changed by H.261
2741             ff_update_block_index(s);
2742
2743             if(CONFIG_H261_ENCODER && s->codec_id == AV_CODEC_ID_H261){
2744                 ff_h261_reorder_mb_index(s);
2745                 xy= s->mb_y*s->mb_stride + s->mb_x;
2746                 mb_type= s->mb_type[xy];
2747             }
2748
2749             /* write gob / video packet header  */
2750             if(s->rtp_mode){
2751                 int current_packet_size, is_gob_start;
2752
2753                 current_packet_size= ((put_bits_count(&s->pb)+7)>>3) - (s->ptr_lastgob - s->pb.buf);
2754
2755                 is_gob_start= s->avctx->rtp_payload_size && current_packet_size >= s->avctx->rtp_payload_size && mb_y + mb_x>0;
2756
2757                 if(s->start_mb_y == mb_y && mb_y > 0 && mb_x==0) is_gob_start=1;
2758
2759                 switch(s->codec_id){
2760                 case AV_CODEC_ID_H263:
2761                 case AV_CODEC_ID_H263P:
2762                     if(!s->h263_slice_structured)
2763                         if(s->mb_x || s->mb_y%s->gob_index) is_gob_start=0;
2764                     break;
2765                 case AV_CODEC_ID_MPEG2VIDEO:
2766                     if(s->mb_x==0 && s->mb_y!=0) is_gob_start=1;
2767                 case AV_CODEC_ID_MPEG1VIDEO:
2768                     if(s->mb_skip_run) is_gob_start=0;
2769                     break;
2770                 }
2771
2772                 if(is_gob_start){
2773                     if(s->start_mb_y != mb_y || mb_x!=0){
2774                         write_slice_end(s);
2775
2776                         if(CONFIG_MPEG4_ENCODER && s->codec_id==AV_CODEC_ID_MPEG4 && s->partitioned_frame){
2777                             ff_mpeg4_init_partitions(s);
2778                         }
2779                     }
2780
2781                     assert((put_bits_count(&s->pb)&7) == 0);
2782                     current_packet_size= put_bits_ptr(&s->pb) - s->ptr_lastgob;
2783
2784                     if (s->error_rate && s->resync_mb_x + s->resync_mb_y > 0) {
2785                         int r= put_bits_count(&s->pb)/8 + s->picture_number + 16 + s->mb_x + s->mb_y;
2786                         int d = 100 / s->error_rate;
2787                         if(r % d == 0){
2788                             current_packet_size=0;
2789                             s->pb.buf_ptr= s->ptr_lastgob;
2790                             assert(put_bits_ptr(&s->pb) == s->ptr_lastgob);
2791                         }
2792                     }
2793
2794 #if FF_API_RTP_CALLBACK
2795 FF_DISABLE_DEPRECATION_WARNINGS
2796                     if (s->avctx->rtp_callback){
2797                         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width + mb_x - s->resync_mb_x;
2798                         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, current_packet_size, number_mb);
2799                     }
2800 FF_ENABLE_DEPRECATION_WARNINGS
2801 #endif
2802                     update_mb_info(s, 1);
2803
2804                     switch(s->codec_id){
2805                     case AV_CODEC_ID_MPEG4:
2806                         if (CONFIG_MPEG4_ENCODER) {
2807                             ff_mpeg4_encode_video_packet_header(s);
2808                             ff_mpeg4_clean_buffers(s);
2809                         }
2810                     break;
2811                     case AV_CODEC_ID_MPEG1VIDEO:
2812                     case AV_CODEC_ID_MPEG2VIDEO:
2813                         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER) {
2814                             ff_mpeg1_encode_slice_header(s);
2815                             ff_mpeg1_clean_buffers(s);
2816                         }
2817                     break;
2818                     case AV_CODEC_ID_H263:
2819                     case AV_CODEC_ID_H263P:
2820                         if (CONFIG_H263_ENCODER)
2821                             ff_h263_encode_gob_header(s, mb_y);
2822                     break;
2823                     }
2824
2825                     if (s->avctx->flags & AV_CODEC_FLAG_PASS1) {
2826                         int bits= put_bits_count(&s->pb);
2827                         s->misc_bits+= bits - s->last_bits;
2828                         s->last_bits= bits;
2829                     }
2830
2831                     s->ptr_lastgob += current_packet_size;
2832                     s->first_slice_line=1;
2833                     s->resync_mb_x=mb_x;
2834                     s->resync_mb_y=mb_y;
2835                 }
2836             }
2837
2838             if(  (s->resync_mb_x   == s->mb_x)
2839                && s->resync_mb_y+1 == s->mb_y){
2840                 s->first_slice_line=0;
2841             }
2842
2843             s->mb_skipped=0;
2844             s->dquant=0; //only for QP_RD
2845
2846             update_mb_info(s, 0);
2847
2848             if (mb_type & (mb_type-1) || (s->mpv_flags & FF_MPV_FLAG_QP_RD)) { // more than 1 MB type possible or FF_MPV_FLAG_QP_RD
2849                 int next_block=0;
2850                 int pb_bits_count, pb2_bits_count, tex_pb_bits_count;
2851
2852                 copy_context_before_encode(&backup_s, s, -1);
2853                 backup_s.pb= s->pb;
2854                 best_s.data_partitioning= s->data_partitioning;
2855                 best_s.partitioned_frame= s->partitioned_frame;
2856                 if(s->data_partitioning){
2857                     backup_s.pb2= s->pb2;
2858                     backup_s.tex_pb= s->tex_pb;
2859                 }
2860
2861                 if(mb_type&CANDIDATE_MB_TYPE_INTER){
2862                     s->mv_dir = MV_DIR_FORWARD;
2863                     s->mv_type = MV_TYPE_16X16;
2864                     s->mb_intra= 0;
2865                     s->mv[0][0][0] = s->p_mv_table[xy][0];
2866                     s->mv[0][0][1] = s->p_mv_table[xy][1];
2867                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER, pb, pb2, tex_pb,
2868                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2869                 }
2870                 if(mb_type&CANDIDATE_MB_TYPE_INTER_I){
2871                     s->mv_dir = MV_DIR_FORWARD;
2872                     s->mv_type = MV_TYPE_FIELD;
2873                     s->mb_intra= 0;
2874                     for(i=0; i<2; i++){
2875                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
2876                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
2877                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
2878                     }
2879                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER_I, pb, pb2, tex_pb,
2880                                  &dmin, &next_block, 0, 0);
2881                 }
2882                 if(mb_type&CANDIDATE_MB_TYPE_SKIPPED){
2883                     s->mv_dir = MV_DIR_FORWARD;
2884                     s->mv_type = MV_TYPE_16X16;
2885                     s->mb_intra= 0;
2886                     s->mv[0][0][0] = 0;
2887                     s->mv[0][0][1] = 0;
2888                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_SKIPPED, pb, pb2, tex_pb,
2889                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2890                 }
2891                 if(mb_type&CANDIDATE_MB_TYPE_INTER4V){
2892                     s->mv_dir = MV_DIR_FORWARD;
2893                     s->mv_type = MV_TYPE_8X8;
2894                     s->mb_intra= 0;
2895                     for(i=0; i<4; i++){
2896                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
2897                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
2898                     }
2899                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER4V, pb, pb2, tex_pb,
2900                                  &dmin, &next_block, 0, 0);
2901                 }
2902                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD){
2903                     s->mv_dir = MV_DIR_FORWARD;
2904                     s->mv_type = MV_TYPE_16X16;
2905                     s->mb_intra= 0;
2906                     s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
2907                     s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
2908                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD, pb, pb2, tex_pb,
2909                                  &dmin, &next_block, s->mv[0][0][0], s->mv[0][0][1]);
2910                 }
2911                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD){
2912                     s->mv_dir = MV_DIR_BACKWARD;
2913                     s->mv_type = MV_TYPE_16X16;
2914                     s->mb_intra= 0;
2915                     s->mv[1][0][0] = s->b_back_mv_table[xy][0];
2916                     s->mv[1][0][1] = s->b_back_mv_table[xy][1];
2917                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD, pb, pb2, tex_pb,
2918                                  &dmin, &next_block, s->mv[1][0][0], s->mv[1][0][1]);
2919                 }
2920                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR){
2921                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2922                     s->mv_type = MV_TYPE_16X16;
2923                     s->mb_intra= 0;
2924                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
2925                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
2926                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
2927                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
2928                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR, pb, pb2, tex_pb,
2929                                  &dmin, &next_block, 0, 0);
2930                 }
2931                 if(mb_type&CANDIDATE_MB_TYPE_FORWARD_I){
2932                     s->mv_dir = MV_DIR_FORWARD;
2933                     s->mv_type = MV_TYPE_FIELD;
2934                     s->mb_intra= 0;
2935                     for(i=0; i<2; i++){
2936                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
2937                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
2938                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
2939                     }
2940                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_FORWARD_I, pb, pb2, tex_pb,
2941                                  &dmin, &next_block, 0, 0);
2942                 }
2943                 if(mb_type&CANDIDATE_MB_TYPE_BACKWARD_I){
2944                     s->mv_dir = MV_DIR_BACKWARD;
2945                     s->mv_type = MV_TYPE_FIELD;
2946                     s->mb_intra= 0;
2947                     for(i=0; i<2; i++){
2948                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
2949                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
2950                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
2951                     }
2952                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BACKWARD_I, pb, pb2, tex_pb,
2953                                  &dmin, &next_block, 0, 0);
2954                 }
2955                 if(mb_type&CANDIDATE_MB_TYPE_BIDIR_I){
2956                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
2957                     s->mv_type = MV_TYPE_FIELD;
2958                     s->mb_intra= 0;
2959                     for(dir=0; dir<2; dir++){
2960                         for(i=0; i<2; i++){
2961                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
2962                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
2963                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
2964                         }
2965                     }
2966                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_BIDIR_I, pb, pb2, tex_pb,
2967                                  &dmin, &next_block, 0, 0);
2968                 }
2969                 if(mb_type&CANDIDATE_MB_TYPE_INTRA){
2970                     s->mv_dir = 0;
2971                     s->mv_type = MV_TYPE_16X16;
2972                     s->mb_intra= 1;
2973                     s->mv[0][0][0] = 0;
2974                     s->mv[0][0][1] = 0;
2975                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTRA, pb, pb2, tex_pb,
2976                                  &dmin, &next_block, 0, 0);
2977                     if(s->h263_pred || s->h263_aic){
2978                         if(best_s.mb_intra)
2979                             s->mbintra_table[mb_x + mb_y*s->mb_stride]=1;
2980                         else
2981                             ff_clean_intra_table_entries(s); //old mode?
2982                     }
2983                 }
2984
2985                 if ((s->mpv_flags & FF_MPV_FLAG_QP_RD) && dmin < INT_MAX) {
2986                     if(best_s.mv_type==MV_TYPE_16X16){ //FIXME move 4mv after QPRD
2987                         const int last_qp= backup_s.qscale;
2988                         int qpi, qp, dc[6];
2989                         int16_t ac[6][16];
2990                         const int mvdir= (best_s.mv_dir&MV_DIR_BACKWARD) ? 1 : 0;
2991                         static const int dquant_tab[4]={-1,1,-2,2};
2992
2993                         assert(backup_s.dquant == 0);
2994
2995                         //FIXME intra
2996                         s->mv_dir= best_s.mv_dir;
2997                         s->mv_type = MV_TYPE_16X16;
2998                         s->mb_intra= best_s.mb_intra;
2999                         s->mv[0][0][0] = best_s.mv[0][0][0];
3000                         s->mv[0][0][1] = best_s.mv[0][0][1];
3001                         s->mv[1][0][0] = best_s.mv[1][0][0];
3002                         s->mv[1][0][1] = best_s.mv[1][0][1];
3003
3004                         qpi = s->pict_type == AV_PICTURE_TYPE_B ? 2 : 0;
3005                         for(; qpi<4; qpi++){
3006                             int dquant= dquant_tab[qpi];
3007                             qp= last_qp + dquant;
3008                             if(qp < s->avctx->qmin || qp > s->avctx->qmax)
3009                                 continue;
3010                             backup_s.dquant= dquant;
3011                             if(s->mb_intra && s->dc_val[0]){
3012                                 for(i=0; i<6; i++){
3013                                     dc[i]= s->dc_val[0][ s->block_index[i] ];
3014                                     memcpy(ac[i], s->ac_val[0][s->block_index[i]], sizeof(int16_t)*16);
3015                                 }
3016                             }
3017
3018                             encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
3019                                          &dmin, &next_block, s->mv[mvdir][0][0], s->mv[mvdir][0][1]);
3020                             if(best_s.qscale != qp){
3021                                 if(s->mb_intra && s->dc_val[0]){
3022                                     for(i=0; i<6; i++){
3023                                         s->dc_val[0][ s->block_index[i] ]= dc[i];
3024                                         memcpy(s->ac_val[0][s->block_index[i]], ac[i], sizeof(int16_t)*16);
3025                                     }
3026                                 }
3027                             }
3028                         }
3029                     }
3030                 }
3031                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT){
3032                     int mx= s->b_direct_mv_table[xy][0];
3033                     int my= s->b_direct_mv_table[xy][1];
3034
3035                     backup_s.dquant = 0;
3036                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
3037                     s->mb_intra= 0;
3038                     ff_mpeg4_set_direct_mv(s, mx, my);
3039                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
3040                                  &dmin, &next_block, mx, my);
3041                 }
3042                 if(CONFIG_MPEG4_ENCODER && mb_type&CANDIDATE_MB_TYPE_DIRECT0){
3043                     backup_s.dquant = 0;
3044                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD | MV_DIRECT;
3045                     s->mb_intra= 0;
3046                     ff_mpeg4_set_direct_mv(s, 0, 0);
3047                     encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_DIRECT, pb, pb2, tex_pb,
3048                                  &dmin, &next_block, 0, 0);
3049                 }
3050                 if (!best_s.mb_intra && s->mpv_flags & FF_MPV_FLAG_SKIP_RD) {
3051                     int coded=0;
3052                     for(i=0; i<6; i++)
3053                         coded |= s->block_last_index[i];
3054                     if(coded){
3055                         int mx,my;
3056                         memcpy(s->mv, best_s.mv, sizeof(s->mv));
3057                         if(CONFIG_MPEG4_ENCODER && best_s.mv_dir & MV_DIRECT){
3058                             mx=my=0; //FIXME find the one we actually used
3059                             ff_mpeg4_set_direct_mv(s, mx, my);
3060                         }else if(best_s.mv_dir&MV_DIR_BACKWARD){
3061                             mx= s->mv[1][0][0];
3062                             my= s->mv[1][0][1];
3063                         }else{
3064                             mx= s->mv[0][0][0];
3065                             my= s->mv[0][0][1];
3066                         }
3067
3068                         s->mv_dir= best_s.mv_dir;
3069                         s->mv_type = best_s.mv_type;
3070                         s->mb_intra= 0;
3071 /*                        s->mv[0][0][0] = best_s.mv[0][0][0];
3072                         s->mv[0][0][1] = best_s.mv[0][0][1];
3073                         s->mv[1][0][0] = best_s.mv[1][0][0];
3074                         s->mv[1][0][1] = best_s.mv[1][0][1];*/
3075                         backup_s.dquant= 0;
3076                         s->skipdct=1;
3077                         encode_mb_hq(s, &backup_s, &best_s, CANDIDATE_MB_TYPE_INTER /* wrong but unused */, pb, pb2, tex_pb,
3078                                         &dmin, &next_block, mx, my);
3079                         s->skipdct=0;
3080                     }
3081                 }
3082
3083                 s->current_picture.qscale_table[xy] = best_s.qscale;
3084
3085                 copy_context_after_encode(s, &best_s, -1);
3086
3087                 pb_bits_count= put_bits_count(&s->pb);
3088                 flush_put_bits(&s->pb);
3089                 avpriv_copy_bits(&backup_s.pb, bit_buf[next_block^1], pb_bits_count);
3090                 s->pb= backup_s.pb;
3091
3092                 if(s->data_partitioning){
3093                     pb2_bits_count= put_bits_count(&s->pb2);
3094                     flush_put_bits(&s->pb2);
3095                     avpriv_copy_bits(&backup_s.pb2, bit_buf2[next_block^1], pb2_bits_count);
3096                     s->pb2= backup_s.pb2;
3097
3098                     tex_pb_bits_count= put_bits_count(&s->tex_pb);
3099                     flush_put_bits(&s->tex_pb);
3100                     avpriv_copy_bits(&backup_s.tex_pb, bit_buf_tex[next_block^1], tex_pb_bits_count);
3101                     s->tex_pb= backup_s.tex_pb;
3102                 }
3103                 s->last_bits= put_bits_count(&s->pb);
3104
3105                 if (CONFIG_H263_ENCODER &&
3106                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
3107                     ff_h263_update_motion_val(s);
3108
3109                 if(next_block==0){ //FIXME 16 vs linesize16
3110                     s->hdsp.put_pixels_tab[0][0](s->dest[0], s->sc.rd_scratchpad                     , s->linesize  ,16);
3111                     s->hdsp.put_pixels_tab[1][0](s->dest[1], s->sc.rd_scratchpad + 16*s->linesize    , s->uvlinesize, 8);
3112                     s->hdsp.put_pixels_tab[1][0](s->dest[2], s->sc.rd_scratchpad + 16*s->linesize + 8, s->uvlinesize, 8);
3113                 }
3114
3115                 if(s->avctx->mb_decision == FF_MB_DECISION_BITS)
3116                     ff_mpv_decode_mb(s, s->block);
3117             } else {
3118                 int motion_x = 0, motion_y = 0;
3119                 s->mv_type=MV_TYPE_16X16;
3120                 // only one MB-Type possible
3121
3122                 switch(mb_type){
3123                 case CANDIDATE_MB_TYPE_INTRA:
3124                     s->mv_dir = 0;
3125                     s->mb_intra= 1;
3126                     motion_x= s->mv[0][0][0] = 0;
3127                     motion_y= s->mv[0][0][1] = 0;
3128                     break;
3129                 case CANDIDATE_MB_TYPE_INTER:
3130                     s->mv_dir = MV_DIR_FORWARD;
3131                     s->mb_intra= 0;
3132                     motion_x= s->mv[0][0][0] = s->p_mv_table[xy][0];
3133                     motion_y= s->mv[0][0][1] = s->p_mv_table[xy][1];
3134                     break;
3135                 case CANDIDATE_MB_TYPE_INTER_I:
3136                     s->mv_dir = MV_DIR_FORWARD;
3137                     s->mv_type = MV_TYPE_FIELD;
3138                     s->mb_intra= 0;
3139                     for(i=0; i<2; i++){
3140                         j= s->field_select[0][i] = s->p_field_select_table[i][xy];
3141                         s->mv[0][i][0] = s->p_field_mv_table[i][j][xy][0];
3142                         s->mv[0][i][1] = s->p_field_mv_table[i][j][xy][1];
3143                     }
3144                     break;
3145                 case CANDIDATE_MB_TYPE_INTER4V:
3146                     s->mv_dir = MV_DIR_FORWARD;
3147                     s->mv_type = MV_TYPE_8X8;
3148                     s->mb_intra= 0;
3149                     for(i=0; i<4; i++){
3150                         s->mv[0][i][0] = s->current_picture.motion_val[0][s->block_index[i]][0];
3151                         s->mv[0][i][1] = s->current_picture.motion_val[0][s->block_index[i]][1];
3152                     }
3153                     break;
3154                 case CANDIDATE_MB_TYPE_DIRECT:
3155                     if (CONFIG_MPEG4_ENCODER) {
3156                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
3157                         s->mb_intra= 0;
3158                         motion_x=s->b_direct_mv_table[xy][0];
3159                         motion_y=s->b_direct_mv_table[xy][1];
3160                         ff_mpeg4_set_direct_mv(s, motion_x, motion_y);
3161                     }
3162                     break;
3163                 case CANDIDATE_MB_TYPE_DIRECT0:
3164                     if (CONFIG_MPEG4_ENCODER) {
3165                         s->mv_dir = MV_DIR_FORWARD|MV_DIR_BACKWARD|MV_DIRECT;
3166                         s->mb_intra= 0;
3167                         ff_mpeg4_set_direct_mv(s, 0, 0);
3168                     }
3169                     break;
3170                 case CANDIDATE_MB_TYPE_BIDIR:
3171                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
3172                     s->mb_intra= 0;
3173                     s->mv[0][0][0] = s->b_bidir_forw_mv_table[xy][0];
3174                     s->mv[0][0][1] = s->b_bidir_forw_mv_table[xy][1];
3175                     s->mv[1][0][0] = s->b_bidir_back_mv_table[xy][0];
3176                     s->mv[1][0][1] = s->b_bidir_back_mv_table[xy][1];
3177                     break;
3178                 case CANDIDATE_MB_TYPE_BACKWARD:
3179                     s->mv_dir = MV_DIR_BACKWARD;
3180                     s->mb_intra= 0;
3181                     motion_x= s->mv[1][0][0] = s->b_back_mv_table[xy][0];
3182                     motion_y= s->mv[1][0][1] = s->b_back_mv_table[xy][1];
3183                     break;
3184                 case CANDIDATE_MB_TYPE_FORWARD:
3185                     s->mv_dir = MV_DIR_FORWARD;
3186                     s->mb_intra= 0;
3187                     motion_x= s->mv[0][0][0] = s->b_forw_mv_table[xy][0];
3188                     motion_y= s->mv[0][0][1] = s->b_forw_mv_table[xy][1];
3189                     break;
3190                 case CANDIDATE_MB_TYPE_FORWARD_I:
3191                     s->mv_dir = MV_DIR_FORWARD;
3192                     s->mv_type = MV_TYPE_FIELD;
3193                     s->mb_intra= 0;
3194                     for(i=0; i<2; i++){
3195                         j= s->field_select[0][i] = s->b_field_select_table[0][i][xy];
3196                         s->mv[0][i][0] = s->b_field_mv_table[0][i][j][xy][0];
3197                         s->mv[0][i][1] = s->b_field_mv_table[0][i][j][xy][1];
3198                     }
3199                     break;
3200                 case CANDIDATE_MB_TYPE_BACKWARD_I:
3201                     s->mv_dir = MV_DIR_BACKWARD;
3202                     s->mv_type = MV_TYPE_FIELD;
3203                     s->mb_intra= 0;
3204                     for(i=0; i<2; i++){
3205                         j= s->field_select[1][i] = s->b_field_select_table[1][i][xy];
3206                         s->mv[1][i][0] = s->b_field_mv_table[1][i][j][xy][0];
3207                         s->mv[1][i][1] = s->b_field_mv_table[1][i][j][xy][1];
3208                     }
3209                     break;
3210                 case CANDIDATE_MB_TYPE_BIDIR_I:
3211                     s->mv_dir = MV_DIR_FORWARD | MV_DIR_BACKWARD;
3212                     s->mv_type = MV_TYPE_FIELD;
3213                     s->mb_intra= 0;
3214                     for(dir=0; dir<2; dir++){
3215                         for(i=0; i<2; i++){
3216                             j= s->field_select[dir][i] = s->b_field_select_table[dir][i][xy];
3217                             s->mv[dir][i][0] = s->b_field_mv_table[dir][i][j][xy][0];
3218                             s->mv[dir][i][1] = s->b_field_mv_table[dir][i][j][xy][1];
3219                         }
3220                     }
3221                     break;
3222                 default:
3223                     av_log(s->avctx, AV_LOG_ERROR, "illegal MB type\n");
3224                 }
3225
3226                 encode_mb(s, motion_x, motion_y);
3227
3228                 // RAL: Update last macroblock type
3229                 s->last_mv_dir = s->mv_dir;
3230
3231                 if (CONFIG_H263_ENCODER &&
3232                     s->out_format == FMT_H263 && s->pict_type!=AV_PICTURE_TYPE_B)
3233                     ff_h263_update_motion_val(s);
3234
3235                 ff_mpv_decode_mb(s, s->block);
3236             }
3237
3238             /* clean the MV table in IPS frames for direct mode in B frames */
3239             if(s->mb_intra /* && I,P,S_TYPE */){
3240                 s->p_mv_table[xy][0]=0;
3241                 s->p_mv_table[xy][1]=0;
3242             }
3243
3244             if (s->avctx->flags & AV_CODEC_FLAG_PSNR) {
3245                 int w= 16;
3246                 int h= 16;
3247
3248                 if(s->mb_x*16 + 16 > s->width ) w= s->width - s->mb_x*16;
3249                 if(s->mb_y*16 + 16 > s->height) h= s->height- s->mb_y*16;
3250
3251                 s->current_picture.encoding_error[0] += sse(
3252                     s, s->new_picture.f->data[0] + s->mb_x*16 + s->mb_y*s->linesize*16,
3253                     s->dest[0], w, h, s->linesize);
3254                 s->current_picture.encoding_error[1] += sse(
3255                     s, s->new_picture.f->data[1] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
3256                     s->dest[1], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
3257                 s->current_picture.encoding_error[2] += sse(
3258                     s, s->new_picture.f->data[2] + s->mb_x*8  + s->mb_y*s->uvlinesize*chr_h,
3259                     s->dest[2], w>>1, h>>s->chroma_y_shift, s->uvlinesize);
3260             }
3261             if(s->loop_filter){
3262                 if(CONFIG_H263_ENCODER && s->out_format == FMT_H263)
3263                     ff_h263_loop_filter(s);
3264             }
3265             ff_dlog(s->avctx, "MB %d %d bits\n",
3266                     s->mb_x + s->mb_y * s->mb_stride, put_bits_count(&s->pb));
3267         }
3268     }
3269
3270     //not beautiful here but we must write it before flushing so it has to be here
3271     if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version && s->msmpeg4_version<4 && s->pict_type == AV_PICTURE_TYPE_I)
3272         ff_msmpeg4_encode_ext_header(s);
3273
3274     write_slice_end(s);
3275
3276 #if FF_API_RTP_CALLBACK
3277 FF_DISABLE_DEPRECATION_WARNINGS
3278     /* Send the last GOB if RTP */
3279     if (s->avctx->rtp_callback) {
3280         int number_mb = (mb_y - s->resync_mb_y)*s->mb_width - s->resync_mb_x;
3281         pdif = put_bits_ptr(&s->pb) - s->ptr_lastgob;
3282         /* Call the RTP callback to send the last GOB */
3283         emms_c();
3284         s->avctx->rtp_callback(s->avctx, s->ptr_lastgob, pdif, number_mb);
3285     }
3286 FF_ENABLE_DEPRECATION_WARNINGS
3287 #endif
3288
3289     return 0;
3290 }
3291
3292 #define MERGE(field) dst->field += src->field; src->field=0
3293 static void merge_context_after_me(MpegEncContext *dst, MpegEncContext *src){
3294     MERGE(me.scene_change_score);
3295     MERGE(me.mc_mb_var_sum_temp);
3296     MERGE(me.mb_var_sum_temp);
3297 }
3298
3299 static void merge_context_after_encode(MpegEncContext *dst, MpegEncContext *src){
3300     int i;
3301
3302     MERGE(dct_count[0]); //note, the other dct vars are not part of the context
3303     MERGE(dct_count[1]);
3304     MERGE(mv_bits);
3305     MERGE(i_tex_bits);
3306     MERGE(p_tex_bits);
3307     MERGE(i_count);
3308     MERGE(f_count);
3309     MERGE(b_count);
3310     MERGE(skip_count);
3311     MERGE(misc_bits);
3312     MERGE(er.error_count);
3313     MERGE(padding_bug_score);
3314     MERGE(current_picture.encoding_error[0]);
3315     MERGE(current_picture.encoding_error[1]);
3316     MERGE(current_picture.encoding_error[2]);
3317
3318     if(dst->avctx->noise_reduction){
3319         for(i=0; i<64; i++){
3320             MERGE(dct_error_sum[0][i]);
3321             MERGE(dct_error_sum[1][i]);
3322         }
3323     }
3324
3325     assert(put_bits_count(&src->pb) % 8 ==0);
3326     assert(put_bits_count(&dst->pb) % 8 ==0);
3327     avpriv_copy_bits(&dst->pb, src->pb.buf, put_bits_count(&src->pb));
3328     flush_put_bits(&dst->pb);
3329 }
3330
3331 static int estimate_qp(MpegEncContext *s, int dry_run){
3332     if (s->next_lambda){
3333         s->current_picture_ptr->f->quality =
3334         s->current_picture.f->quality = s->next_lambda;
3335         if(!dry_run) s->next_lambda= 0;
3336     } else if (!s->fixed_qscale) {
3337         s->current_picture_ptr->f->quality =
3338         s->current_picture.f->quality = ff_rate_estimate_qscale(s, dry_run);
3339         if (s->current_picture.f->quality < 0)
3340             return -1;
3341     }
3342
3343     if(s->adaptive_quant){
3344         switch(s->codec_id){
3345         case AV_CODEC_ID_MPEG4:
3346             if (CONFIG_MPEG4_ENCODER)
3347                 ff_clean_mpeg4_qscales(s);
3348             break;
3349         case AV_CODEC_ID_H263:
3350         case AV_CODEC_ID_H263P:
3351         case AV_CODEC_ID_FLV1:
3352             if (CONFIG_H263_ENCODER)
3353                 ff_clean_h263_qscales(s);
3354             break;
3355         default:
3356             ff_init_qscale_tab(s);
3357         }
3358
3359         s->lambda= s->lambda_table[0];
3360         //FIXME broken
3361     }else
3362         s->lambda = s->current_picture.f->quality;
3363     update_qscale(s);
3364     return 0;
3365 }
3366
3367 /* must be called before writing the header */
3368 static void set_frame_distances(MpegEncContext * s){
3369     assert(s->current_picture_ptr->f->pts != AV_NOPTS_VALUE);
3370     s->time = s->current_picture_ptr->f->pts * s->avctx->time_base.num;
3371
3372     if(s->pict_type==AV_PICTURE_TYPE_B){
3373         s->pb_time= s->pp_time - (s->last_non_b_time - s->time);
3374         assert(s->pb_time > 0 && s->pb_time < s->pp_time);
3375     }else{
3376         s->pp_time= s->time - s->last_non_b_time;
3377         s->last_non_b_time= s->time;
3378         assert(s->picture_number==0 || s->pp_time > 0);
3379     }
3380 }
3381
3382 static int encode_picture(MpegEncContext *s, int picture_number)
3383 {
3384     int i, ret;
3385     int bits;
3386     int context_count = s->slice_context_count;
3387
3388     s->picture_number = picture_number;
3389
3390     /* Reset the average MB variance */
3391     s->me.mb_var_sum_temp    =
3392     s->me.mc_mb_var_sum_temp = 0;
3393
3394     /* we need to initialize some time vars before we can encode b-frames */
3395     // RAL: Condition added for MPEG1VIDEO
3396     if (s->codec_id == AV_CODEC_ID_MPEG1VIDEO || s->codec_id == AV_CODEC_ID_MPEG2VIDEO || (s->h263_pred && !s->msmpeg4_version))
3397         set_frame_distances(s);
3398     if(CONFIG_MPEG4_ENCODER && s->codec_id == AV_CODEC_ID_MPEG4)
3399         ff_set_mpeg4_time(s);
3400
3401     s->me.scene_change_score=0;
3402
3403 //    s->lambda= s->current_picture_ptr->quality; //FIXME qscale / ... stuff for ME rate distortion
3404
3405     if(s->pict_type==AV_PICTURE_TYPE_I){
3406         if(s->msmpeg4_version >= 3) s->no_rounding=1;
3407         else                        s->no_rounding=0;
3408     }else if(s->pict_type!=AV_PICTURE_TYPE_B){
3409         if(s->flipflop_rounding || s->codec_id == AV_CODEC_ID_H263P || s->codec_id == AV_CODEC_ID_MPEG4)
3410             s->no_rounding ^= 1;
3411     }
3412
3413     if (s->avctx->flags & AV_CODEC_FLAG_PASS2) {
3414         if (estimate_qp(s,1) < 0)
3415             return -1;
3416         ff_get_2pass_fcode(s);
3417     } else if (!(s->avctx->flags & AV_CODEC_FLAG_QSCALE)) {
3418         if(s->pict_type==AV_PICTURE_TYPE_B)
3419             s->lambda= s->last_lambda_for[s->pict_type];
3420         else
3421             s->lambda= s->last_lambda_for[s->last_non_b_pict_type];
3422         update_qscale(s);
3423     }
3424
3425     s->mb_intra=0; //for the rate distortion & bit compare functions
3426     for(i=1; i<context_count; i++){
3427         ret = ff_update_duplicate_context(s->thread_context[i], s);
3428         if (ret < 0)
3429             return ret;
3430     }
3431
3432     if(ff_init_me(s)<0)
3433         return -1;
3434
3435     /* Estimate motion for every MB */
3436     if(s->pict_type != AV_PICTURE_TYPE_I){
3437         s->lambda = (s->lambda * s->avctx->me_penalty_compensation + 128)>>8;
3438         s->lambda2= (s->lambda2* (int64_t)s->avctx->me_penalty_compensation + 128)>>8;
3439         if (s->pict_type != AV_PICTURE_TYPE_B) {
3440             if((s->avctx->pre_me && s->last_non_b_pict_type==AV_PICTURE_TYPE_I) || s->avctx->pre_me==2){
3441                 s->avctx->execute(s->avctx, pre_estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3442             }
3443         }
3444
3445         s->avctx->execute(s->avctx, estimate_motion_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3446     }else /* if(s->pict_type == AV_PICTURE_TYPE_I) */{
3447         /* I-Frame */
3448         for(i=0; i<s->mb_stride*s->mb_height; i++)
3449             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
3450
3451         if(!s->fixed_qscale){
3452             /* finding spatial complexity for I-frame rate control */
3453             s->avctx->execute(s->avctx, mb_var_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3454         }
3455     }
3456     for(i=1; i<context_count; i++){
3457         merge_context_after_me(s, s->thread_context[i]);
3458     }
3459     s->current_picture.mc_mb_var_sum= s->current_picture_ptr->mc_mb_var_sum= s->me.mc_mb_var_sum_temp;
3460     s->current_picture.   mb_var_sum= s->current_picture_ptr->   mb_var_sum= s->me.   mb_var_sum_temp;
3461     emms_c();
3462
3463     if (s->me.scene_change_score > s->scenechange_threshold &&
3464         s->pict_type == AV_PICTURE_TYPE_P) {
3465         s->pict_type= AV_PICTURE_TYPE_I;
3466         for(i=0; i<s->mb_stride*s->mb_height; i++)
3467             s->mb_type[i]= CANDIDATE_MB_TYPE_INTRA;
3468         ff_dlog(s, "Scene change detected, encoding as I Frame %d %d\n",
3469                 s->current_picture.mb_var_sum, s->current_picture.mc_mb_var_sum);
3470     }
3471
3472     if(!s->umvplus){
3473         if(s->pict_type==AV_PICTURE_TYPE_P || s->pict_type==AV_PICTURE_TYPE_S) {
3474             s->f_code= ff_get_best_fcode(s, s->p_mv_table, CANDIDATE_MB_TYPE_INTER);
3475
3476             if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_ME) {
3477                 int a,b;
3478                 a= ff_get_best_fcode(s, s->p_field_mv_table[0][0], CANDIDATE_MB_TYPE_INTER_I); //FIXME field_select
3479                 b= ff_get_best_fcode(s, s->p_field_mv_table[1][1], CANDIDATE_MB_TYPE_INTER_I);
3480                 s->f_code= FFMAX3(s->f_code, a, b);
3481             }
3482
3483             ff_fix_long_p_mvs(s);
3484             ff_fix_long_mvs(s, NULL, 0, s->p_mv_table, s->f_code, CANDIDATE_MB_TYPE_INTER, 0);
3485             if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_ME) {
3486                 int j;
3487                 for(i=0; i<2; i++){
3488                     for(j=0; j<2; j++)
3489                         ff_fix_long_mvs(s, s->p_field_select_table[i], j,
3490                                         s->p_field_mv_table[i][j], s->f_code, CANDIDATE_MB_TYPE_INTER_I, 0);
3491                 }
3492             }
3493         }
3494
3495         if(s->pict_type==AV_PICTURE_TYPE_B){
3496             int a, b;
3497
3498             a = ff_get_best_fcode(s, s->b_forw_mv_table, CANDIDATE_MB_TYPE_FORWARD);
3499             b = ff_get_best_fcode(s, s->b_bidir_forw_mv_table, CANDIDATE_MB_TYPE_BIDIR);
3500             s->f_code = FFMAX(a, b);
3501
3502             a = ff_get_best_fcode(s, s->b_back_mv_table, CANDIDATE_MB_TYPE_BACKWARD);
3503             b = ff_get_best_fcode(s, s->b_bidir_back_mv_table, CANDIDATE_MB_TYPE_BIDIR);
3504             s->b_code = FFMAX(a, b);
3505
3506             ff_fix_long_mvs(s, NULL, 0, s->b_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_FORWARD, 1);
3507             ff_fix_long_mvs(s, NULL, 0, s->b_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BACKWARD, 1);
3508             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_forw_mv_table, s->f_code, CANDIDATE_MB_TYPE_BIDIR, 1);
3509             ff_fix_long_mvs(s, NULL, 0, s->b_bidir_back_mv_table, s->b_code, CANDIDATE_MB_TYPE_BIDIR, 1);
3510             if (s->avctx->flags & AV_CODEC_FLAG_INTERLACED_ME) {
3511                 int dir, j;
3512                 for(dir=0; dir<2; dir++){
3513                     for(i=0; i<2; i++){
3514                         for(j=0; j<2; j++){
3515                             int type= dir ? (CANDIDATE_MB_TYPE_BACKWARD_I|CANDIDATE_MB_TYPE_BIDIR_I)
3516                                           : (CANDIDATE_MB_TYPE_FORWARD_I |CANDIDATE_MB_TYPE_BIDIR_I);
3517                             ff_fix_long_mvs(s, s->b_field_select_table[dir][i], j,
3518                                             s->b_field_mv_table[dir][i][j], dir ? s->b_code : s->f_code, type, 1);
3519                         }
3520                     }
3521                 }
3522             }
3523         }
3524     }
3525
3526     if (estimate_qp(s, 0) < 0)
3527         return -1;
3528
3529     if (s->qscale < 3 && s->max_qcoeff <= 128 &&
3530         s->pict_type == AV_PICTURE_TYPE_I &&
3531         !(s->avctx->flags & AV_CODEC_FLAG_QSCALE))
3532         s->qscale= 3; //reduce clipping problems
3533
3534     if (s->out_format == FMT_MJPEG) {
3535         /* for mjpeg, we do include qscale in the matrix */
3536         for(i=1;i<64;i++){
3537             int j = s->idsp.idct_permutation[i];
3538
3539             s->intra_matrix[j] = av_clip_uint8((ff_mpeg1_default_intra_matrix[i] * s->qscale) >> 3);
3540         }
3541         s->y_dc_scale_table=
3542         s->c_dc_scale_table= ff_mpeg2_dc_scale_table[s->intra_dc_precision];
3543         s->intra_matrix[0] = ff_mpeg2_dc_scale_table[s->intra_dc_precision][8];
3544         ff_convert_matrix(s, s->q_intra_matrix, s->q_intra_matrix16,
3545                        s->intra_matrix, s->intra_quant_bias, 8, 8, 1);
3546         s->qscale= 8;
3547     }
3548
3549     //FIXME var duplication
3550     s->current_picture_ptr->f->key_frame =
3551     s->current_picture.f->key_frame = s->pict_type == AV_PICTURE_TYPE_I; //FIXME pic_ptr
3552     s->current_picture_ptr->f->pict_type =
3553     s->current_picture.f->pict_type = s->pict_type;
3554
3555     if (s->current_picture.f->key_frame)
3556         s->picture_in_gop_number=0;
3557
3558     s->last_bits= put_bits_count(&s->pb);
3559     switch(s->out_format) {
3560     case FMT_MJPEG:
3561         if (CONFIG_MJPEG_ENCODER)
3562             ff_mjpeg_encode_picture_header(s->avctx, &s->pb, &s->intra_scantable,
3563                                            s->intra_matrix);
3564         break;
3565     case FMT_H261:
3566         if (CONFIG_H261_ENCODER)
3567             ff_h261_encode_picture_header(s, picture_number);
3568         break;
3569     case FMT_H263:
3570         if (CONFIG_WMV2_ENCODER && s->codec_id == AV_CODEC_ID_WMV2)
3571             ff_wmv2_encode_picture_header(s, picture_number);
3572         else if (CONFIG_MSMPEG4_ENCODER && s->msmpeg4_version)
3573             ff_msmpeg4_encode_picture_header(s, picture_number);
3574         else if (CONFIG_MPEG4_ENCODER && s->h263_pred)
3575             ff_mpeg4_encode_picture_header(s, picture_number);
3576         else if (CONFIG_RV10_ENCODER && s->codec_id == AV_CODEC_ID_RV10) {
3577             ret = ff_rv10_encode_picture_header(s, picture_number);
3578             if (ret < 0)
3579                 return ret;
3580         }
3581         else if (CONFIG_RV20_ENCODER && s->codec_id == AV_CODEC_ID_RV20)
3582             ff_rv20_encode_picture_header(s, picture_number);
3583         else if (CONFIG_FLV_ENCODER && s->codec_id == AV_CODEC_ID_FLV1)
3584             ff_flv_encode_picture_header(s, picture_number);
3585         else if (CONFIG_H263_ENCODER)
3586             ff_h263_encode_picture_header(s, picture_number);
3587         break;
3588     case FMT_MPEG1:
3589         if (CONFIG_MPEG1VIDEO_ENCODER || CONFIG_MPEG2VIDEO_ENCODER)
3590             ff_mpeg1_encode_picture_header(s, picture_number);
3591         break;
3592     default:
3593         assert(0);
3594     }
3595     bits= put_bits_count(&s->pb);
3596     s->header_bits= bits - s->last_bits;
3597
3598     for(i=1; i<context_count; i++){
3599         update_duplicate_context_after_me(s->thread_context[i], s);
3600     }
3601     s->avctx->execute(s->avctx, encode_thread, &s->thread_context[0], NULL, context_count, sizeof(void*));
3602     for(i=1; i<context_count; i++){
3603         merge_context_after_encode(s, s->thread_context[i]);
3604     }
3605     emms_c();
3606     return 0;
3607 }
3608
3609 static void denoise_dct_c(MpegEncContext *s, int16_t *block){
3610     const int intra= s->mb_intra;
3611     int i;
3612
3613     s->dct_count[intra]++;
3614
3615     for(i=0; i<64; i++){
3616         int level= block[i];
3617
3618         if(level){
3619             if(level>0){
3620                 s->dct_error_sum[intra][i] += level;
3621                 level -= s->dct_offset[intra][i];
3622                 if(level<0) level=0;
3623             }else{
3624                 s->dct_error_sum[intra][i] -= level;
3625                 level += s->dct_offset[intra][i];
3626                 if(level>0) level=0;
3627             }
3628             block[i]= level;
3629         }
3630     }
3631 }
3632
3633 static int dct_quantize_trellis_c(MpegEncContext *s,
3634                                   int16_t *block, int n,
3635                                   int qscale, int *overflow){
3636     const int *qmat;
3637     const uint8_t *scantable= s->intra_scantable.scantable;
3638     const uint8_t *perm_scantable= s->intra_scantable.permutated;
3639     int max=0;
3640     unsigned int threshold1, threshold2;
3641     int bias=0;
3642     int run_tab[65];
3643     int level_tab[65];
3644     int score_tab[65];
3645     int survivor[65];
3646     int survivor_count;
3647     int last_run=0;
3648     int last_level=0;
3649     int last_score= 0;
3650     int last_i;
3651     int coeff[2][64];
3652     int coeff_count[64];
3653     int qmul, qadd, start_i, last_non_zero, i, dc;
3654     const int esc_length= s->ac_esc_length;
3655     uint8_t * length;
3656     uint8_t * last_length;
3657     const int lambda= s->lambda2 >> (FF_LAMBDA_SHIFT - 6);
3658
3659     s->fdsp.fdct(block);
3660
3661     if(s->dct_error_sum)
3662         s->denoise_dct(s, block);
3663     qmul= qscale*16;
3664     qadd= ((qscale-1)|1)*8;
3665
3666     if (s->mb_intra) {
3667         int q;
3668         if (!s->h263_aic) {
3669             if (n < 4)
3670                 q = s->y_dc_scale;
3671             else
3672                 q = s->c_dc_scale;
3673             q = q << 3;
3674         } else{
3675             /* For AIC we skip quant/dequant of INTRADC */
3676             q = 1 << 3;
3677             qadd=0;
3678         }
3679
3680         /* note: block[0] is assumed to be positive */
3681         block[0] = (block[0] + (q >> 1)) / q;
3682         start_i = 1;
3683         last_non_zero = 0;
3684         qmat = s->q_intra_matrix[qscale];
3685         if(s->mpeg_quant || s->out_format == FMT_MPEG1)
3686             bias= 1<<(QMAT_SHIFT-1);
3687         length     = s->intra_ac_vlc_length;
3688         last_length= s->intra_ac_vlc_last_length;
3689     } else {
3690         start_i = 0;
3691         last_non_zero = -1;
3692         qmat = s->q_inter_matrix[qscale];
3693         length     = s->inter_ac_vlc_length;
3694         last_length= s->inter_ac_vlc_last_length;
3695     }
3696     last_i= start_i;
3697
3698     threshold1= (1<<QMAT_SHIFT) - bias - 1;
3699     threshold2= (threshold1<<1);
3700
3701     for(i=63; i>=start_i; i--) {
3702         const int j = scantable[i];
3703         int level = block[j] * qmat[j];
3704
3705         if(((unsigned)(level+threshold1))>threshold2){
3706             last_non_zero = i;
3707             break;
3708         }
3709     }
3710
3711     for(i=start_i; i<=last_non_zero; i++) {
3712         const int j = scantable[i];
3713         int level = block[j] * qmat[j];
3714
3715 //        if(   bias+level >= (1<<(QMAT_SHIFT - 3))
3716 //           || bias-level >= (1<<(QMAT_SHIFT - 3))){
3717         if(((unsigned)(level+threshold1))>threshold2){
3718             if(level>0){
3719                 level= (bias + level)>>QMAT_SHIFT;
3720                 coeff[0][i]= level;
3721                 coeff[1][i]= level-1;
3722 //                coeff[2][k]= level-2;
3723             }else{
3724                 level= (bias - level)>>QMAT_SHIFT;
3725                 coeff[0][i]= -level;
3726                 coeff[1][i]= -level+1;
3727 //                coeff[2][k]= -level+2;
3728             }
3729             coeff_count[i]= FFMIN(level, 2);
3730             assert(coeff_count[i]);
3731             max |=level;
3732         }else{
3733             coeff[0][i]= (level>>31)|1;
3734             coeff_count[i]= 1;
3735         }
3736     }
3737
3738     *overflow= s->max_qcoeff < max; //overflow might have happened
3739
3740     if(last_non_zero < start_i){
3741         memset(block + start_i, 0, (64-start_i)*sizeof(int16_t));
3742         return last_non_zero;
3743     }
3744
3745     score_tab[start_i]= 0;
3746     survivor[0]= start_i;
3747     survivor_count= 1;
3748
3749     for(i=start_i; i<=last_non_zero; i++){
3750         int level_index, j, zero_distortion;
3751         int dct_coeff= FFABS(block[ scantable[i] ]);
3752         int best_score=256*256*256*120;
3753
3754         if (s->fdsp.fdct == ff_fdct_ifast)
3755             dct_coeff= (dct_coeff*ff_inv_aanscales[ scantable[i] ]) >> 12;
3756         zero_distortion= dct_coeff*dct_coeff;
3757
3758         for(level_index=0; level_index < coeff_count[i]; level_index++){
3759             int distortion;
3760             int level= coeff[level_index][i];
3761             const int alevel= FFABS(level);
3762             int unquant_coeff;
3763
3764             assert(level);
3765
3766             if(s->out_format == FMT_H263){
3767                 unquant_coeff= alevel*qmul + qadd;
3768             }else{ //MPEG1
3769                 j = s->idsp.idct_permutation[scantable[i]]; // FIXME: optimize
3770                 if(s->mb_intra){
3771                         unquant_coeff = (int)(  alevel  * qscale * s->intra_matrix[j]) >> 3;
3772                         unquant_coeff =   (unquant_coeff - 1) | 1;
3773                 }else{
3774                         unquant_coeff = (((  alevel  << 1) + 1) * qscale * ((int) s->inter_matrix[j])) >> 4;
3775                         unquant_coeff =   (unquant_coeff - 1) | 1;
3776                 }
3777                 unquant_coeff<<= 3;
3778             }
3779
3780             distortion= (unquant_coeff - dct_coeff) * (unquant_coeff - dct_coeff) - zero_distortion;
3781             level+=64;
3782             if((level&(~127)) == 0){
3783                 for(j=survivor_count-1; j>=0; j--){
3784                     int run= i - survivor[j];
3785                     int score= distortion + length[UNI_AC_ENC_INDEX(run, level)]*lambda;
3786                     score += score_tab[i-run];
3787
3788                     if(score < best_score){
3789                         best_score= score;
3790                         run_tab[i+1]= run;
3791                         level_tab[i+1]= level-64;
3792                     }
3793                 }
3794
3795                 if(s->out_format == FMT_H263){
3796                     for(j=survivor_count-1; j>=0; j--){
3797                         int run= i - survivor[j];
3798                         int score= distortion + last_length[UNI_AC_ENC_INDEX(run, level)]*lambda;
3799                         score += score_tab[i-run];
3800                         if(score < last_score){
3801                             last_score= score;
3802                             last_run= run;
3803                             last_level= level-64;
3804                             last_i= i+1;
3805                         }
3806                     }
3807                 }
3808             }else{
3809                 distortion += esc_length*lambda;
3810                 for(j=survivor_count-1; j>=0; j--){
3811                     int run= i - survivor[j];
3812                     int score= distortion + score_tab[i-run];
3813
3814                     if(score < best_score){
3815                         best_score= score;
3816                         run_tab[i+1]= run;
3817                         level_tab[i+1]= level-64;
3818                     }
3819                 }
3820
3821                 if(s->out_format == FMT_H263){
3822                   for(j=survivor_count-1; j>=0; j--){
3823                         int run= i - survivor[j];
3824                         int score= distortion + score_tab[i-run];
3825                         if(score < last_score){
3826                             last_score= score;
3827                             last_run= run;
3828                             last_level= level-64;
3829                             last_i= i+1;
3830                         }
3831                     }
3832                 }
3833             }
3834         }
3835
3836         score_tab[i+1]= best_score;
3837
3838         //Note: there is a vlc code in mpeg4 which is 1 bit shorter then another one with a shorter run and the same level
3839         if(last_non_zero <= 27){
3840             for(; survivor_count; survivor_count--){
3841                 if(score_tab[ survivor[survivor_count-1] ] <= best_score)
3842                     break;
3843             }
3844         }else{
3845             for(; survivor_count; survivor_count--){
3846                 if(score_tab[ survivor[survivor_count-1] ] <= best_score + lambda)
3847                     break;
3848             }
3849         }
3850
3851         survivor[ survivor_count++ ]= i+1;
3852     }
3853
3854     if(s->out_format != FMT_H263){
3855         last_score= 256*256*256*120;
3856         for(i= survivor[0]; i<=last_non_zero + 1; i++){
3857             int score= score_tab[i];
3858             if(i) score += lambda*2; //FIXME exacter?
3859
3860             if(score < last_score){
3861                 last_score= score;
3862                 last_i= i;
3863                 last_level= level_tab[i];
3864                 last_run= run_tab[i];
3865             }
3866         }
3867     }
3868
3869     s->coded_score[n] = last_score;
3870
3871     dc= FFABS(block[0]);
3872     last_non_zero= last_i - 1;
3873     memset(block + start_i, 0, (64-start_i)*sizeof(int16_t));
3874
3875     if(last_non_zero < start_i)
3876         return last_non_zero;
3877
3878     if(last_non_zero == 0 && start_i == 0){
3879         int best_level= 0;
3880         int best_score= dc * dc;
3881
3882         for(i=0; i<coeff_count[0]; i++){
3883             int level= coeff[i][0];
3884             int alevel= FFABS(level);
3885             int unquant_coeff, score, distortion;
3886
3887             if(s->out_format == FMT_H263){
3888                     unquant_coeff= (alevel*qmul + qadd)>>3;
3889             }else{ //MPEG1
3890                     unquant_coeff = (((  alevel  << 1) + 1) * qscale * ((int) s->inter_matrix[0])) >> 4;
3891                     unquant_coeff =   (unquant_coeff - 1) | 1;
3892             }
3893             unquant_coeff = (unquant_coeff + 4) >> 3;
3894             unquant_coeff<<= 3 + 3;
3895
3896             distortion= (unquant_coeff - dc) * (unquant_coeff - dc);
3897             level+=64;
3898             if((level&(~127)) == 0) score= distortion + last_length[UNI_AC_ENC_INDEX(0, level)]*lambda;
3899             else                    score= distortion + esc_length*lambda;
3900
3901             if(score < best_score){
3902                 best_score= score;
3903                 best_level= level - 64;
3904             }
3905         }
3906         block[0]= best_level;
3907         s->coded_score[n] = best_score - dc*dc;
3908         if(best_level == 0) return -1;
3909         else                return last_non_zero;
3910     }
3911
3912     i= last_i;
3913     assert(last_level);
3914
3915     block[ perm_scantable[last_non_zero] ]= last_level;
3916     i -= last_run + 1;
3917
3918     for(; i>start_i; i -= run_tab[i] + 1){
3919         block[ perm_scantable[i-1] ]= level_tab[i];
3920     }
3921
3922     return last_non_zero;
3923 }
3924
3925 //#define REFINE_STATS 1
3926 static int16_t basis[64][64];
3927
3928 static void build_basis(uint8_t *perm){
3929     int i, j, x, y;
3930     emms_c();
3931     for(i=0; i<8; i++){
3932         for(j=0; j<8; j++){
3933             for(y=0; y<8; y++){
3934                 for(x=0; x<8; x++){
3935                     double s= 0.25*(1<<BASIS_SHIFT);
3936                     int index= 8*i + j;
3937                     int perm_index= perm[index];
3938                     if(i==0) s*= sqrt(0.5);
3939                     if(j==0) s*= sqrt(0.5);
3940                     basis[perm_index][8*x + y]= lrintf(s * cos((M_PI/8.0)*i*(x+0.5)) * cos((M_PI/8.0)*j*(y+0.5)));
3941                 }
3942             }
3943         }
3944     }
3945 }
3946
3947 static int dct_quantize_refine(MpegEncContext *s, //FIXME breaks denoise?
3948                         int16_t *block, int16_t *weight, int16_t *orig,
3949                         int n, int qscale){
3950     int16_t rem[64];
3951     LOCAL_ALIGNED_16(int16_t, d1, [64]);
3952     const uint8_t *scantable= s->intra_scantable.scantable;
3953     const uint8_t *perm_scantable= s->intra_scantable.permutated;
3954 //    unsigned int threshold1, threshold2;
3955 //    int bias=0;
3956     int run_tab[65];
3957     int prev_run=0;
3958     int prev_level=0;
3959     int qmul, qadd, start_i, last_non_zero, i, dc;
3960     uint8_t * length;
3961     uint8_t * last_length;
3962     int lambda;
3963     int rle_index, run, q = 1, sum; //q is only used when s->mb_intra is true
3964 #ifdef REFINE_STATS
3965 static int count=0;
3966 static int after_last=0;
3967 static int to_zero=0;
3968 static int from_zero=0;
3969 static int raise=0;
3970 static int lower=0;
3971 static int messed_sign=0;
3972 #endif
3973
3974     if(basis[0][0] == 0)
3975         build_basis(s->idsp.idct_permutation);
3976
3977     qmul= qscale*2;
3978     qadd= (qscale-1)|1;
3979     if (s->mb_intra) {
3980         if (!s->h263_aic) {
3981             if (n < 4)
3982                 q = s->y_dc_scale;
3983             else
3984                 q = s->c_dc_scale;
3985         } else{
3986             /* For AIC we skip quant/dequant of INTRADC */
3987             q = 1;
3988             qadd=0;
3989         }
3990         q <<= RECON_SHIFT-3;
3991         /* note: block[0] is assumed to be positive */
3992         dc= block[0]*q;
3993 //        block[0] = (block[0] + (q >> 1)) / q;
3994         start_i = 1;
3995 //        if(s->mpeg_quant || s->out_format == FMT_MPEG1)
3996 //            bias= 1<<(QMAT_SHIFT-1);
3997         length     = s->intra_ac_vlc_length;
3998         last_length= s->intra_ac_vlc_last_length;
3999     } else {
4000         dc= 0;
4001         start_i = 0;
4002         length     = s->inter_ac_vlc_length;
4003         last_length= s->inter_ac_vlc_last_length;
4004     }
4005     last_non_zero = s->block_last_index[n];
4006
4007 #ifdef REFINE_STATS
4008 {START_TIMER
4009 #endif
4010     dc += (1<<(RECON_SHIFT-1));
4011     for(i=0; i<64; i++){
4012         rem[i]= dc - (orig[i]<<RECON_SHIFT); //FIXME  use orig dirrectly instead of copying to rem[]
4013     }
4014 #ifdef REFINE_STATS
4015 STOP_TIMER("memset rem[]")}
4016 #endif
4017     sum=0;
4018     for(i=0; i<64; i++){
4019         int one= 36;
4020         int qns=4;
4021         int w;
4022
4023         w= FFABS(weight[i]) + qns*one;
4024         w= 15 + (48*qns*one + w/2)/w; // 16 .. 63
4025
4026         weight[i] = w;
4027 //        w=weight[i] = (63*qns + (w/2)) / w;
4028
4029         assert(w>0);
4030         assert(w<(1<<6));
4031         sum += w*w;
4032     }
4033     lambda= sum*(uint64_t)s->lambda2 >> (FF_LAMBDA_SHIFT - 6 + 6 + 6 + 6);
4034 #ifdef REFINE_STATS
4035 {START_TIMER
4036 #endif
4037     run=0;
4038     rle_index=0;
4039     for(i=start_i; i<=last_non_zero; i++){
4040         int j= perm_scantable[i];
4041         const int level= block[j];
4042         int coeff;
4043
4044         if(level){
4045             if(level<0) coeff= qmul*level - qadd;
4046             else        coeff= qmul*level + qadd;
4047             run_tab[rle_index++]=run;
4048             run=0;
4049
4050             s->mpvencdsp.add_8x8basis(rem, basis[j], coeff);
4051         }else{
4052             run++;
4053         }
4054     }
4055 #ifdef REFINE_STATS
4056 if(last_non_zero>0){
4057 STOP_TIMER("init rem[]")
4058 }
4059 }
4060
4061 {START_TIMER
4062 #endif
4063     for(;;){
4064         int best_score = s->mpvencdsp.try_8x8basis(rem, weight, basis[0], 0);
4065         int best_coeff=0;
4066         int best_change=0;
4067         int run2, best_unquant_change=0, analyze_gradient;
4068 #ifdef REFINE_STATS
4069 {START_TIMER
4070 #endif
4071         analyze_gradient = last_non_zero > 2 || s->quantizer_noise_shaping >= 3;
4072
4073         if(analyze_gradient){
4074 #ifdef REFINE_STATS
4075 {START_TIMER
4076 #endif
4077             for(i=0; i<64; i++){
4078                 int w= weight[i];
4079
4080                 d1[i] = (rem[i]*w*w + (1<<(RECON_SHIFT+12-1)))>>(RECON_SHIFT+12);
4081             }
4082 #ifdef REFINE_STATS
4083 STOP_TIMER("rem*w*w")}
4084 {START_TIMER
4085 #endif
4086             s->fdsp.fdct(d1);
4087 #ifdef REFINE_STATS
4088 STOP_TIMER("dct")}
4089 #endif
4090         }
4091
4092         if(start_i){
4093             const int level= block[0];
4094             int change, old_coeff;
4095
4096             assert(s->mb_intra);
4097
4098             old_coeff= q*level;
4099
4100             for(change=-1; change<=1; change+=2){
4101                 int new_level= level + change;
4102                 int score, new_coeff;
4103
4104                 new_coeff= q*new_level;
4105                 if(new_coeff >= 2048 || new_coeff < 0)
4106                     continue;
4107
4108                 score = s->mpvencdsp.try_8x8basis(rem, weight, basis[0],
4109                                                   new_coeff - old_coeff);
4110                 if(score<best_score){
4111                     best_score= score;
4112                     best_coeff= 0;
4113                     best_change= change;
4114                     best_unquant_change= new_coeff - old_coeff;
4115                 }
4116             }
4117         }
4118
4119         run=0;
4120         rle_index=0;
4121         run2= run_tab[rle_index++];
4122         prev_level=0;
4123         prev_run=0;
4124
4125         for(i=start_i; i<64; i++){
4126             int j= perm_scantable[i];
4127             const int level= block[j];
4128             int change, old_coeff;
4129
4130             if(s->quantizer_noise_shaping < 3 && i > last_non_zero + 1)
4131                 break;
4132
4133             if(level){
4134                 if(level<0) old_coeff= qmul*level - qadd;
4135                 else        old_coeff= qmul*level + qadd;
4136                 run2= run_tab[rle_index++]; //FIXME ! maybe after last
4137             }else{
4138                 old_coeff=0;
4139                 run2--;
4140                 assert(run2>=0 || i >= last_non_zero );
4141             }
4142
4143             for(change=-1; change<=1; change+=2){
4144                 int new_level= level + change;
4145                 int score, new_coeff, unquant_change;
4146
4147                 score=0;
4148                 if(s->quantizer_noise_shaping < 2 && FFABS(new_level) > FFABS(level))
4149                    continue;
4150
4151                 if(new_level){
4152                     if(new_level<0) new_coeff= qmul*new_level - qadd;
4153                     else            new_coeff= qmul*new_level + qadd;
4154                     if(new_coeff >= 2048 || new_coeff <= -2048)
4155                         continue;
4156                     //FIXME check for overflow
4157
4158                     if(level){
4159                         if(level < 63 && level > -63){
4160                             if(i < last_non_zero)
4161                                 score +=   length[UNI_AC_ENC_INDEX(run, new_level+64)]
4162                                          - length[UNI_AC_ENC_INDEX(run, level+64)];
4163                             else
4164                                 score +=   last_length[UNI_AC_ENC_INDEX(run, new_level+64)]
4165                                          - last_length[UNI_AC_ENC_INDEX(run, level+64)];
4166                         }
4167                     }else{
4168                         assert(FFABS(new_level)==1);
4169
4170                         if(analyze_gradient){
4171                             int g= d1[ scantable[i] ];
4172                             if(g && (g^new_level) >= 0)
4173                                 continue;
4174                         }
4175
4176                         if(i < last_non_zero){
4177                             int next_i= i + run2 + 1;
4178                             int next_level= block[ perm_scantable[next_i] ] + 64;
4179
4180                             if(next_level&(~127))
4181                                 next_level= 0;
4182
4183                             if(next_i < last_non_zero)
4184                                 score +=   length[UNI_AC_ENC_INDEX(run, 65)]
4185                                          + length[UNI_AC_ENC_INDEX(run2, next_level)]
4186                                          - length[UNI_AC_ENC_INDEX(run + run2 + 1, next_level)];
4187                             else
4188                                 score +=  length[UNI_AC_ENC_INDEX(run, 65)]
4189                                         + last_length[UNI_AC_ENC_INDEX(run2, next_level)]
4190                                         - last_length[UNI_AC_ENC_INDEX(run + run2 + 1, next_level)];
4191                         }else{
4192                             score += last_length[UNI_AC_ENC_INDEX(run, 65)];
4193                             if(prev_level){
4194                                 score +=  length[UNI_AC_ENC_INDEX(prev_run, prev_level)]
4195                                         - last_length[UNI_AC_ENC_INDEX(prev_run, prev_level)];
4196                             }
4197                         }
4198                     }
4199                 }else{
4200                     new_coeff=0;
4201                     assert(FFABS(level)==1);
4202
4203                     if(i < last_non_zero){
4204                         int next_i= i + run2 + 1;
4205                         int next_level= block[ perm_scantable[next_i] ] + 64;
4206
4207                         if(next_level&(~127))
4208                             next_level= 0;
4209
4210                         if(next_i < last_non_zero)
4211                             score +=   length[UNI_AC_ENC_INDEX(run + run2 + 1, next_level)]
4212                                      - length[UNI_AC_ENC_INDEX(run2, next_level)]
4213                                      - length[UNI_AC_ENC_INDEX(run, 65)];
4214                         else
4215                             score +=   last_length[UNI_AC_ENC_INDEX(run + run2 + 1, next_level)]
4216                                      - last_length[UNI_AC_ENC_INDEX(run2, next_level)]
4217                                      - length[UNI_AC_ENC_INDEX(run, 65)];
4218                     }else{
4219                         score += -last_length[UNI_AC_ENC_INDEX(run, 65)];
4220                         if(prev_level){
4221                             score +=  last_length[UNI_AC_ENC_INDEX(prev_run, prev_level)]
4222                                     - length[UNI_AC_ENC_INDEX(prev_run, prev_level)];
4223                         }
4224                     }
4225                 }
4226
4227                 score *= lambda;
4228
4229                 unquant_change= new_coeff - old_coeff;
4230                 assert((score < 100*lambda && score > -100*lambda) || lambda==0);
4231
4232                 score += s->mpvencdsp.try_8x8basis(rem, weight, basis[j],
4233                                                    unquant_change);
4234                 if(score<best_score){
4235                     best_score= score;
4236                     best_coeff= i;
4237                     best_change= change;
4238                     best_unquant_change= unquant_change;
4239                 }
4240             }
4241             if(level){
4242                 prev_level= level + 64;
4243                 if(prev_level&(~127))
4244                     prev_level= 0;
4245                 prev_run= run;
4246                 run=0;
4247             }else{
4248                 run++;
4249             }
4250         }
4251 #ifdef REFINE_STATS
4252 STOP_TIMER("iterative step")}
4253 #endif
4254
4255         if(best_change){
4256             int j= perm_scantable[ best_coeff ];
4257
4258             block[j] += best_change;
4259
4260             if(best_coeff > last_non_zero){
4261                 last_non_zero= best_coeff;
4262                 assert(block[j]);
4263 #ifdef REFINE_STATS
4264 after_last++;
4265 #endif
4266             }else{
4267 #ifdef REFINE_STATS
4268 if(block[j]){
4269     if(block[j] - best_change){
4270         if(FFABS(block[j]) > FFABS(block[j] - best_change)){
4271             raise++;
4272         }else{
4273             lower++;
4274         }
4275     }else{
4276         from_zero++;
4277     }
4278 }else{
4279     to_zero++;
4280 }
4281 #endif
4282                 for(; last_non_zero>=start_i; last_non_zero--){
4283                     if(block[perm_scantable[last_non_zero]])
4284                         break;
4285                 }
4286             }
4287 #ifdef REFINE_STATS
4288 count++;
4289 if(256*256*256*64 % count == 0){
4290     printf("after_last:%d to_zero:%d from_zero:%d raise:%d lower:%d sign:%d xyp:%d/%d/%d\n", after_last, to_zero, from_zero, raise, lower, messed_sign, s->mb_x, s->mb_y, s->picture_number);
4291 }
4292 #endif
4293             run=0;
4294             rle_index=0;
4295             for(i=start_i; i<=last_non_zero; i++){
4296                 int j= perm_scantable[i];
4297                 const int level= block[j];
4298
4299                  if(level){
4300                      run_tab[rle_index++]=run;
4301                      run=0;
4302                  }else{
4303                      run++;
4304                  }
4305             }
4306
4307             s->mpvencdsp.add_8x8basis(rem, basis[j], best_unquant_change);
4308         }else{
4309             break;
4310         }
4311     }
4312 #ifdef REFINE_STATS
4313 if(last_non_zero>0){
4314 STOP_TIMER("iterative search")
4315 }
4316 }
4317 #endif
4318
4319     return last_non_zero;
4320 }
4321
4322 /**
4323  * Permute an 8x8 block according to permuatation.
4324  * @param block the block which will be permuted according to
4325  *              the given permutation vector
4326  * @param permutation the permutation vector
4327  * @param last the last non zero coefficient in scantable order, used to
4328  *             speed the permutation up
4329  * @param scantable the used scantable, this is only used to speed the
4330  *                  permutation up, the block is not (inverse) permutated
4331  *                  to scantable order!
4332  */
4333 static void block_permute(int16_t *block, uint8_t *permutation,
4334                           const uint8_t *scantable, int last)
4335 {
4336     int i;
4337     int16_t temp[64];
4338
4339     if (last <= 0)
4340         return;
4341     //FIXME it is ok but not clean and might fail for some permutations
4342     // if (permutation[1] == 1)
4343     // return;
4344
4345     for (i = 0; i <= last; i++) {
4346         const int j = scantable[i];
4347         temp[j] = block[j];
4348         block[j] = 0;
4349     }
4350
4351     for (i = 0; i <= last; i++) {
4352         const int j = scantable[i];
4353         const int perm_j = permutation[j];
4354         block[perm_j] = temp[j];
4355     }
4356 }
4357
4358 int ff_dct_quantize_c(MpegEncContext *s,
4359                         int16_t *block, int n,
4360                         int qscale, int *overflow)
4361 {
4362     int i, j, level, last_non_zero, q, start_i;
4363     const int *qmat;
4364     const uint8_t *scantable= s->intra_scantable.scantable;
4365     int bias;
4366     int max=0;
4367     unsigned int threshold1, threshold2;
4368
4369     s->fdsp.fdct(block);
4370
4371     if(s->dct_error_sum)
4372         s->denoise_dct(s, block);
4373
4374     if (s->mb_intra) {
4375         if (!s->h263_aic) {
4376             if (n < 4)
4377                 q = s->y_dc_scale;
4378             else
4379                 q = s->c_dc_scale;
4380             q = q << 3;
4381         } else
4382             /* For AIC we skip quant/dequant of INTRADC */
4383             q = 1 << 3;
4384
4385         /* note: block[0] is assumed to be positive */
4386         block[0] = (block[0] + (q >> 1)) / q;
4387         start_i = 1;
4388         last_non_zero = 0;
4389         qmat = s->q_intra_matrix[qscale];
4390         bias= s->intra_quant_bias<<(QMAT_SHIFT - QUANT_BIAS_SHIFT);
4391     } else {
4392         start_i = 0;
4393         last_non_zero = -1;
4394         qmat = s->q_inter_matrix[qscale];
4395         bias= s->inter_quant_bias<<(QMAT_SHIFT - QUANT_BIAS_SHIFT);
4396     }
4397     threshold1= (1<<QMAT_SHIFT) - bias - 1;
4398     threshold2= (threshold1<<1);
4399     for(i=63;i>=start_i;i--) {
4400         j = scantable[i];
4401         level = block[j] * qmat[j];
4402
4403         if(((unsigned)(level+threshold1))>threshold2){
4404             last_non_zero = i;
4405             break;
4406         }else{
4407             block[j]=0;
4408         }
4409     }
4410     for(i=start_i; i<=last_non_zero; i++) {
4411         j = scantable[i];
4412         level = block[j] * qmat[j];
4413
4414 //        if(   bias+level >= (1<<QMAT_SHIFT)
4415 //           || bias-level >= (1<<QMAT_SHIFT)){
4416         if(((unsigned)(level+threshold1))>threshold2){
4417             if(level>0){
4418                 level= (bias + level)>>QMAT_SHIFT;
4419                 block[j]= level;
4420             }else{
4421                 level= (bias - level)>>QMAT_SHIFT;
4422                 block[j]= -level;
4423             }
4424             max |=level;
4425         }else{
4426             block[j]=0;
4427         }
4428     }
4429     *overflow= s->max_qcoeff < max; //overflow might have happened
4430
4431     /* we need this permutation so that we correct the IDCT, we only permute the !=0 elements */
4432     if (s->idsp.perm_type != FF_IDCT_PERM_NONE)
4433         block_permute(block, s->idsp.idct_permutation,
4434                       scantable, last_non_zero);
4435
4436     return last_non_zero;
4437 }
4438
4439 #define OFFSET(x) offsetof(MpegEncContext, x)
4440 #define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM
4441 static const AVOption h263_options[] = {
4442     { "obmc",         "use overlapped block motion compensation.", OFFSET(obmc), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, VE },
4443     { "structured_slices","Write slice start position at every GOB header instead of just GOB number.", OFFSET(h263_slice_structured), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, VE},
4444     { "mb_info",      "emit macroblock info for RFC 2190 packetization, the parameter value is the maximum payload size", OFFSET(mb_info), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, VE },
4445     FF_MPV_COMMON_OPTS
4446     { NULL },
4447 };
4448
4449 static const AVClass h263_class = {
4450     .class_name = "H.263 encoder",
4451     .item_name  = av_default_item_name,
4452     .option     = h263_options,
4453     .version    = LIBAVUTIL_VERSION_INT,
4454 };
4455
4456 AVCodec ff_h263_encoder = {
4457     .name           = "h263",
4458     .long_name      = NULL_IF_CONFIG_SMALL("H.263 / H.263-1996"),
4459     .type           = AVMEDIA_TYPE_VIDEO,
4460     .id             = AV_CODEC_ID_H263,
4461     .priv_data_size = sizeof(MpegEncContext),
4462     .init           = ff_mpv_encode_init,
4463     .encode2        = ff_mpv_encode_picture,
4464     .close          = ff_mpv_encode_end,
4465     .pix_fmts= (const enum AVPixelFormat[]){AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE},
4466     .priv_class     = &h263_class,
4467 };
4468
4469 static const AVOption h263p_options[] = {
4470     { "umv",        "Use unlimited motion vectors.",    OFFSET(umvplus), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, VE },
4471     { "aiv",        "Use alternative inter VLC.",       OFFSET(alt_inter_vlc), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, VE },
4472     { "obmc",       "use overlapped block motion compensation.", OFFSET(obmc), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, VE },
4473     { "structured_slices", "Write slice start position at every GOB header instead of just GOB number.", OFFSET(h263_slice_structured), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, VE},
4474     FF_MPV_COMMON_OPTS
4475     { NULL },
4476 };
4477 static const AVClass h263p_class = {
4478     .class_name = "H.263p encoder",
4479     .item_name  = av_default_item_name,
4480     .option     = h263p_options,
4481     .version    = LIBAVUTIL_VERSION_INT,
4482 };
4483
4484 AVCodec ff_h263p_encoder = {
4485     .name           = "h263p",
4486     .long_name      = NULL_IF_CONFIG_SMALL("H.263+ / H.263-1998 / H.263 version 2"),
4487     .type           = AVMEDIA_TYPE_VIDEO,
4488     .id             = AV_CODEC_ID_H263P,
4489     .priv_data_size = sizeof(MpegEncContext),
4490     .init           = ff_mpv_encode_init,
4491     .encode2        = ff_mpv_encode_picture,
4492     .close          = ff_mpv_encode_end,
4493     .capabilities   = AV_CODEC_CAP_SLICE_THREADS,
4494     .pix_fmts       = (const enum AVPixelFormat[]){ AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE },
4495     .priv_class     = &h263p_class,
4496 };
4497
4498 static const AVClass msmpeg4v2_class = {
4499     .class_name = "msmpeg4v2 encoder",
4500     .item_name  = av_default_item_name,
4501     .option     = ff_mpv_generic_options,
4502     .version    = LIBAVUTIL_VERSION_INT,
4503 };
4504
4505 AVCodec ff_msmpeg4v2_encoder = {
4506     .name           = "msmpeg4v2",
4507     .long_name      = NULL_IF_CONFIG_SMALL("MPEG-4 part 2 Microsoft variant version 2"),
4508     .type           = AVMEDIA_TYPE_VIDEO,
4509     .id             = AV_CODEC_ID_MSMPEG4V2,
4510     .priv_data_size = sizeof(MpegEncContext),
4511     .init           = ff_mpv_encode_init,
4512     .encode2        = ff_mpv_encode_picture,
4513     .close          = ff_mpv_encode_end,
4514     .pix_fmts       = (const enum AVPixelFormat[]){ AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE },
4515     .priv_class     = &msmpeg4v2_class,
4516 };
4517
4518 static const AVClass msmpeg4v3_class = {
4519     .class_name = "msmpeg4v3 encoder",
4520     .item_name  = av_default_item_name,
4521     .option     = ff_mpv_generic_options,
4522     .version    = LIBAVUTIL_VERSION_INT,
4523 };
4524
4525 AVCodec ff_msmpeg4v3_encoder = {
4526     .name           = "msmpeg4",
4527     .long_name      = NULL_IF_CONFIG_SMALL("MPEG-4 part 2 Microsoft variant version 3"),
4528     .type           = AVMEDIA_TYPE_VIDEO,
4529     .id             = AV_CODEC_ID_MSMPEG4V3,
4530     .priv_data_size = sizeof(MpegEncContext),
4531     .init           = ff_mpv_encode_init,
4532     .encode2        = ff_mpv_encode_picture,
4533     .close          = ff_mpv_encode_end,
4534     .pix_fmts       = (const enum AVPixelFormat[]){ AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE },
4535     .priv_class     = &msmpeg4v3_class,
4536 };
4537
4538 static const AVClass wmv1_class = {
4539     .class_name = "wmv1 encoder",
4540     .item_name  = av_default_item_name,
4541     .option     = ff_mpv_generic_options,
4542     .version    = LIBAVUTIL_VERSION_INT,
4543 };
4544
4545 AVCodec ff_wmv1_encoder = {
4546     .name           = "wmv1",
4547     .long_name      = NULL_IF_CONFIG_SMALL("Windows Media Video 7"),
4548     .type           = AVMEDIA_TYPE_VIDEO,
4549     .id             = AV_CODEC_ID_WMV1,
4550     .priv_data_size = sizeof(MpegEncContext),
4551     .init           = ff_mpv_encode_init,
4552     .encode2        = ff_mpv_encode_picture,
4553     .close          = ff_mpv_encode_end,
4554     .pix_fmts       = (const enum AVPixelFormat[]){ AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE },
4555     .priv_class     = &wmv1_class,
4556 };