git.sesse.net Git - ffmpeg/blob - libavcodec/videotoolboxenc.c

   1 /*
   2  * copyright (c) 2015 Rick Kern <kernrj@gmail.com>
   3  *
   4  * This file is part of FFmpeg.
   5  *
   6  * FFmpeg is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * FFmpeg is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with FFmpeg; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  19  */
  20
  21 #include <VideoToolbox/VideoToolbox.h>
  22 #include <CoreVideo/CoreVideo.h>
  23 #include <CoreMedia/CoreMedia.h>
  24 #include <TargetConditionals.h>
  25 #include <Availability.h>
  26 #include "avcodec.h"
  27 #include "libavutil/opt.h"
  28 #include "libavutil/avassert.h"
  29 #include "libavutil/avstring.h"
  30 #include "libavcodec/avcodec.h"
  31 #include "libavutil/pixdesc.h"
  32 #include "internal.h"
  33 #include <pthread.h>
  34 #include "atsc_a53.h"
  35 #include "h264.h"
  36 #include "h264_sei.h"
  37 #include <dlfcn.h>
  38
  39 #if !HAVE_KCMVIDEOCODECTYPE_HEVC
  40 enum { kCMVideoCodecType_HEVC = 'hvc1' };
  41 #endif
  42
  43 #if !HAVE_KCMVIDEOCODECTYPE_HEVCWITHALPHA
  44 enum { kCMVideoCodecType_HEVCWithAlpha = 'muxa' };
  45 #endif
  46
  47 #if !HAVE_KCVPIXELFORMATTYPE_420YPCBCR10BIPLANARVIDEORANGE
  48 enum { kCVPixelFormatType_420YpCbCr10BiPlanarFullRange = 'xf20' };
  49 enum { kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange = 'x420' };
  50 #endif
  51
  52 typedef OSStatus (*getParameterSetAtIndex)(CMFormatDescriptionRef videoDesc,
  53                                            size_t parameterSetIndex,
  54                                            const uint8_t **parameterSetPointerOut,
  55                                            size_t *parameterSetSizeOut,
  56                                            size_t *parameterSetCountOut,
  57                                            int *NALUnitHeaderLengthOut);
  58
  59 //These symbols may not be present
  60 static struct{
  61     CFStringRef kCVImageBufferColorPrimaries_ITU_R_2020;
  62     CFStringRef kCVImageBufferTransferFunction_ITU_R_2020;
  63     CFStringRef kCVImageBufferYCbCrMatrix_ITU_R_2020;
  64
  65     CFStringRef kVTCompressionPropertyKey_H264EntropyMode;
  66     CFStringRef kVTH264EntropyMode_CAVLC;
  67     CFStringRef kVTH264EntropyMode_CABAC;
  68
  69     CFStringRef kVTProfileLevel_H264_Baseline_4_0;
  70     CFStringRef kVTProfileLevel_H264_Baseline_4_2;
  71     CFStringRef kVTProfileLevel_H264_Baseline_5_0;
  72     CFStringRef kVTProfileLevel_H264_Baseline_5_1;
  73     CFStringRef kVTProfileLevel_H264_Baseline_5_2;
  74     CFStringRef kVTProfileLevel_H264_Baseline_AutoLevel;
  75     CFStringRef kVTProfileLevel_H264_Main_4_2;
  76     CFStringRef kVTProfileLevel_H264_Main_5_1;
  77     CFStringRef kVTProfileLevel_H264_Main_5_2;
  78     CFStringRef kVTProfileLevel_H264_Main_AutoLevel;
  79     CFStringRef kVTProfileLevel_H264_High_3_0;
  80     CFStringRef kVTProfileLevel_H264_High_3_1;
  81     CFStringRef kVTProfileLevel_H264_High_3_2;
  82     CFStringRef kVTProfileLevel_H264_High_4_0;
  83     CFStringRef kVTProfileLevel_H264_High_4_1;
  84     CFStringRef kVTProfileLevel_H264_High_4_2;
  85     CFStringRef kVTProfileLevel_H264_High_5_1;
  86     CFStringRef kVTProfileLevel_H264_High_5_2;
  87     CFStringRef kVTProfileLevel_H264_High_AutoLevel;
  88     CFStringRef kVTProfileLevel_H264_Extended_5_0;
  89     CFStringRef kVTProfileLevel_H264_Extended_AutoLevel;
  90
  91     CFStringRef kVTProfileLevel_HEVC_Main_AutoLevel;
  92     CFStringRef kVTProfileLevel_HEVC_Main10_AutoLevel;
  93
  94     CFStringRef kVTCompressionPropertyKey_RealTime;
  95     CFStringRef kVTCompressionPropertyKey_TargetQualityForAlpha;
  96
  97     CFStringRef kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder;
  98     CFStringRef kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder;
  99
 100     getParameterSetAtIndex CMVideoFormatDescriptionGetHEVCParameterSetAtIndex;
 101 } compat_keys;
 102
 103 #define GET_SYM(symbol, defaultVal)                                     \
 104 do{                                                                     \
 105     CFStringRef* handle = (CFStringRef*)dlsym(RTLD_DEFAULT, #symbol);   \
 106     if(!handle)                                                         \
 107         compat_keys.symbol = CFSTR(defaultVal);                         \
 108     else                                                                \
 109         compat_keys.symbol = *handle;                                   \
 110 }while(0)
 111
 112 static pthread_once_t once_ctrl = PTHREAD_ONCE_INIT;
 113
 114 static void loadVTEncSymbols(){
 115     compat_keys.CMVideoFormatDescriptionGetHEVCParameterSetAtIndex =
 116         (getParameterSetAtIndex)dlsym(
 117             RTLD_DEFAULT,
 118             "CMVideoFormatDescriptionGetHEVCParameterSetAtIndex"
 119         );
 120
 121     GET_SYM(kCVImageBufferColorPrimaries_ITU_R_2020,   "ITU_R_2020");
 122     GET_SYM(kCVImageBufferTransferFunction_ITU_R_2020, "ITU_R_2020");
 123     GET_SYM(kCVImageBufferYCbCrMatrix_ITU_R_2020,      "ITU_R_2020");
 124
 125     GET_SYM(kVTCompressionPropertyKey_H264EntropyMode, "H264EntropyMode");
 126     GET_SYM(kVTH264EntropyMode_CAVLC, "CAVLC");
 127     GET_SYM(kVTH264EntropyMode_CABAC, "CABAC");
 128
 129     GET_SYM(kVTProfileLevel_H264_Baseline_4_0,       "H264_Baseline_4_0");
 130     GET_SYM(kVTProfileLevel_H264_Baseline_4_2,       "H264_Baseline_4_2");
 131     GET_SYM(kVTProfileLevel_H264_Baseline_5_0,       "H264_Baseline_5_0");
 132     GET_SYM(kVTProfileLevel_H264_Baseline_5_1,       "H264_Baseline_5_1");
 133     GET_SYM(kVTProfileLevel_H264_Baseline_5_2,       "H264_Baseline_5_2");
 134     GET_SYM(kVTProfileLevel_H264_Baseline_AutoLevel, "H264_Baseline_AutoLevel");
 135     GET_SYM(kVTProfileLevel_H264_Main_4_2,           "H264_Main_4_2");
 136     GET_SYM(kVTProfileLevel_H264_Main_5_1,           "H264_Main_5_1");
 137     GET_SYM(kVTProfileLevel_H264_Main_5_2,           "H264_Main_5_2");
 138     GET_SYM(kVTProfileLevel_H264_Main_AutoLevel,     "H264_Main_AutoLevel");
 139     GET_SYM(kVTProfileLevel_H264_High_3_0,           "H264_High_3_0");
 140     GET_SYM(kVTProfileLevel_H264_High_3_1,           "H264_High_3_1");
 141     GET_SYM(kVTProfileLevel_H264_High_3_2,           "H264_High_3_2");
 142     GET_SYM(kVTProfileLevel_H264_High_4_0,           "H264_High_4_0");
 143     GET_SYM(kVTProfileLevel_H264_High_4_1,           "H264_High_4_1");
 144     GET_SYM(kVTProfileLevel_H264_High_4_2,           "H264_High_4_2");
 145     GET_SYM(kVTProfileLevel_H264_High_5_1,           "H264_High_5_1");
 146     GET_SYM(kVTProfileLevel_H264_High_5_2,           "H264_High_5_2");
 147     GET_SYM(kVTProfileLevel_H264_High_AutoLevel,     "H264_High_AutoLevel");
 148     GET_SYM(kVTProfileLevel_H264_Extended_5_0,       "H264_Extended_5_0");
 149     GET_SYM(kVTProfileLevel_H264_Extended_AutoLevel, "H264_Extended_AutoLevel");
 150
 151     GET_SYM(kVTProfileLevel_HEVC_Main_AutoLevel,     "HEVC_Main_AutoLevel");
 152     GET_SYM(kVTProfileLevel_HEVC_Main10_AutoLevel,   "HEVC_Main10_AutoLevel");
 153
 154     GET_SYM(kVTCompressionPropertyKey_RealTime, "RealTime");
 155     GET_SYM(kVTCompressionPropertyKey_TargetQualityForAlpha,
 156             "TargetQualityForAlpha");
 157
 158     GET_SYM(kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder,
 159             "EnableHardwareAcceleratedVideoEncoder");
 160     GET_SYM(kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder,
 161             "RequireHardwareAcceleratedVideoEncoder");
 162 }
 163
 164 typedef enum VT_H264Profile {
 165     H264_PROF_AUTO,
 166     H264_PROF_BASELINE,
 167     H264_PROF_MAIN,
 168     H264_PROF_HIGH,
 169     H264_PROF_EXTENDED,
 170     H264_PROF_COUNT
 171 } VT_H264Profile;
 172
 173 typedef enum VTH264Entropy{
 174     VT_ENTROPY_NOT_SET,
 175     VT_CAVLC,
 176     VT_CABAC
 177 } VTH264Entropy;
 178
 179 typedef enum VT_HEVCProfile {
 180     HEVC_PROF_AUTO,
 181     HEVC_PROF_MAIN,
 182     HEVC_PROF_MAIN10,
 183     HEVC_PROF_COUNT
 184 } VT_HEVCProfile;
 185
 186 static const uint8_t start_code[] = { 0, 0, 0, 1 };
 187
 188 typedef struct ExtraSEI {
 189   void *data;
 190   size_t size;
 191 } ExtraSEI;
 192
 193 typedef struct BufNode {
 194     CMSampleBufferRef cm_buffer;
 195     ExtraSEI *sei;
 196     struct BufNode* next;
 197     int error;
 198 } BufNode;
 199
 200 typedef struct VTEncContext {
 201     AVClass *class;
 202     enum AVCodecID codec_id;
 203     VTCompressionSessionRef session;
 204     CFStringRef ycbcr_matrix;
 205     CFStringRef color_primaries;
 206     CFStringRef transfer_function;
 207     getParameterSetAtIndex get_param_set_func;
 208
 209     pthread_mutex_t lock;
 210     pthread_cond_t  cv_sample_sent;
 211
 212     int async_error;
 213
 214     BufNode *q_head;
 215     BufNode *q_tail;
 216
 217     int64_t frame_ct_out;
 218     int64_t frame_ct_in;
 219
 220     int64_t first_pts;
 221     int64_t dts_delta;
 222
 223     int64_t profile;
 224     int64_t level;
 225     int64_t entropy;
 226     int64_t realtime;
 227     int64_t frames_before;
 228     int64_t frames_after;
 229
 230     int64_t allow_sw;
 231     int64_t require_sw;
 232     double alpha_quality;
 233
 234     bool flushing;
 235     int has_b_frames;
 236     bool warned_color_range;
 237
 238     /* can't be bool type since AVOption will access it as int */
 239     int a53_cc;
 240 } VTEncContext;
 241
 242 static int vtenc_populate_extradata(AVCodecContext   *avctx,
 243                                     CMVideoCodecType codec_type,
 244                                     CFStringRef      profile_level,
 245                                     CFNumberRef      gamma_level,
 246                                     CFDictionaryRef  enc_info,
 247                                     CFDictionaryRef  pixel_buffer_info);
 248
 249 /**
 250  * NULL-safe release of *refPtr, and sets value to NULL.
 251  */
 252 static void vt_release_num(CFNumberRef* refPtr){
 253     if (!*refPtr) {
 254         return;
 255     }
 256
 257     CFRelease(*refPtr);
 258     *refPtr = NULL;
 259 }
 260
 261 static void set_async_error(VTEncContext *vtctx, int err)
 262 {
 263     BufNode *info;
 264
 265     pthread_mutex_lock(&vtctx->lock);
 266
 267     vtctx->async_error = err;
 268
 269     info = vtctx->q_head;
 270     vtctx->q_head = vtctx->q_tail = NULL;
 271
 272     while (info) {
 273         BufNode *next = info->next;
 274         CFRelease(info->cm_buffer);
 275         av_free(info);
 276         info = next;
 277     }
 278
 279     pthread_mutex_unlock(&vtctx->lock);
 280 }
 281
 282 static void clear_frame_queue(VTEncContext *vtctx)
 283 {
 284     set_async_error(vtctx, 0);
 285 }
 286
 287 static int vtenc_q_pop(VTEncContext *vtctx, bool wait, CMSampleBufferRef *buf, ExtraSEI **sei)
 288 {
 289     BufNode *info;
 290
 291     pthread_mutex_lock(&vtctx->lock);
 292
 293     if (vtctx->async_error) {
 294         pthread_mutex_unlock(&vtctx->lock);
 295         return vtctx->async_error;
 296     }
 297
 298     if (vtctx->flushing && vtctx->frame_ct_in == vtctx->frame_ct_out) {
 299         *buf = NULL;
 300
 301         pthread_mutex_unlock(&vtctx->lock);
 302         return 0;
 303     }
 304
 305     while (!vtctx->q_head && !vtctx->async_error && wait && !vtctx->flushing) {
 306         pthread_cond_wait(&vtctx->cv_sample_sent, &vtctx->lock);
 307     }
 308
 309     if (!vtctx->q_head) {
 310         pthread_mutex_unlock(&vtctx->lock);
 311         *buf = NULL;
 312         return 0;
 313     }
 314
 315     info = vtctx->q_head;
 316     vtctx->q_head = vtctx->q_head->next;
 317     if (!vtctx->q_head) {
 318         vtctx->q_tail = NULL;
 319     }
 320
 321     vtctx->frame_ct_out++;
 322     pthread_mutex_unlock(&vtctx->lock);
 323
 324     *buf = info->cm_buffer;
 325     if (sei && *buf) {
 326         *sei = info->sei;
 327     } else if (info->sei) {
 328         if (info->sei->data) av_free(info->sei->data);
 329         av_free(info->sei);
 330     }
 331     av_free(info);
 332
 333
 334     return 0;
 335 }
 336
 337 static void vtenc_q_push(VTEncContext *vtctx, CMSampleBufferRef buffer, ExtraSEI *sei)
 338 {
 339     BufNode *info = av_malloc(sizeof(BufNode));
 340     if (!info) {
 341         set_async_error(vtctx, AVERROR(ENOMEM));
 342         return;
 343     }
 344
 345     CFRetain(buffer);
 346     info->cm_buffer = buffer;
 347     info->sei = sei;
 348     info->next = NULL;
 349
 350     pthread_mutex_lock(&vtctx->lock);
 351
 352     if (!vtctx->q_head) {
 353         vtctx->q_head = info;
 354     } else {
 355         vtctx->q_tail->next = info;
 356     }
 357
 358     vtctx->q_tail = info;
 359
 360     pthread_cond_signal(&vtctx->cv_sample_sent);
 361     pthread_mutex_unlock(&vtctx->lock);
 362 }
 363
 364 static int count_nalus(size_t length_code_size,
 365                        CMSampleBufferRef sample_buffer,
 366                        int *count)
 367 {
 368     size_t offset = 0;
 369     int status;
 370     int nalu_ct = 0;
 371     uint8_t size_buf[4];
 372     size_t src_size = CMSampleBufferGetTotalSampleSize(sample_buffer);
 373     CMBlockBufferRef block = CMSampleBufferGetDataBuffer(sample_buffer);
 374
 375     if (length_code_size > 4)
 376         return AVERROR_INVALIDDATA;
 377
 378     while (offset < src_size) {
 379         size_t curr_src_len;
 380         size_t box_len = 0;
 381         size_t i;
 382
 383         status = CMBlockBufferCopyDataBytes(block,
 384                                             offset,
 385                                             length_code_size,
 386                                             size_buf);
 387
 388         for (i = 0; i < length_code_size; i++) {
 389             box_len <<= 8;
 390             box_len |= size_buf[i];
 391         }
 392
 393         curr_src_len = box_len + length_code_size;
 394         offset += curr_src_len;
 395
 396         nalu_ct++;
 397     }
 398
 399     *count = nalu_ct;
 400     return 0;
 401 }
 402
 403 static CMVideoCodecType get_cm_codec_type(enum AVCodecID id,
 404                                           enum AVPixelFormat fmt,
 405                                           double alpha_quality)
 406 {
 407     switch (id) {
 408     case AV_CODEC_ID_H264: return kCMVideoCodecType_H264;
 409     case AV_CODEC_ID_HEVC:
 410         if (fmt == AV_PIX_FMT_BGRA && alpha_quality > 0.0) {
 411             return kCMVideoCodecType_HEVCWithAlpha;
 412         }
 413         return kCMVideoCodecType_HEVC;
 414     default:               return 0;
 415     }
 416 }
 417
 418 /**
 419  * Get the parameter sets from a CMSampleBufferRef.
 420  * @param dst If *dst isn't NULL, the parameters are copied into existing
 421  *            memory. *dst_size must be set accordingly when *dst != NULL.
 422  *            If *dst is NULL, it will be allocated.
 423  *            In all cases, *dst_size is set to the number of bytes used starting
 424  *            at *dst.
 425  */
 426 static int get_params_size(
 427     AVCodecContext              *avctx,
 428     CMVideoFormatDescriptionRef vid_fmt,
 429     size_t                      *size)
 430 {
 431     VTEncContext *vtctx = avctx->priv_data;
 432     size_t total_size = 0;
 433     size_t ps_count;
 434     int is_count_bad = 0;
 435     size_t i;
 436     int status;
 437     status = vtctx->get_param_set_func(vid_fmt,
 438                                        0,
 439                                        NULL,
 440                                        NULL,
 441                                        &ps_count,
 442                                        NULL);
 443     if (status) {
 444         is_count_bad = 1;
 445         ps_count     = 0;
 446         status       = 0;
 447     }
 448
 449     for (i = 0; i < ps_count || is_count_bad; i++) {
 450         const uint8_t *ps;
 451         size_t ps_size;
 452         status = vtctx->get_param_set_func(vid_fmt,
 453                                            i,
 454                                            &ps,
 455                                            &ps_size,
 456                                            NULL,
 457                                            NULL);
 458         if (status) {
 459             /*
 460              * When ps_count is invalid, status != 0 ends the loop normally
 461              * unless we didn't get any parameter sets.
 462              */
 463             if (i > 0 && is_count_bad) status = 0;
 464
 465             break;
 466         }
 467
 468         total_size += ps_size + sizeof(start_code);
 469     }
 470
 471     if (status) {
 472         av_log(avctx, AV_LOG_ERROR, "Error getting parameter set sizes: %d\n", status);
 473         return AVERROR_EXTERNAL;
 474     }
 475
 476     *size = total_size;
 477     return 0;
 478 }
 479
 480 static int copy_param_sets(
 481     AVCodecContext              *avctx,
 482     CMVideoFormatDescriptionRef vid_fmt,
 483     uint8_t                     *dst,
 484     size_t                      dst_size)
 485 {
 486     VTEncContext *vtctx = avctx->priv_data;
 487     size_t ps_count;
 488     int is_count_bad = 0;
 489     int status;
 490     size_t offset = 0;
 491     size_t i;
 492
 493     status = vtctx->get_param_set_func(vid_fmt,
 494                                        0,
 495                                        NULL,
 496                                        NULL,
 497                                        &ps_count,
 498                                        NULL);
 499     if (status) {
 500         is_count_bad = 1;
 501         ps_count     = 0;
 502         status       = 0;
 503     }
 504
 505
 506     for (i = 0; i < ps_count || is_count_bad; i++) {
 507         const uint8_t *ps;
 508         size_t ps_size;
 509         size_t next_offset;
 510
 511         status = vtctx->get_param_set_func(vid_fmt,
 512                                            i,
 513                                            &ps,
 514                                            &ps_size,
 515                                            NULL,
 516                                            NULL);
 517         if (status) {
 518             if (i > 0 && is_count_bad) status = 0;
 519
 520             break;
 521         }
 522
 523         next_offset = offset + sizeof(start_code) + ps_size;
 524         if (dst_size < next_offset) {
 525             av_log(avctx, AV_LOG_ERROR, "Error: buffer too small for parameter sets.\n");
 526             return AVERROR_BUFFER_TOO_SMALL;
 527         }
 528
 529         memcpy(dst + offset, start_code, sizeof(start_code));
 530         offset += sizeof(start_code);
 531
 532         memcpy(dst + offset, ps, ps_size);
 533         offset = next_offset;
 534     }
 535
 536     if (status) {
 537         av_log(avctx, AV_LOG_ERROR, "Error getting parameter set data: %d\n", status);
 538         return AVERROR_EXTERNAL;
 539     }
 540
 541     return 0;
 542 }
 543
 544 static int set_extradata(AVCodecContext *avctx, CMSampleBufferRef sample_buffer)
 545 {
 546     CMVideoFormatDescriptionRef vid_fmt;
 547     size_t total_size;
 548     int status;
 549
 550     vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer);
 551     if (!vid_fmt) {
 552         av_log(avctx, AV_LOG_ERROR, "No video format.\n");
 553         return AVERROR_EXTERNAL;
 554     }
 555
 556     status = get_params_size(avctx, vid_fmt, &total_size);
 557     if (status) {
 558         av_log(avctx, AV_LOG_ERROR, "Could not get parameter sets.\n");
 559         return status;
 560     }
 561
 562     avctx->extradata = av_mallocz(total_size + AV_INPUT_BUFFER_PADDING_SIZE);
 563     if (!avctx->extradata) {
 564         return AVERROR(ENOMEM);
 565     }
 566     avctx->extradata_size = total_size;
 567
 568     status = copy_param_sets(avctx, vid_fmt, avctx->extradata, total_size);
 569
 570     if (status) {
 571         av_log(avctx, AV_LOG_ERROR, "Could not copy param sets.\n");
 572         return status;
 573     }
 574
 575     return 0;
 576 }
 577
 578 static void vtenc_output_callback(
 579     void *ctx,
 580     void *sourceFrameCtx,
 581     OSStatus status,
 582     VTEncodeInfoFlags flags,
 583     CMSampleBufferRef sample_buffer)
 584 {
 585     AVCodecContext *avctx = ctx;
 586     VTEncContext   *vtctx = avctx->priv_data;
 587     ExtraSEI *sei = sourceFrameCtx;
 588
 589     if (vtctx->async_error) {
 590         return;
 591     }
 592
 593     if (status) {
 594         av_log(avctx, AV_LOG_ERROR, "Error encoding frame: %d\n", (int)status);
 595         set_async_error(vtctx, AVERROR_EXTERNAL);
 596         return;
 597     }
 598
 599     if (!sample_buffer) {
 600         return;
 601     }
 602
 603     if (!avctx->extradata && (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER)) {
 604         int set_status = set_extradata(avctx, sample_buffer);
 605         if (set_status) {
 606             set_async_error(vtctx, set_status);
 607             return;
 608         }
 609     }
 610
 611     vtenc_q_push(vtctx, sample_buffer, sei);
 612 }
 613
 614 static int get_length_code_size(
 615     AVCodecContext    *avctx,
 616     CMSampleBufferRef sample_buffer,
 617     size_t            *size)
 618 {
 619     VTEncContext *vtctx = avctx->priv_data;
 620     CMVideoFormatDescriptionRef vid_fmt;
 621     int isize;
 622     int status;
 623
 624     vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer);
 625     if (!vid_fmt) {
 626         av_log(avctx, AV_LOG_ERROR, "Error getting buffer format description.\n");
 627         return AVERROR_EXTERNAL;
 628     }
 629
 630     status = vtctx->get_param_set_func(vid_fmt,
 631                                        0,
 632                                        NULL,
 633                                        NULL,
 634                                        NULL,
 635                                        &isize);
 636     if (status) {
 637         av_log(avctx, AV_LOG_ERROR, "Error getting length code size: %d\n", status);
 638         return AVERROR_EXTERNAL;
 639     }
 640
 641     *size = isize;
 642     return 0;
 643 }
 644
 645 /*
 646  * Returns true on success.
 647  *
 648  * If profile_level_val is NULL and this method returns true, don't specify the
 649  * profile/level to the encoder.
 650  */
 651 static bool get_vt_h264_profile_level(AVCodecContext *avctx,
 652                                       CFStringRef    *profile_level_val)
 653 {
 654     VTEncContext *vtctx = avctx->priv_data;
 655     int64_t profile = vtctx->profile;
 656
 657     if (profile == H264_PROF_AUTO && vtctx->level) {
 658         //Need to pick a profile if level is not auto-selected.
 659         profile = vtctx->has_b_frames ? H264_PROF_MAIN : H264_PROF_BASELINE;
 660     }
 661
 662     *profile_level_val = NULL;
 663
 664     switch (profile) {
 665         case H264_PROF_AUTO:
 666             return true;
 667
 668         case H264_PROF_BASELINE:
 669             switch (vtctx->level) {
 670                 case  0: *profile_level_val =
 671                                   compat_keys.kVTProfileLevel_H264_Baseline_AutoLevel; break;
 672                 case 13: *profile_level_val = kVTProfileLevel_H264_Baseline_1_3;       break;
 673                 case 30: *profile_level_val = kVTProfileLevel_H264_Baseline_3_0;       break;
 674                 case 31: *profile_level_val = kVTProfileLevel_H264_Baseline_3_1;       break;
 675                 case 32: *profile_level_val = kVTProfileLevel_H264_Baseline_3_2;       break;
 676                 case 40: *profile_level_val =
 677                                   compat_keys.kVTProfileLevel_H264_Baseline_4_0;       break;
 678                 case 41: *profile_level_val = kVTProfileLevel_H264_Baseline_4_1;       break;
 679                 case 42: *profile_level_val =
 680                                   compat_keys.kVTProfileLevel_H264_Baseline_4_2;       break;
 681                 case 50: *profile_level_val =
 682                                   compat_keys.kVTProfileLevel_H264_Baseline_5_0;       break;
 683                 case 51: *profile_level_val =
 684                                   compat_keys.kVTProfileLevel_H264_Baseline_5_1;       break;
 685                 case 52: *profile_level_val =
 686                                   compat_keys.kVTProfileLevel_H264_Baseline_5_2;       break;
 687             }
 688             break;
 689
 690         case H264_PROF_MAIN:
 691             switch (vtctx->level) {
 692                 case  0: *profile_level_val =
 693                                   compat_keys.kVTProfileLevel_H264_Main_AutoLevel; break;
 694                 case 30: *profile_level_val = kVTProfileLevel_H264_Main_3_0;       break;
 695                 case 31: *profile_level_val = kVTProfileLevel_H264_Main_3_1;       break;
 696                 case 32: *profile_level_val = kVTProfileLevel_H264_Main_3_2;       break;
 697                 case 40: *profile_level_val = kVTProfileLevel_H264_Main_4_0;       break;
 698                 case 41: *profile_level_val = kVTProfileLevel_H264_Main_4_1;       break;
 699                 case 42: *profile_level_val =
 700                                   compat_keys.kVTProfileLevel_H264_Main_4_2;       break;
 701                 case 50: *profile_level_val = kVTProfileLevel_H264_Main_5_0;       break;
 702                 case 51: *profile_level_val =
 703                                   compat_keys.kVTProfileLevel_H264_Main_5_1;       break;
 704                 case 52: *profile_level_val =
 705                                   compat_keys.kVTProfileLevel_H264_Main_5_2;       break;
 706             }
 707             break;
 708
 709         case H264_PROF_HIGH:
 710             switch (vtctx->level) {
 711                 case  0: *profile_level_val =
 712                                   compat_keys.kVTProfileLevel_H264_High_AutoLevel; break;
 713                 case 30: *profile_level_val =
 714                                   compat_keys.kVTProfileLevel_H264_High_3_0;       break;
 715                 case 31: *profile_level_val =
 716                                   compat_keys.kVTProfileLevel_H264_High_3_1;       break;
 717                 case 32: *profile_level_val =
 718                                   compat_keys.kVTProfileLevel_H264_High_3_2;       break;
 719                 case 40: *profile_level_val =
 720                                   compat_keys.kVTProfileLevel_H264_High_4_0;       break;
 721                 case 41: *profile_level_val =
 722                                   compat_keys.kVTProfileLevel_H264_High_4_1;       break;
 723                 case 42: *profile_level_val =
 724                                   compat_keys.kVTProfileLevel_H264_High_4_2;       break;
 725                 case 50: *profile_level_val = kVTProfileLevel_H264_High_5_0;       break;
 726                 case 51: *profile_level_val =
 727                                   compat_keys.kVTProfileLevel_H264_High_5_1;       break;
 728                 case 52: *profile_level_val =
 729                                   compat_keys.kVTProfileLevel_H264_High_5_2;       break;
 730             }
 731             break;
 732         case H264_PROF_EXTENDED:
 733             switch (vtctx->level) {
 734                 case  0: *profile_level_val =
 735                                   compat_keys.kVTProfileLevel_H264_Extended_AutoLevel; break;
 736                 case 50: *profile_level_val =
 737                                   compat_keys.kVTProfileLevel_H264_Extended_5_0;       break;
 738             }
 739             break;
 740     }
 741
 742     if (!*profile_level_val) {
 743         av_log(avctx, AV_LOG_ERROR, "Invalid Profile/Level.\n");
 744         return false;
 745     }
 746
 747     return true;
 748 }
 749
 750 /*
 751  * Returns true on success.
 752  *
 753  * If profile_level_val is NULL and this method returns true, don't specify the
 754  * profile/level to the encoder.
 755  */
 756 static bool get_vt_hevc_profile_level(AVCodecContext *avctx,
 757                                       CFStringRef    *profile_level_val)
 758 {
 759     VTEncContext *vtctx = avctx->priv_data;
 760     int64_t profile = vtctx->profile;
 761
 762     *profile_level_val = NULL;
 763
 764     switch (profile) {
 765         case HEVC_PROF_AUTO:
 766             return true;
 767         case HEVC_PROF_MAIN:
 768             *profile_level_val =
 769                 compat_keys.kVTProfileLevel_HEVC_Main_AutoLevel;
 770             break;
 771         case HEVC_PROF_MAIN10:
 772             *profile_level_val =
 773                 compat_keys.kVTProfileLevel_HEVC_Main10_AutoLevel;
 774             break;
 775     }
 776
 777     if (!*profile_level_val) {
 778         av_log(avctx, AV_LOG_ERROR, "Invalid Profile/Level.\n");
 779         return false;
 780     }
 781
 782     return true;
 783 }
 784
 785 static int get_cv_pixel_format(AVCodecContext* avctx,
 786                                enum AVPixelFormat fmt,
 787                                enum AVColorRange range,
 788                                int* av_pixel_format,
 789                                int* range_guessed)
 790 {
 791     if (range_guessed) *range_guessed = range != AVCOL_RANGE_MPEG &&
 792                                         range != AVCOL_RANGE_JPEG;
 793
 794     //MPEG range is used when no range is set
 795     if (fmt == AV_PIX_FMT_NV12) {
 796         *av_pixel_format = range == AVCOL_RANGE_JPEG ?
 797                                         kCVPixelFormatType_420YpCbCr8BiPlanarFullRange :
 798                                         kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
 799     } else if (fmt == AV_PIX_FMT_YUV420P) {
 800         *av_pixel_format = range == AVCOL_RANGE_JPEG ?
 801                                         kCVPixelFormatType_420YpCbCr8PlanarFullRange :
 802                                         kCVPixelFormatType_420YpCbCr8Planar;
 803     } else if (fmt == AV_PIX_FMT_BGRA) {
 804         *av_pixel_format = kCVPixelFormatType_32BGRA;
 805     } else if (fmt == AV_PIX_FMT_P010LE) {
 806         *av_pixel_format = range == AVCOL_RANGE_JPEG ?
 807                                         kCVPixelFormatType_420YpCbCr10BiPlanarFullRange :
 808                                         kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange;
 809     } else {
 810         return AVERROR(EINVAL);
 811     }
 812
 813     return 0;
 814 }
 815
 816 static void add_color_attr(AVCodecContext *avctx, CFMutableDictionaryRef dict) {
 817     VTEncContext *vtctx = avctx->priv_data;
 818
 819     if (vtctx->color_primaries) {
 820         CFDictionarySetValue(dict,
 821                              kCVImageBufferColorPrimariesKey,
 822                              vtctx->color_primaries);
 823     }
 824
 825     if (vtctx->transfer_function) {
 826         CFDictionarySetValue(dict,
 827                              kCVImageBufferTransferFunctionKey,
 828                              vtctx->transfer_function);
 829     }
 830
 831     if (vtctx->ycbcr_matrix) {
 832         CFDictionarySetValue(dict,
 833                              kCVImageBufferYCbCrMatrixKey,
 834                              vtctx->ycbcr_matrix);
 835     }
 836 }
 837
 838 static int create_cv_pixel_buffer_info(AVCodecContext* avctx,
 839                                        CFMutableDictionaryRef* dict)
 840 {
 841     CFNumberRef cv_color_format_num = NULL;
 842     CFNumberRef width_num = NULL;
 843     CFNumberRef height_num = NULL;
 844     CFMutableDictionaryRef pixel_buffer_info = NULL;
 845     int cv_color_format;
 846     int status = get_cv_pixel_format(avctx,
 847                                      avctx->pix_fmt,
 848                                      avctx->color_range,
 849                                      &cv_color_format,
 850                                      NULL);
 851     if (status) return status;
 852
 853     pixel_buffer_info = CFDictionaryCreateMutable(
 854                             kCFAllocatorDefault,
 855                             20,
 856                             &kCFCopyStringDictionaryKeyCallBacks,
 857                             &kCFTypeDictionaryValueCallBacks);
 858
 859     if (!pixel_buffer_info) goto pbinfo_nomem;
 860
 861     cv_color_format_num = CFNumberCreate(kCFAllocatorDefault,
 862                                          kCFNumberSInt32Type,
 863                                          &cv_color_format);
 864     if (!cv_color_format_num) goto pbinfo_nomem;
 865
 866     CFDictionarySetValue(pixel_buffer_info,
 867                          kCVPixelBufferPixelFormatTypeKey,
 868                          cv_color_format_num);
 869     vt_release_num(&cv_color_format_num);
 870
 871     width_num = CFNumberCreate(kCFAllocatorDefault,
 872                                kCFNumberSInt32Type,
 873                                &avctx->width);
 874     if (!width_num) return AVERROR(ENOMEM);
 875
 876     CFDictionarySetValue(pixel_buffer_info,
 877                          kCVPixelBufferWidthKey,
 878                          width_num);
 879     vt_release_num(&width_num);
 880
 881     height_num = CFNumberCreate(kCFAllocatorDefault,
 882                                 kCFNumberSInt32Type,
 883                                 &avctx->height);
 884     if (!height_num) goto pbinfo_nomem;
 885
 886     CFDictionarySetValue(pixel_buffer_info,
 887                          kCVPixelBufferHeightKey,
 888                          height_num);
 889     vt_release_num(&height_num);
 890
 891     add_color_attr(avctx, pixel_buffer_info);
 892
 893     *dict = pixel_buffer_info;
 894     return 0;
 895
 896 pbinfo_nomem:
 897     vt_release_num(&cv_color_format_num);
 898     vt_release_num(&width_num);
 899     vt_release_num(&height_num);
 900     if (pixel_buffer_info) CFRelease(pixel_buffer_info);
 901
 902     return AVERROR(ENOMEM);
 903 }
 904
 905 static int get_cv_color_primaries(AVCodecContext *avctx,
 906                                   CFStringRef *primaries)
 907 {
 908     enum AVColorPrimaries pri = avctx->color_primaries;
 909     switch (pri) {
 910         case AVCOL_PRI_UNSPECIFIED:
 911             *primaries = NULL;
 912             break;
 913
 914         case AVCOL_PRI_BT470BG:
 915             *primaries = kCVImageBufferColorPrimaries_EBU_3213;
 916             break;
 917
 918         case AVCOL_PRI_SMPTE170M:
 919             *primaries = kCVImageBufferColorPrimaries_SMPTE_C;
 920             break;
 921
 922         case AVCOL_PRI_BT709:
 923             *primaries = kCVImageBufferColorPrimaries_ITU_R_709_2;
 924             break;
 925
 926         case AVCOL_PRI_BT2020:
 927             *primaries = compat_keys.kCVImageBufferColorPrimaries_ITU_R_2020;
 928             break;
 929
 930         default:
 931             av_log(avctx, AV_LOG_ERROR, "Color primaries %s is not supported.\n", av_color_primaries_name(pri));
 932             *primaries = NULL;
 933             return -1;
 934     }
 935
 936     return 0;
 937 }
 938
 939 static int get_cv_transfer_function(AVCodecContext *avctx,
 940                                     CFStringRef *transfer_fnc,
 941                                     CFNumberRef *gamma_level)
 942 {
 943     enum AVColorTransferCharacteristic trc = avctx->color_trc;
 944     Float32 gamma;
 945     *gamma_level = NULL;
 946
 947     switch (trc) {
 948         case AVCOL_TRC_UNSPECIFIED:
 949             *transfer_fnc = NULL;
 950             break;
 951
 952         case AVCOL_TRC_BT709:
 953             *transfer_fnc = kCVImageBufferTransferFunction_ITU_R_709_2;
 954             break;
 955
 956         case AVCOL_TRC_SMPTE240M:
 957             *transfer_fnc = kCVImageBufferTransferFunction_SMPTE_240M_1995;
 958             break;
 959
 960 #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_SMPTE_ST_2084_PQ
 961         case AVCOL_TRC_SMPTE2084:
 962             *transfer_fnc = kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ;
 963             break;
 964 #endif
 965 #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_LINEAR
 966         case AVCOL_TRC_LINEAR:
 967             *transfer_fnc = kCVImageBufferTransferFunction_Linear;
 968             break;
 969 #endif
 970 #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_ITU_R_2100_HLG
 971         case AVCOL_TRC_ARIB_STD_B67:
 972             *transfer_fnc = kCVImageBufferTransferFunction_ITU_R_2100_HLG;
 973             break;
 974 #endif
 975
 976         case AVCOL_TRC_GAMMA22:
 977             gamma = 2.2;
 978             *transfer_fnc = kCVImageBufferTransferFunction_UseGamma;
 979             *gamma_level = CFNumberCreate(NULL, kCFNumberFloat32Type, &gamma);
 980             break;
 981
 982         case AVCOL_TRC_GAMMA28:
 983             gamma = 2.8;
 984             *transfer_fnc = kCVImageBufferTransferFunction_UseGamma;
 985             *gamma_level = CFNumberCreate(NULL, kCFNumberFloat32Type, &gamma);
 986             break;
 987
 988         case AVCOL_TRC_BT2020_10:
 989         case AVCOL_TRC_BT2020_12:
 990             *transfer_fnc = compat_keys.kCVImageBufferTransferFunction_ITU_R_2020;
 991             break;
 992
 993         default:
 994             *transfer_fnc = NULL;
 995             av_log(avctx, AV_LOG_ERROR, "Transfer function %s is not supported.\n", av_color_transfer_name(trc));
 996             return -1;
 997     }
 998
 999     return 0;
1000 }
1001
1002 static int get_cv_ycbcr_matrix(AVCodecContext *avctx, CFStringRef *matrix) {
1003     switch(avctx->colorspace) {
1004         case AVCOL_SPC_BT709:
1005             *matrix = kCVImageBufferYCbCrMatrix_ITU_R_709_2;
1006             break;
1007
1008         case AVCOL_SPC_UNSPECIFIED:
1009             *matrix = NULL;
1010             break;
1011
1012         case AVCOL_SPC_BT470BG:
1013         case AVCOL_SPC_SMPTE170M:
1014             *matrix = kCVImageBufferYCbCrMatrix_ITU_R_601_4;
1015             break;
1016
1017         case AVCOL_SPC_SMPTE240M:
1018             *matrix = kCVImageBufferYCbCrMatrix_SMPTE_240M_1995;
1019             break;
1020
1021         case AVCOL_SPC_BT2020_NCL:
1022             *matrix = compat_keys.kCVImageBufferYCbCrMatrix_ITU_R_2020;
1023             break;
1024
1025         default:
1026             av_log(avctx, AV_LOG_ERROR, "Color space %s is not supported.\n", av_color_space_name(avctx->colorspace));
1027             return -1;
1028     }
1029
1030     return 0;
1031 }
1032
1033 // constant quality only on Macs with Apple Silicon
1034 static bool vtenc_qscale_enabled(void)
1035 {
1036     return TARGET_OS_OSX && TARGET_CPU_ARM64;
1037 }
1038
1039 static int vtenc_create_encoder(AVCodecContext   *avctx,
1040                                 CMVideoCodecType codec_type,
1041                                 CFStringRef      profile_level,
1042                                 CFNumberRef      gamma_level,
1043                                 CFDictionaryRef  enc_info,
1044                                 CFDictionaryRef  pixel_buffer_info,
1045                                 VTCompressionSessionRef *session)
1046 {
1047     VTEncContext *vtctx = avctx->priv_data;
1048     SInt32       bit_rate = avctx->bit_rate;
1049     SInt32       max_rate = avctx->rc_max_rate;
1050     Float32      quality = avctx->global_quality / FF_QP2LAMBDA;
1051     CFNumberRef  bit_rate_num;
1052     CFNumberRef  quality_num;
1053     CFNumberRef  bytes_per_second;
1054     CFNumberRef  one_second;
1055     CFArrayRef   data_rate_limits;
1056     int64_t      bytes_per_second_value = 0;
1057     int64_t      one_second_value = 0;
1058     void         *nums[2];
1059
1060     int status = VTCompressionSessionCreate(kCFAllocatorDefault,
1061                                             avctx->width,
1062                                             avctx->height,
1063                                             codec_type,
1064                                             enc_info,
1065                                             pixel_buffer_info,
1066                                             kCFAllocatorDefault,
1067                                             vtenc_output_callback,
1068                                             avctx,
1069                                             session);
1070
1071     if (status || !vtctx->session) {
1072         av_log(avctx, AV_LOG_ERROR, "Error: cannot create compression session: %d\n", status);
1073
1074 #if !TARGET_OS_IPHONE
1075         if (!vtctx->allow_sw) {
1076             av_log(avctx, AV_LOG_ERROR, "Try -allow_sw 1. The hardware encoder may be busy, or not supported.\n");
1077         }
1078 #endif
1079
1080         return AVERROR_EXTERNAL;
1081     }
1082
1083     if (avctx->flags & AV_CODEC_FLAG_QSCALE && !vtenc_qscale_enabled()) {
1084         av_log(avctx, AV_LOG_ERROR, "Error: -q:v qscale not available for encoder. Use -b:v bitrate instead.\n");
1085         return AVERROR_EXTERNAL;
1086     }
1087
1088     if (avctx->flags & AV_CODEC_FLAG_QSCALE) {
1089         quality = quality >= 100 ? 1.0 : quality / 100;
1090         quality_num = CFNumberCreate(kCFAllocatorDefault,
1091                                      kCFNumberFloat32Type,
1092                                      &quality);
1093         if (!quality_num) return AVERROR(ENOMEM);
1094
1095         status = VTSessionSetProperty(vtctx->session,
1096                                       kVTCompressionPropertyKey_Quality,
1097                                       quality_num);
1098         CFRelease(quality_num);
1099     } else {
1100         bit_rate_num = CFNumberCreate(kCFAllocatorDefault,
1101                                       kCFNumberSInt32Type,
1102                                       &bit_rate);
1103         if (!bit_rate_num) return AVERROR(ENOMEM);
1104
1105         status = VTSessionSetProperty(vtctx->session,
1106                                       kVTCompressionPropertyKey_AverageBitRate,
1107                                       bit_rate_num);
1108         CFRelease(bit_rate_num);
1109     }
1110
1111     if (status) {
1112         av_log(avctx, AV_LOG_ERROR, "Error setting bitrate property: %d\n", status);
1113         return AVERROR_EXTERNAL;
1114     }
1115
1116     if (vtctx->codec_id == AV_CODEC_ID_H264 && max_rate > 0) {
1117         // kVTCompressionPropertyKey_DataRateLimits is not available for HEVC
1118         bytes_per_second_value = max_rate >> 3;
1119         bytes_per_second = CFNumberCreate(kCFAllocatorDefault,
1120                                           kCFNumberSInt64Type,
1121                                           &bytes_per_second_value);
1122         if (!bytes_per_second) {
1123             return AVERROR(ENOMEM);
1124         }
1125         one_second_value = 1;
1126         one_second = CFNumberCreate(kCFAllocatorDefault,
1127                                     kCFNumberSInt64Type,
1128                                     &one_second_value);
1129         if (!one_second) {
1130             CFRelease(bytes_per_second);
1131             return AVERROR(ENOMEM);
1132         }
1133         nums[0] = (void *)bytes_per_second;
1134         nums[1] = (void *)one_second;
1135         data_rate_limits = CFArrayCreate(kCFAllocatorDefault,
1136                                          (const void **)nums,
1137                                          2,
1138                                          &kCFTypeArrayCallBacks);
1139
1140         if (!data_rate_limits) {
1141             CFRelease(bytes_per_second);
1142             CFRelease(one_second);
1143             return AVERROR(ENOMEM);
1144         }
1145         status = VTSessionSetProperty(vtctx->session,
1146                                       kVTCompressionPropertyKey_DataRateLimits,
1147                                       data_rate_limits);
1148
1149         CFRelease(bytes_per_second);
1150         CFRelease(one_second);
1151         CFRelease(data_rate_limits);
1152
1153         if (status) {
1154             av_log(avctx, AV_LOG_ERROR, "Error setting max bitrate property: %d\n", status);
1155             return AVERROR_EXTERNAL;
1156         }
1157     }
1158
1159     if (vtctx->codec_id == AV_CODEC_ID_HEVC) {
1160         if (avctx->pix_fmt == AV_PIX_FMT_BGRA && vtctx->alpha_quality > 0.0) {
1161             CFNumberRef alpha_quality_num = CFNumberCreate(kCFAllocatorDefault,
1162                                                            kCFNumberDoubleType,
1163                                                            &vtctx->alpha_quality);
1164             if (!alpha_quality_num) return AVERROR(ENOMEM);
1165
1166             status = VTSessionSetProperty(vtctx->session,
1167                                           compat_keys.kVTCompressionPropertyKey_TargetQualityForAlpha,
1168                                           alpha_quality_num);
1169             CFRelease(alpha_quality_num);
1170         }
1171     }
1172
1173     if (profile_level) {
1174         status = VTSessionSetProperty(vtctx->session,
1175                                       kVTCompressionPropertyKey_ProfileLevel,
1176                                       profile_level);
1177         if (status) {
1178             av_log(avctx, AV_LOG_ERROR, "Error setting profile/level property: %d. Output will be encoded using a supported profile/level combination.\n", status);
1179         }
1180     }
1181
1182     if (avctx->gop_size > 0) {
1183         CFNumberRef interval = CFNumberCreate(kCFAllocatorDefault,
1184                                               kCFNumberIntType,
1185                                               &avctx->gop_size);
1186         if (!interval) {
1187             return AVERROR(ENOMEM);
1188         }
1189
1190         status = VTSessionSetProperty(vtctx->session,
1191                                       kVTCompressionPropertyKey_MaxKeyFrameInterval,
1192                                       interval);
1193         CFRelease(interval);
1194
1195         if (status) {
1196             av_log(avctx, AV_LOG_ERROR, "Error setting 'max key-frame interval' property: %d\n", status);
1197             return AVERROR_EXTERNAL;
1198         }
1199     }
1200
1201     if (vtctx->frames_before) {
1202         status = VTSessionSetProperty(vtctx->session,
1203                                       kVTCompressionPropertyKey_MoreFramesBeforeStart,
1204                                       kCFBooleanTrue);
1205
1206         if (status == kVTPropertyNotSupportedErr) {
1207             av_log(avctx, AV_LOG_WARNING, "frames_before property is not supported on this device. Ignoring.\n");
1208         } else if (status) {
1209             av_log(avctx, AV_LOG_ERROR, "Error setting frames_before property: %d\n", status);
1210         }
1211     }
1212
1213     if (vtctx->frames_after) {
1214         status = VTSessionSetProperty(vtctx->session,
1215                                       kVTCompressionPropertyKey_MoreFramesAfterEnd,
1216                                       kCFBooleanTrue);
1217
1218         if (status == kVTPropertyNotSupportedErr) {
1219             av_log(avctx, AV_LOG_WARNING, "frames_after property is not supported on this device. Ignoring.\n");
1220         } else if (status) {
1221             av_log(avctx, AV_LOG_ERROR, "Error setting frames_after property: %d\n", status);
1222         }
1223     }
1224
1225     if (avctx->sample_aspect_ratio.num != 0) {
1226         CFNumberRef num;
1227         CFNumberRef den;
1228         CFMutableDictionaryRef par;
1229         AVRational *avpar = &avctx->sample_aspect_ratio;
1230
1231         av_reduce(&avpar->num, &avpar->den,
1232                    avpar->num,  avpar->den,
1233                   0xFFFFFFFF);
1234
1235         num = CFNumberCreate(kCFAllocatorDefault,
1236                              kCFNumberIntType,
1237                              &avpar->num);
1238
1239         den = CFNumberCreate(kCFAllocatorDefault,
1240                              kCFNumberIntType,
1241                              &avpar->den);
1242
1243
1244
1245         par = CFDictionaryCreateMutable(kCFAllocatorDefault,
1246                                         2,
1247                                         &kCFCopyStringDictionaryKeyCallBacks,
1248                                         &kCFTypeDictionaryValueCallBacks);
1249
1250         if (!par || !num || !den) {
1251             if (par) CFRelease(par);
1252             if (num) CFRelease(num);
1253             if (den) CFRelease(den);
1254
1255             return AVERROR(ENOMEM);
1256         }
1257
1258         CFDictionarySetValue(
1259             par,
1260             kCMFormatDescriptionKey_PixelAspectRatioHorizontalSpacing,
1261             num);
1262
1263         CFDictionarySetValue(
1264             par,
1265             kCMFormatDescriptionKey_PixelAspectRatioVerticalSpacing,
1266             den);
1267
1268         status = VTSessionSetProperty(vtctx->session,
1269                                       kVTCompressionPropertyKey_PixelAspectRatio,
1270                                       par);
1271
1272         CFRelease(par);
1273         CFRelease(num);
1274         CFRelease(den);
1275
1276         if (status) {
1277             av_log(avctx,
1278                    AV_LOG_ERROR,
1279                    "Error setting pixel aspect ratio to %d:%d: %d.\n",
1280                    avctx->sample_aspect_ratio.num,
1281                    avctx->sample_aspect_ratio.den,
1282                    status);
1283
1284             return AVERROR_EXTERNAL;
1285         }
1286     }
1287
1288
1289     if (vtctx->transfer_function) {
1290         status = VTSessionSetProperty(vtctx->session,
1291                                       kVTCompressionPropertyKey_TransferFunction,
1292                                       vtctx->transfer_function);
1293
1294         if (status) {
1295             av_log(avctx, AV_LOG_WARNING, "Could not set transfer function: %d\n", status);
1296         }
1297     }
1298
1299
1300     if (vtctx->ycbcr_matrix) {
1301         status = VTSessionSetProperty(vtctx->session,
1302                                       kVTCompressionPropertyKey_YCbCrMatrix,
1303                                       vtctx->ycbcr_matrix);
1304
1305         if (status) {
1306             av_log(avctx, AV_LOG_WARNING, "Could not set ycbcr matrix: %d\n", status);
1307         }
1308     }
1309
1310
1311     if (vtctx->color_primaries) {
1312         status = VTSessionSetProperty(vtctx->session,
1313                                       kVTCompressionPropertyKey_ColorPrimaries,
1314                                       vtctx->color_primaries);
1315
1316         if (status) {
1317             av_log(avctx, AV_LOG_WARNING, "Could not set color primaries: %d\n", status);
1318         }
1319     }
1320
1321     if (gamma_level) {
1322         status = VTSessionSetProperty(vtctx->session,
1323                                       kCVImageBufferGammaLevelKey,
1324                                       gamma_level);
1325
1326         if (status) {
1327             av_log(avctx, AV_LOG_WARNING, "Could not set gamma level: %d\n", status);
1328         }
1329     }
1330
1331     if (!vtctx->has_b_frames) {
1332         status = VTSessionSetProperty(vtctx->session,
1333                                       kVTCompressionPropertyKey_AllowFrameReordering,
1334                                       kCFBooleanFalse);
1335
1336         if (status) {
1337             av_log(avctx, AV_LOG_ERROR, "Error setting 'allow frame reordering' property: %d\n", status);
1338             return AVERROR_EXTERNAL;
1339         }
1340     }
1341
1342     if (vtctx->entropy != VT_ENTROPY_NOT_SET) {
1343         CFStringRef entropy = vtctx->entropy == VT_CABAC ?
1344                                 compat_keys.kVTH264EntropyMode_CABAC:
1345                                 compat_keys.kVTH264EntropyMode_CAVLC;
1346
1347         status = VTSessionSetProperty(vtctx->session,
1348                                       compat_keys.kVTCompressionPropertyKey_H264EntropyMode,
1349                                       entropy);
1350
1351         if (status) {
1352             av_log(avctx, AV_LOG_ERROR, "Error setting entropy property: %d\n", status);
1353         }
1354     }
1355
1356     if (vtctx->realtime) {
1357         status = VTSessionSetProperty(vtctx->session,
1358                                       compat_keys.kVTCompressionPropertyKey_RealTime,
1359                                       kCFBooleanTrue);
1360
1361         if (status) {
1362             av_log(avctx, AV_LOG_ERROR, "Error setting realtime property: %d\n", status);
1363         }
1364     }
1365
1366     status = VTCompressionSessionPrepareToEncodeFrames(vtctx->session);
1367     if (status) {
1368         av_log(avctx, AV_LOG_ERROR, "Error: cannot prepare encoder: %d\n", status);
1369         return AVERROR_EXTERNAL;
1370     }
1371
1372     return 0;
1373 }
1374
1375 static int vtenc_configure_encoder(AVCodecContext *avctx)
1376 {
1377     CFMutableDictionaryRef enc_info;
1378     CFMutableDictionaryRef pixel_buffer_info;
1379     CMVideoCodecType       codec_type;
1380     VTEncContext           *vtctx = avctx->priv_data;
1381     CFStringRef            profile_level;
1382     CFNumberRef            gamma_level = NULL;
1383     int                    status;
1384
1385     codec_type = get_cm_codec_type(avctx->codec_id, avctx->pix_fmt, vtctx->alpha_quality);
1386     if (!codec_type) {
1387         av_log(avctx, AV_LOG_ERROR, "Error: no mapping for AVCodecID %d\n", avctx->codec_id);
1388         return AVERROR(EINVAL);
1389     }
1390
1391     vtctx->codec_id = avctx->codec_id;
1392     avctx->max_b_frames = 16;
1393
1394     if (vtctx->codec_id == AV_CODEC_ID_H264) {
1395         vtctx->get_param_set_func = CMVideoFormatDescriptionGetH264ParameterSetAtIndex;
1396
1397         vtctx->has_b_frames = avctx->max_b_frames > 0;
1398         if(vtctx->has_b_frames && vtctx->profile == H264_PROF_BASELINE){
1399             av_log(avctx, AV_LOG_WARNING, "Cannot use B-frames with baseline profile. Output will not contain B-frames.\n");
1400             vtctx->has_b_frames = 0;
1401         }
1402
1403         if (vtctx->entropy == VT_CABAC && vtctx->profile == H264_PROF_BASELINE) {
1404             av_log(avctx, AV_LOG_WARNING, "CABAC entropy requires 'main' or 'high' profile, but baseline was requested. Encode will not use CABAC entropy.\n");
1405             vtctx->entropy = VT_ENTROPY_NOT_SET;
1406         }
1407
1408         if (!get_vt_h264_profile_level(avctx, &profile_level)) return AVERROR(EINVAL);
1409     } else {
1410         vtctx->get_param_set_func = compat_keys.CMVideoFormatDescriptionGetHEVCParameterSetAtIndex;
1411         if (!vtctx->get_param_set_func) return AVERROR(EINVAL);
1412         if (!get_vt_hevc_profile_level(avctx, &profile_level)) return AVERROR(EINVAL);
1413         // HEVC has b-byramid
1414         vtctx->has_b_frames = avctx->max_b_frames > 0 ? 2 : 0;
1415     }
1416
1417     enc_info = CFDictionaryCreateMutable(
1418         kCFAllocatorDefault,
1419         20,
1420         &kCFCopyStringDictionaryKeyCallBacks,
1421         &kCFTypeDictionaryValueCallBacks
1422     );
1423
1424     if (!enc_info) return AVERROR(ENOMEM);
1425
1426 #if !TARGET_OS_IPHONE
1427     if(vtctx->require_sw) {
1428         CFDictionarySetValue(enc_info,
1429                              compat_keys.kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder,
1430                              kCFBooleanFalse);
1431     } else if (!vtctx->allow_sw) {
1432         CFDictionarySetValue(enc_info,
1433                              compat_keys.kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder,
1434                              kCFBooleanTrue);
1435     } else {
1436         CFDictionarySetValue(enc_info,
1437                              compat_keys.kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder,
1438                              kCFBooleanTrue);
1439     }
1440 #endif
1441
1442     if (avctx->pix_fmt != AV_PIX_FMT_VIDEOTOOLBOX) {
1443         status = create_cv_pixel_buffer_info(avctx, &pixel_buffer_info);
1444         if (status)
1445             goto init_cleanup;
1446     } else {
1447         pixel_buffer_info = NULL;
1448     }
1449
1450     vtctx->dts_delta = vtctx->has_b_frames ? -1 : 0;
1451
1452     get_cv_transfer_function(avctx, &vtctx->transfer_function, &gamma_level);
1453     get_cv_ycbcr_matrix(avctx, &vtctx->ycbcr_matrix);
1454     get_cv_color_primaries(avctx, &vtctx->color_primaries);
1455
1456
1457     if (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER) {
1458         status = vtenc_populate_extradata(avctx,
1459                                           codec_type,
1460                                           profile_level,
1461                                           gamma_level,
1462                                           enc_info,
1463                                           pixel_buffer_info);
1464         if (status)
1465             goto init_cleanup;
1466     }
1467
1468     status = vtenc_create_encoder(avctx,
1469                                   codec_type,
1470                                   profile_level,
1471                                   gamma_level,
1472                                   enc_info,
1473                                   pixel_buffer_info,
1474                                   &vtctx->session);
1475
1476 init_cleanup:
1477     if (gamma_level)
1478         CFRelease(gamma_level);
1479
1480     if (pixel_buffer_info)
1481         CFRelease(pixel_buffer_info);
1482
1483     CFRelease(enc_info);
1484
1485     return status;
1486 }
1487
1488 static av_cold int vtenc_init(AVCodecContext *avctx)
1489 {
1490     VTEncContext    *vtctx = avctx->priv_data;
1491     CFBooleanRef    has_b_frames_cfbool;
1492     int             status;
1493
1494     pthread_once(&once_ctrl, loadVTEncSymbols);
1495
1496     pthread_mutex_init(&vtctx->lock, NULL);
1497     pthread_cond_init(&vtctx->cv_sample_sent, NULL);
1498
1499     vtctx->session = NULL;
1500     status = vtenc_configure_encoder(avctx);
1501     if (status) return status;
1502
1503     status = VTSessionCopyProperty(vtctx->session,
1504                                    kVTCompressionPropertyKey_AllowFrameReordering,
1505                                    kCFAllocatorDefault,
1506                                    &has_b_frames_cfbool);
1507
1508     if (!status && has_b_frames_cfbool) {
1509         //Some devices don't output B-frames for main profile, even if requested.
1510         // HEVC has b-pyramid
1511         vtctx->has_b_frames = (CFBooleanGetValue(has_b_frames_cfbool) && avctx->codec_id == AV_CODEC_ID_HEVC) ? 2 : 1;
1512         CFRelease(has_b_frames_cfbool);
1513     }
1514     avctx->has_b_frames = vtctx->has_b_frames;
1515
1516     return 0;
1517 }
1518
1519 static void vtenc_get_frame_info(CMSampleBufferRef buffer, bool *is_key_frame)
1520 {
1521     CFArrayRef      attachments;
1522     CFDictionaryRef attachment;
1523     CFBooleanRef    not_sync;
1524     CFIndex         len;
1525
1526     attachments = CMSampleBufferGetSampleAttachmentsArray(buffer, false);
1527     len = !attachments ? 0 : CFArrayGetCount(attachments);
1528
1529     if (!len) {
1530         *is_key_frame = true;
1531         return;
1532     }
1533
1534     attachment = CFArrayGetValueAtIndex(attachments, 0);
1535
1536     if (CFDictionaryGetValueIfPresent(attachment,
1537                                       kCMSampleAttachmentKey_NotSync,
1538                                       (const void **)&not_sync))
1539     {
1540         *is_key_frame = !CFBooleanGetValue(not_sync);
1541     } else {
1542         *is_key_frame = true;
1543     }
1544 }
1545
1546 static int is_post_sei_nal_type(int nal_type){
1547     return nal_type != H264_NAL_SEI &&
1548            nal_type != H264_NAL_SPS &&
1549            nal_type != H264_NAL_PPS &&
1550            nal_type != H264_NAL_AUD;
1551 }
1552
1553 /*
1554  * Finds the sei message start/size of type find_sei_type.
1555  * If more than one of that type exists, the last one is returned.
1556  */
1557 static int find_sei_end(AVCodecContext *avctx,
1558                         uint8_t        *nal_data,
1559                         size_t          nal_size,
1560                         uint8_t       **sei_end)
1561 {
1562     int nal_type;
1563     size_t sei_payload_size = 0;
1564     int sei_payload_type = 0;
1565     *sei_end = NULL;
1566     uint8_t *nal_start = nal_data;
1567
1568     if (!nal_size)
1569         return 0;
1570
1571     nal_type = *nal_data & 0x1F;
1572     if (nal_type != H264_NAL_SEI)
1573         return 0;
1574
1575     nal_data++;
1576     nal_size--;
1577
1578     if (nal_data[nal_size - 1] == 0x80)
1579         nal_size--;
1580
1581     while (nal_size > 0 && *nal_data > 0) {
1582         do{
1583             sei_payload_type += *nal_data;
1584             nal_data++;
1585             nal_size--;
1586         } while (nal_size > 0 && *nal_data == 0xFF);
1587
1588         if (!nal_size) {
1589             av_log(avctx, AV_LOG_ERROR, "Unexpected end of SEI NAL Unit parsing type.\n");
1590             return AVERROR_INVALIDDATA;
1591         }
1592
1593         do{
1594             sei_payload_size += *nal_data;
1595             nal_data++;
1596             nal_size--;
1597         } while (nal_size > 0 && *nal_data == 0xFF);
1598
1599         if (nal_size < sei_payload_size) {
1600             av_log(avctx, AV_LOG_ERROR, "Unexpected end of SEI NAL Unit parsing size.\n");
1601             return AVERROR_INVALIDDATA;
1602         }
1603
1604         nal_data += sei_payload_size;
1605         nal_size -= sei_payload_size;
1606     }
1607
1608     *sei_end = nal_data;
1609
1610     return nal_data - nal_start + 1;
1611 }
1612
1613 /**
1614  * Copies the data inserting emulation prevention bytes as needed.
1615  * Existing data in the destination can be taken into account by providing
1616  * dst with a dst_offset > 0.
1617  *
1618  * @return The number of bytes copied on success. On failure, the negative of
1619  *         the number of bytes needed to copy src is returned.
1620  */
1621 static int copy_emulation_prev(const uint8_t *src,
1622                                size_t         src_size,
1623                                uint8_t       *dst,
1624                                ssize_t        dst_offset,
1625                                size_t         dst_size)
1626 {
1627     int zeros = 0;
1628     int wrote_bytes;
1629     uint8_t* dst_start;
1630     uint8_t* dst_end = dst + dst_size;
1631     const uint8_t* src_end = src + src_size;
1632     int start_at = dst_offset > 2 ? dst_offset - 2 : 0;
1633     int i;
1634     for (i = start_at; i < dst_offset && i < dst_size; i++) {
1635         if (!dst[i])
1636             zeros++;
1637         else
1638             zeros = 0;
1639     }
1640
1641     dst += dst_offset;
1642     dst_start = dst;
1643     for (; src < src_end; src++, dst++) {
1644         if (zeros == 2) {
1645             int insert_ep3_byte = *src <= 3;
1646             if (insert_ep3_byte) {
1647                 if (dst < dst_end)
1648                     *dst = 3;
1649                 dst++;
1650             }
1651
1652             zeros = 0;
1653         }
1654
1655         if (dst < dst_end)
1656             *dst = *src;
1657
1658         if (!*src)
1659             zeros++;
1660         else
1661             zeros = 0;
1662     }
1663
1664     wrote_bytes = dst - dst_start;
1665
1666     if (dst > dst_end)
1667         return -wrote_bytes;
1668
1669     return wrote_bytes;
1670 }
1671
1672 static int write_sei(const ExtraSEI *sei,
1673                      int             sei_type,
1674                      uint8_t        *dst,
1675                      size_t          dst_size)
1676 {
1677     uint8_t *sei_start = dst;
1678     size_t remaining_sei_size = sei->size;
1679     size_t remaining_dst_size = dst_size;
1680     int header_bytes;
1681     int bytes_written;
1682     ssize_t offset;
1683
1684     if (!remaining_dst_size)
1685         return AVERROR_BUFFER_TOO_SMALL;
1686
1687     while (sei_type && remaining_dst_size != 0) {
1688         int sei_byte = sei_type > 255 ? 255 : sei_type;
1689         *dst = sei_byte;
1690
1691         sei_type -= sei_byte;
1692         dst++;
1693         remaining_dst_size--;
1694     }
1695
1696     if (!dst_size)
1697         return AVERROR_BUFFER_TOO_SMALL;
1698
1699     while (remaining_sei_size && remaining_dst_size != 0) {
1700         int size_byte = remaining_sei_size > 255 ? 255 : remaining_sei_size;
1701         *dst = size_byte;
1702
1703         remaining_sei_size -= size_byte;
1704         dst++;
1705         remaining_dst_size--;
1706     }
1707
1708     if (remaining_dst_size < sei->size)
1709         return AVERROR_BUFFER_TOO_SMALL;
1710
1711     header_bytes = dst - sei_start;
1712
1713     offset = header_bytes;
1714     bytes_written = copy_emulation_prev(sei->data,
1715                                         sei->size,
1716                                         sei_start,
1717                                         offset,
1718                                         dst_size);
1719     if (bytes_written < 0)
1720         return AVERROR_BUFFER_TOO_SMALL;
1721
1722     bytes_written += header_bytes;
1723     return bytes_written;
1724 }
1725
1726 /**
1727  * Copies NAL units and replaces length codes with
1728  * H.264 Annex B start codes. On failure, the contents of
1729  * dst_data may have been modified.
1730  *
1731  * @param length_code_size Byte length of each length code
1732  * @param sample_buffer NAL units prefixed with length codes.
1733  * @param sei Optional A53 closed captions SEI data.
1734  * @param dst_data Must be zeroed before calling this function.
1735  *                 Contains the copied NAL units prefixed with
1736  *                 start codes when the function returns
1737  *                 successfully.
1738  * @param dst_size Length of dst_data
1739  * @return 0 on success
1740  *         AVERROR_INVALIDDATA if length_code_size is invalid
1741  *         AVERROR_BUFFER_TOO_SMALL if dst_data is too small
1742  *         or if a length_code in src_data specifies data beyond
1743  *         the end of its buffer.
1744  */
1745 static int copy_replace_length_codes(
1746     AVCodecContext *avctx,
1747     size_t        length_code_size,
1748     CMSampleBufferRef sample_buffer,
1749     ExtraSEI      *sei,
1750     uint8_t       *dst_data,
1751     size_t        dst_size)
1752 {
1753     size_t src_size = CMSampleBufferGetTotalSampleSize(sample_buffer);
1754     size_t remaining_src_size = src_size;
1755     size_t remaining_dst_size = dst_size;
1756     size_t src_offset = 0;
1757     int wrote_sei = 0;
1758     int status;
1759     uint8_t size_buf[4];
1760     uint8_t nal_type;
1761     CMBlockBufferRef block = CMSampleBufferGetDataBuffer(sample_buffer);
1762
1763     if (length_code_size > 4) {
1764         return AVERROR_INVALIDDATA;
1765     }
1766
1767     while (remaining_src_size > 0) {
1768         size_t curr_src_len;
1769         size_t curr_dst_len;
1770         size_t box_len = 0;
1771         size_t i;
1772
1773         uint8_t       *dst_box;
1774
1775         status = CMBlockBufferCopyDataBytes(block,
1776                                             src_offset,
1777                                             length_code_size,
1778                                             size_buf);
1779         if (status) {
1780             av_log(avctx, AV_LOG_ERROR, "Cannot copy length: %d\n", status);
1781             return AVERROR_EXTERNAL;
1782         }
1783
1784         status = CMBlockBufferCopyDataBytes(block,
1785                                             src_offset + length_code_size,
1786                                             1,
1787                                             &nal_type);
1788
1789         if (status) {
1790             av_log(avctx, AV_LOG_ERROR, "Cannot copy type: %d\n", status);
1791             return AVERROR_EXTERNAL;
1792         }
1793
1794         nal_type &= 0x1F;
1795
1796         for (i = 0; i < length_code_size; i++) {
1797             box_len <<= 8;
1798             box_len |= size_buf[i];
1799         }
1800
1801         if (sei && !wrote_sei && is_post_sei_nal_type(nal_type)) {
1802             //No SEI NAL unit - insert.
1803             int wrote_bytes;
1804
1805             memcpy(dst_data, start_code, sizeof(start_code));
1806             dst_data += sizeof(start_code);
1807             remaining_dst_size -= sizeof(start_code);
1808
1809             *dst_data = H264_NAL_SEI;
1810             dst_data++;
1811             remaining_dst_size--;
1812
1813             wrote_bytes = write_sei(sei,
1814                                     SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35,
1815                                     dst_data,
1816                                     remaining_dst_size);
1817
1818             if (wrote_bytes < 0)
1819                 return wrote_bytes;
1820
1821             remaining_dst_size -= wrote_bytes;
1822             dst_data += wrote_bytes;
1823
1824             if (remaining_dst_size <= 0)
1825                 return AVERROR_BUFFER_TOO_SMALL;
1826
1827             *dst_data = 0x80;
1828
1829             dst_data++;
1830             remaining_dst_size--;
1831
1832             wrote_sei = 1;
1833         }
1834
1835         curr_src_len = box_len + length_code_size;
1836         curr_dst_len = box_len + sizeof(start_code);
1837
1838         if (remaining_src_size < curr_src_len) {
1839             return AVERROR_BUFFER_TOO_SMALL;
1840         }
1841
1842         if (remaining_dst_size < curr_dst_len) {
1843             return AVERROR_BUFFER_TOO_SMALL;
1844         }
1845
1846         dst_box = dst_data + sizeof(start_code);
1847
1848         memcpy(dst_data, start_code, sizeof(start_code));
1849         status = CMBlockBufferCopyDataBytes(block,
1850                                             src_offset + length_code_size,
1851                                             box_len,
1852                                             dst_box);
1853
1854         if (status) {
1855             av_log(avctx, AV_LOG_ERROR, "Cannot copy data: %d\n", status);
1856             return AVERROR_EXTERNAL;
1857         }
1858
1859         if (sei && !wrote_sei && nal_type == H264_NAL_SEI) {
1860             //Found SEI NAL unit - append.
1861             int wrote_bytes;
1862             int old_sei_length;
1863             int extra_bytes;
1864             uint8_t *new_sei;
1865             old_sei_length = find_sei_end(avctx, dst_box, box_len, &new_sei);
1866             if (old_sei_length < 0)
1867                 return status;
1868
1869             wrote_bytes = write_sei(sei,
1870                                     SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35,
1871                                     new_sei,
1872                                     remaining_dst_size - old_sei_length);
1873             if (wrote_bytes < 0)
1874                 return wrote_bytes;
1875
1876             if (new_sei + wrote_bytes >= dst_data + remaining_dst_size)
1877                 return AVERROR_BUFFER_TOO_SMALL;
1878
1879             new_sei[wrote_bytes++] = 0x80;
1880             extra_bytes = wrote_bytes - (dst_box + box_len - new_sei);
1881
1882             dst_data += extra_bytes;
1883             remaining_dst_size -= extra_bytes;
1884
1885             wrote_sei = 1;
1886         }
1887
1888         src_offset += curr_src_len;
1889         dst_data += curr_dst_len;
1890
1891         remaining_src_size -= curr_src_len;
1892         remaining_dst_size -= curr_dst_len;
1893     }
1894
1895     return 0;
1896 }
1897
1898 /**
1899  * Returns a sufficient number of bytes to contain the sei data.
1900  * It may be greater than the minimum required.
1901  */
1902 static int get_sei_msg_bytes(const ExtraSEI* sei, int type){
1903     int copied_size;
1904     if (sei->size == 0)
1905         return 0;
1906
1907     copied_size = -copy_emulation_prev(sei->data,
1908                                        sei->size,
1909                                        NULL,
1910                                        0,
1911                                        0);
1912
1913     if ((sei->size % 255) == 0) //may result in an extra byte
1914         copied_size++;
1915
1916     return copied_size + sei->size / 255 + 1 + type / 255 + 1;
1917 }
1918
1919 static int vtenc_cm_to_avpacket(
1920     AVCodecContext    *avctx,
1921     CMSampleBufferRef sample_buffer,
1922     AVPacket          *pkt,
1923     ExtraSEI          *sei)
1924 {
1925     VTEncContext *vtctx = avctx->priv_data;
1926
1927     int     status;
1928     bool    is_key_frame;
1929     bool    add_header;
1930     size_t  length_code_size;
1931     size_t  header_size = 0;
1932     size_t  in_buf_size;
1933     size_t  out_buf_size;
1934     size_t  sei_nalu_size = 0;
1935     int64_t dts_delta;
1936     int64_t time_base_num;
1937     int nalu_count;
1938     CMTime  pts;
1939     CMTime  dts;
1940     CMVideoFormatDescriptionRef vid_fmt;
1941
1942
1943     vtenc_get_frame_info(sample_buffer, &is_key_frame);
1944     status = get_length_code_size(avctx, sample_buffer, &length_code_size);
1945     if (status) return status;
1946
1947     add_header = is_key_frame && !(avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER);
1948
1949     if (add_header) {
1950         vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer);
1951         if (!vid_fmt) {
1952             av_log(avctx, AV_LOG_ERROR, "Cannot get format description.\n");
1953             return AVERROR_EXTERNAL;
1954         }
1955
1956         int status = get_params_size(avctx, vid_fmt, &header_size);
1957         if (status) return status;
1958     }
1959
1960     status = count_nalus(length_code_size, sample_buffer, &nalu_count);
1961     if(status)
1962         return status;
1963
1964     if (sei) {
1965         size_t msg_size = get_sei_msg_bytes(sei,
1966                                             SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35);
1967
1968         sei_nalu_size = sizeof(start_code) + 1 + msg_size + 1;
1969     }
1970
1971     in_buf_size = CMSampleBufferGetTotalSampleSize(sample_buffer);
1972     out_buf_size = header_size +
1973                    in_buf_size +
1974                    sei_nalu_size +
1975                    nalu_count * ((int)sizeof(start_code) - (int)length_code_size);
1976
1977     status = ff_alloc_packet2(avctx, pkt, out_buf_size, out_buf_size);
1978     if (status < 0)
1979         return status;
1980
1981     if (add_header) {
1982         status = copy_param_sets(avctx, vid_fmt, pkt->data, out_buf_size);
1983         if(status) return status;
1984     }
1985
1986     status = copy_replace_length_codes(
1987         avctx,
1988         length_code_size,
1989         sample_buffer,
1990         sei,
1991         pkt->data + header_size,
1992         pkt->size - header_size
1993     );
1994
1995     if (status) {
1996         av_log(avctx, AV_LOG_ERROR, "Error copying packet data: %d\n", status);
1997         return status;
1998     }
1999
2000     if (is_key_frame) {
2001         pkt->flags |= AV_PKT_FLAG_KEY;
2002     }
2003
2004     pts = CMSampleBufferGetPresentationTimeStamp(sample_buffer);
2005     dts = CMSampleBufferGetDecodeTimeStamp      (sample_buffer);
2006
2007     if (CMTIME_IS_INVALID(dts)) {
2008         if (!vtctx->has_b_frames) {
2009             dts = pts;
2010         } else {
2011             av_log(avctx, AV_LOG_ERROR, "DTS is invalid.\n");
2012             return AVERROR_EXTERNAL;
2013         }
2014     }
2015
2016     dts_delta = vtctx->dts_delta >= 0 ? vtctx->dts_delta : 0;
2017     time_base_num = avctx->time_base.num;
2018     pkt->pts = pts.value / time_base_num;
2019     pkt->dts = dts.value / time_base_num - dts_delta;
2020     pkt->size = out_buf_size;
2021
2022     return 0;
2023 }
2024
2025 /*
2026  * contiguous_buf_size is 0 if not contiguous, and the size of the buffer
2027  * containing all planes if so.
2028  */
2029 static int get_cv_pixel_info(
2030     AVCodecContext *avctx,
2031     const AVFrame  *frame,
2032     int            *color,
2033     int            *plane_count,
2034     size_t         *widths,
2035     size_t         *heights,
2036     size_t         *strides,
2037     size_t         *contiguous_buf_size)
2038 {
2039     VTEncContext *vtctx = avctx->priv_data;
2040     int av_format       = frame->format;
2041     int av_color_range  = frame->color_range;
2042     int i;
2043     int range_guessed;
2044     int status;
2045
2046     status = get_cv_pixel_format(avctx, av_format, av_color_range, color, &range_guessed);
2047     if (status) {
2048         av_log(avctx,
2049             AV_LOG_ERROR,
2050             "Could not get pixel format for color format '%s' range '%s'.\n",
2051             av_get_pix_fmt_name(av_format),
2052             av_color_range > AVCOL_RANGE_UNSPECIFIED &&
2053             av_color_range < AVCOL_RANGE_NB ?
2054                av_color_range_name(av_color_range) :
2055                "Unknown");
2056
2057         return AVERROR(EINVAL);
2058     }
2059
2060     if (range_guessed) {
2061         if (!vtctx->warned_color_range) {
2062             vtctx->warned_color_range = true;
2063             av_log(avctx,
2064                    AV_LOG_WARNING,
2065                    "Color range not set for %s. Using MPEG range.\n",
2066                    av_get_pix_fmt_name(av_format));
2067         }
2068     }
2069
2070     switch (av_format) {
2071     case AV_PIX_FMT_NV12:
2072         *plane_count = 2;
2073
2074         widths [0] = avctx->width;
2075         heights[0] = avctx->height;
2076         strides[0] = frame ? frame->linesize[0] : avctx->width;
2077
2078         widths [1] = (avctx->width  + 1) / 2;
2079         heights[1] = (avctx->height + 1) / 2;
2080         strides[1] = frame ? frame->linesize[1] : (avctx->width + 1) & -2;
2081         break;
2082
2083     case AV_PIX_FMT_YUV420P:
2084         *plane_count = 3;
2085
2086         widths [0] = avctx->width;
2087         heights[0] = avctx->height;
2088         strides[0] = frame ? frame->linesize[0] : avctx->width;
2089
2090         widths [1] = (avctx->width  + 1) / 2;
2091         heights[1] = (avctx->height + 1) / 2;
2092         strides[1] = frame ? frame->linesize[1] : (avctx->width + 1) / 2;
2093
2094         widths [2] = (avctx->width  + 1) / 2;
2095         heights[2] = (avctx->height + 1) / 2;
2096         strides[2] = frame ? frame->linesize[2] : (avctx->width + 1) / 2;
2097         break;
2098
2099     case AV_PIX_FMT_BGRA:
2100         *plane_count = 1;
2101
2102         widths [0] = avctx->width;
2103         heights[0] = avctx->height;
2104         strides[0] = frame ? frame->linesize[0] : avctx->width * 4;
2105         break;
2106
2107     case AV_PIX_FMT_P010LE:
2108         *plane_count = 2;
2109         widths[0] = avctx->width;
2110         heights[0] = avctx->height;
2111         strides[0] = frame ? frame->linesize[0] : (avctx->width * 2 + 63) & -64;
2112
2113         widths[1] = (avctx->width + 1) / 2;
2114         heights[1] = (avctx->height + 1) / 2;
2115         strides[1] = frame ? frame->linesize[1] : ((avctx->width + 1) / 2 + 63) & -64;
2116         break;
2117
2118     default:
2119         av_log(
2120                avctx,
2121                AV_LOG_ERROR,
2122                "Could not get frame format info for color %d range %d.\n",
2123                av_format,
2124                av_color_range);
2125
2126         return AVERROR(EINVAL);
2127     }
2128
2129     *contiguous_buf_size = 0;
2130     for (i = 0; i < *plane_count; i++) {
2131         if (i < *plane_count - 1 &&
2132             frame->data[i] + strides[i] * heights[i] != frame->data[i + 1]) {
2133             *contiguous_buf_size = 0;
2134             break;
2135         }
2136
2137         *contiguous_buf_size += strides[i] * heights[i];
2138     }
2139
2140     return 0;
2141 }
2142
2143 //Not used on OSX - frame is never copied.
2144 static int copy_avframe_to_pixel_buffer(AVCodecContext   *avctx,
2145                                         const AVFrame    *frame,
2146                                         CVPixelBufferRef cv_img,
2147                                         const size_t     *plane_strides,
2148                                         const size_t     *plane_rows)
2149 {
2150     int i, j;
2151     size_t plane_count;
2152     int status;
2153     int rows;
2154     int src_stride;
2155     int dst_stride;
2156     uint8_t *src_addr;
2157     uint8_t *dst_addr;
2158     size_t copy_bytes;
2159
2160     status = CVPixelBufferLockBaseAddress(cv_img, 0);
2161     if (status) {
2162         av_log(
2163             avctx,
2164             AV_LOG_ERROR,
2165             "Error: Could not lock base address of CVPixelBuffer: %d.\n",
2166             status
2167         );
2168     }
2169
2170     if (CVPixelBufferIsPlanar(cv_img)) {
2171         plane_count = CVPixelBufferGetPlaneCount(cv_img);
2172         for (i = 0; frame->data[i]; i++) {
2173             if (i == plane_count) {
2174                 CVPixelBufferUnlockBaseAddress(cv_img, 0);
2175                 av_log(avctx,
2176                     AV_LOG_ERROR,
2177                     "Error: different number of planes in AVFrame and CVPixelBuffer.\n"
2178                 );
2179
2180                 return AVERROR_EXTERNAL;
2181             }
2182
2183             dst_addr = (uint8_t*)CVPixelBufferGetBaseAddressOfPlane(cv_img, i);
2184             src_addr = (uint8_t*)frame->data[i];
2185             dst_stride = CVPixelBufferGetBytesPerRowOfPlane(cv_img, i);
2186             src_stride = plane_strides[i];
2187             rows = plane_rows[i];
2188
2189             if (dst_stride == src_stride) {
2190                 memcpy(dst_addr, src_addr, src_stride * rows);
2191             } else {
2192                 copy_bytes = dst_stride < src_stride ? dst_stride : src_stride;
2193
2194                 for (j = 0; j < rows; j++) {
2195                     memcpy(dst_addr + j * dst_stride, src_addr + j * src_stride, copy_bytes);
2196                 }
2197             }
2198         }
2199     } else {
2200         if (frame->data[1]) {
2201             CVPixelBufferUnlockBaseAddress(cv_img, 0);
2202             av_log(avctx,
2203                 AV_LOG_ERROR,
2204                 "Error: different number of planes in AVFrame and non-planar CVPixelBuffer.\n"
2205             );
2206
2207             return AVERROR_EXTERNAL;
2208         }
2209
2210         dst_addr = (uint8_t*)CVPixelBufferGetBaseAddress(cv_img);
2211         src_addr = (uint8_t*)frame->data[0];
2212         dst_stride = CVPixelBufferGetBytesPerRow(cv_img);
2213         src_stride = plane_strides[0];
2214         rows = plane_rows[0];
2215
2216         if (dst_stride == src_stride) {
2217             memcpy(dst_addr, src_addr, src_stride * rows);
2218         } else {
2219             copy_bytes = dst_stride < src_stride ? dst_stride : src_stride;
2220
2221             for (j = 0; j < rows; j++) {
2222                 memcpy(dst_addr + j * dst_stride, src_addr + j * src_stride, copy_bytes);
2223             }
2224         }
2225     }
2226
2227     status = CVPixelBufferUnlockBaseAddress(cv_img, 0);
2228     if (status) {
2229         av_log(avctx, AV_LOG_ERROR, "Error: Could not unlock CVPixelBuffer base address: %d.\n", status);
2230         return AVERROR_EXTERNAL;
2231     }
2232
2233     return 0;
2234 }
2235
2236 static int create_cv_pixel_buffer(AVCodecContext   *avctx,
2237                                   const AVFrame    *frame,
2238                                   CVPixelBufferRef *cv_img)
2239 {
2240     int plane_count;
2241     int color;
2242     size_t widths [AV_NUM_DATA_POINTERS];
2243     size_t heights[AV_NUM_DATA_POINTERS];
2244     size_t strides[AV_NUM_DATA_POINTERS];
2245     int status;
2246     size_t contiguous_buf_size;
2247     CVPixelBufferPoolRef pix_buf_pool;
2248     VTEncContext* vtctx = avctx->priv_data;
2249
2250     if (avctx->pix_fmt == AV_PIX_FMT_VIDEOTOOLBOX) {
2251         av_assert0(frame->format == AV_PIX_FMT_VIDEOTOOLBOX);
2252
2253         *cv_img = (CVPixelBufferRef)frame->data[3];
2254         av_assert0(*cv_img);
2255
2256         CFRetain(*cv_img);
2257         return 0;
2258     }
2259
2260     memset(widths,  0, sizeof(widths));
2261     memset(heights, 0, sizeof(heights));
2262     memset(strides, 0, sizeof(strides));
2263
2264     status = get_cv_pixel_info(
2265         avctx,
2266         frame,
2267         &color,
2268         &plane_count,
2269         widths,
2270         heights,
2271         strides,
2272         &contiguous_buf_size
2273     );
2274
2275     if (status) {
2276         av_log(
2277             avctx,
2278             AV_LOG_ERROR,
2279             "Error: Cannot convert format %d color_range %d: %d\n",
2280             frame->format,
2281             frame->color_range,
2282             status
2283         );
2284
2285         return AVERROR_EXTERNAL;
2286     }
2287
2288     pix_buf_pool = VTCompressionSessionGetPixelBufferPool(vtctx->session);
2289     if (!pix_buf_pool) {
2290         /* On iOS, the VT session is invalidated when the APP switches from
2291          * foreground to background and vice versa. Fetch the actual error code
2292          * of the VT session to detect that case and restart the VT session
2293          * accordingly. */
2294         OSStatus vtstatus;
2295
2296         vtstatus = VTCompressionSessionPrepareToEncodeFrames(vtctx->session);
2297         if (vtstatus == kVTInvalidSessionErr) {
2298             CFRelease(vtctx->session);
2299             vtctx->session = NULL;
2300             status = vtenc_configure_encoder(avctx);
2301             if (status == 0)
2302                 pix_buf_pool = VTCompressionSessionGetPixelBufferPool(vtctx->session);
2303         }
2304         if (!pix_buf_pool) {
2305             av_log(avctx, AV_LOG_ERROR, "Could not get pixel buffer pool.\n");
2306             return AVERROR_EXTERNAL;
2307         }
2308         else
2309             av_log(avctx, AV_LOG_WARNING, "VT session restarted because of a "
2310                    "kVTInvalidSessionErr error.\n");
2311     }
2312
2313     status = CVPixelBufferPoolCreatePixelBuffer(NULL,
2314                                                 pix_buf_pool,
2315                                                 cv_img);
2316
2317
2318     if (status) {
2319         av_log(avctx, AV_LOG_ERROR, "Could not create pixel buffer from pool: %d.\n", status);
2320         return AVERROR_EXTERNAL;
2321     }
2322
2323     status = copy_avframe_to_pixel_buffer(avctx, frame, *cv_img, strides, heights);
2324     if (status) {
2325         CFRelease(*cv_img);
2326         *cv_img = NULL;
2327         return status;
2328     }
2329
2330     return 0;
2331 }
2332
2333 static int create_encoder_dict_h264(const AVFrame *frame,
2334                                     CFDictionaryRef* dict_out)
2335 {
2336     CFDictionaryRef dict = NULL;
2337     if (frame->pict_type == AV_PICTURE_TYPE_I) {
2338         const void *keys[] = { kVTEncodeFrameOptionKey_ForceKeyFrame };
2339         const void *vals[] = { kCFBooleanTrue };
2340
2341         dict = CFDictionaryCreate(NULL, keys, vals, 1, NULL, NULL);
2342         if(!dict) return AVERROR(ENOMEM);
2343     }
2344
2345     *dict_out = dict;
2346     return 0;
2347 }
2348
2349 static int vtenc_send_frame(AVCodecContext *avctx,
2350                             VTEncContext   *vtctx,
2351                             const AVFrame  *frame)
2352 {
2353     CMTime time;
2354     CFDictionaryRef frame_dict;
2355     CVPixelBufferRef cv_img = NULL;
2356     AVFrameSideData *side_data = NULL;
2357     ExtraSEI *sei = NULL;
2358     int status = create_cv_pixel_buffer(avctx, frame, &cv_img);
2359
2360     if (status) return status;
2361
2362     status = create_encoder_dict_h264(frame, &frame_dict);
2363     if (status) {
2364         CFRelease(cv_img);
2365         return status;
2366     }
2367
2368     side_data = av_frame_get_side_data(frame, AV_FRAME_DATA_A53_CC);
2369     if (vtctx->a53_cc && side_data && side_data->size) {
2370         sei = av_mallocz(sizeof(*sei));
2371         if (!sei) {
2372             av_log(avctx, AV_LOG_ERROR, "Not enough memory for closed captions, skipping\n");
2373         } else {
2374             int ret = ff_alloc_a53_sei(frame, 0, &sei->data, &sei->size);
2375             if (ret < 0) {
2376                 av_log(avctx, AV_LOG_ERROR, "Not enough memory for closed captions, skipping\n");
2377                 av_free(sei);
2378                 sei = NULL;
2379             }
2380         }
2381     }
2382
2383     time = CMTimeMake(frame->pts * avctx->time_base.num, avctx->time_base.den);
2384     status = VTCompressionSessionEncodeFrame(
2385         vtctx->session,
2386         cv_img,
2387         time,
2388         kCMTimeInvalid,
2389         frame_dict,
2390         sei,
2391         NULL
2392     );
2393
2394     if (frame_dict) CFRelease(frame_dict);
2395     CFRelease(cv_img);
2396
2397     if (status) {
2398         av_log(avctx, AV_LOG_ERROR, "Error: cannot encode frame: %d\n", status);
2399         return AVERROR_EXTERNAL;
2400     }
2401
2402     return 0;
2403 }
2404
2405 static av_cold int vtenc_frame(
2406     AVCodecContext *avctx,
2407     AVPacket       *pkt,
2408     const AVFrame  *frame,
2409     int            *got_packet)
2410 {
2411     VTEncContext *vtctx = avctx->priv_data;
2412     bool get_frame;
2413     int status;
2414     CMSampleBufferRef buf = NULL;
2415     ExtraSEI *sei = NULL;
2416
2417     if (frame) {
2418         status = vtenc_send_frame(avctx, vtctx, frame);
2419
2420         if (status) {
2421             status = AVERROR_EXTERNAL;
2422             goto end_nopkt;
2423         }
2424
2425         if (vtctx->frame_ct_in == 0) {
2426             vtctx->first_pts = frame->pts;
2427         } else if(vtctx->frame_ct_in == vtctx->has_b_frames) {
2428             vtctx->dts_delta = frame->pts - vtctx->first_pts;
2429         }
2430
2431         vtctx->frame_ct_in++;
2432     } else if(!vtctx->flushing) {
2433         vtctx->flushing = true;
2434
2435         status = VTCompressionSessionCompleteFrames(vtctx->session,
2436                                                     kCMTimeIndefinite);
2437
2438         if (status) {
2439             av_log(avctx, AV_LOG_ERROR, "Error flushing frames: %d\n", status);
2440             status = AVERROR_EXTERNAL;
2441             goto end_nopkt;
2442         }
2443     }
2444
2445     *got_packet = 0;
2446     get_frame = vtctx->dts_delta >= 0 || !frame;
2447     if (!get_frame) {
2448         status = 0;
2449         goto end_nopkt;
2450     }
2451
2452     status = vtenc_q_pop(vtctx, !frame, &buf, &sei);
2453     if (status) goto end_nopkt;
2454     if (!buf)   goto end_nopkt;
2455
2456     status = vtenc_cm_to_avpacket(avctx, buf, pkt, sei);
2457     if (sei) {
2458         if (sei->data) av_free(sei->data);
2459         av_free(sei);
2460     }
2461     CFRelease(buf);
2462     if (status) goto end_nopkt;
2463
2464     *got_packet = 1;
2465     return 0;
2466
2467 end_nopkt:
2468     av_packet_unref(pkt);
2469     return status;
2470 }
2471
2472 static int vtenc_populate_extradata(AVCodecContext   *avctx,
2473                                     CMVideoCodecType codec_type,
2474                                     CFStringRef      profile_level,
2475                                     CFNumberRef      gamma_level,
2476                                     CFDictionaryRef  enc_info,
2477                                     CFDictionaryRef  pixel_buffer_info)
2478 {
2479     VTEncContext *vtctx = avctx->priv_data;
2480     int status;
2481     CVPixelBufferPoolRef pool = NULL;
2482     CVPixelBufferRef pix_buf = NULL;
2483     CMTime time;
2484     CMSampleBufferRef buf = NULL;
2485
2486     status = vtenc_create_encoder(avctx,
2487                                   codec_type,
2488                                   profile_level,
2489                                   gamma_level,
2490                                   enc_info,
2491                                   pixel_buffer_info,
2492                                   &vtctx->session);
2493     if (status)
2494         goto pe_cleanup;
2495
2496     pool = VTCompressionSessionGetPixelBufferPool(vtctx->session);
2497     if(!pool){
2498         av_log(avctx, AV_LOG_ERROR, "Error getting pixel buffer pool.\n");
2499         goto pe_cleanup;
2500     }
2501
2502     status = CVPixelBufferPoolCreatePixelBuffer(NULL,
2503                                                 pool,
2504                                                 &pix_buf);
2505
2506     if(status != kCVReturnSuccess){
2507         av_log(avctx, AV_LOG_ERROR, "Error creating frame from pool: %d\n", status);
2508         goto pe_cleanup;
2509     }
2510
2511     time = CMTimeMake(0, avctx->time_base.den);
2512     status = VTCompressionSessionEncodeFrame(vtctx->session,
2513                                              pix_buf,
2514                                              time,
2515                                              kCMTimeInvalid,
2516                                              NULL,
2517                                              NULL,
2518                                              NULL);
2519
2520     if (status) {
2521         av_log(avctx,
2522                AV_LOG_ERROR,
2523                "Error sending frame for extradata: %d\n",
2524                status);
2525
2526         goto pe_cleanup;
2527     }
2528
2529     //Populates extradata - output frames are flushed and param sets are available.
2530     status = VTCompressionSessionCompleteFrames(vtctx->session,
2531                                                 kCMTimeIndefinite);
2532
2533     if (status)
2534         goto pe_cleanup;
2535
2536     status = vtenc_q_pop(vtctx, 0, &buf, NULL);
2537     if (status) {
2538         av_log(avctx, AV_LOG_ERROR, "popping: %d\n", status);
2539         goto pe_cleanup;
2540     }
2541
2542     CFRelease(buf);
2543
2544
2545
2546 pe_cleanup:
2547     if(vtctx->session)
2548         CFRelease(vtctx->session);
2549
2550     vtctx->session = NULL;
2551     vtctx->frame_ct_out = 0;
2552
2553     av_assert0(status != 0 || (avctx->extradata && avctx->extradata_size > 0));
2554
2555     return status;
2556 }
2557
2558 static av_cold int vtenc_close(AVCodecContext *avctx)
2559 {
2560     VTEncContext *vtctx = avctx->priv_data;
2561
2562     if(!vtctx->session) {
2563         pthread_cond_destroy(&vtctx->cv_sample_sent);
2564         pthread_mutex_destroy(&vtctx->lock);
2565         return 0;
2566     }
2567
2568     VTCompressionSessionCompleteFrames(vtctx->session,
2569                                        kCMTimeIndefinite);
2570     clear_frame_queue(vtctx);
2571     pthread_cond_destroy(&vtctx->cv_sample_sent);
2572     pthread_mutex_destroy(&vtctx->lock);
2573     CFRelease(vtctx->session);
2574     vtctx->session = NULL;
2575
2576     if (vtctx->color_primaries) {
2577         CFRelease(vtctx->color_primaries);
2578         vtctx->color_primaries = NULL;
2579     }
2580
2581     if (vtctx->transfer_function) {
2582         CFRelease(vtctx->transfer_function);
2583         vtctx->transfer_function = NULL;
2584     }
2585
2586     if (vtctx->ycbcr_matrix) {
2587         CFRelease(vtctx->ycbcr_matrix);
2588         vtctx->ycbcr_matrix = NULL;
2589     }
2590
2591     return 0;
2592 }
2593
2594 static const enum AVPixelFormat avc_pix_fmts[] = {
2595     AV_PIX_FMT_VIDEOTOOLBOX,
2596     AV_PIX_FMT_NV12,
2597     AV_PIX_FMT_YUV420P,
2598     AV_PIX_FMT_NONE
2599 };
2600
2601 static const enum AVPixelFormat hevc_pix_fmts[] = {
2602     AV_PIX_FMT_VIDEOTOOLBOX,
2603     AV_PIX_FMT_NV12,
2604     AV_PIX_FMT_YUV420P,
2605     AV_PIX_FMT_BGRA,
2606     AV_PIX_FMT_P010LE,
2607     AV_PIX_FMT_NONE
2608 };
2609
2610 #define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM
2611 #define COMMON_OPTIONS \
2612     { "allow_sw", "Allow software encoding", OFFSET(allow_sw), AV_OPT_TYPE_BOOL, \
2613         { .i64 = 0 }, 0, 1, VE }, \
2614     { "require_sw", "Require software encoding", OFFSET(require_sw), AV_OPT_TYPE_BOOL, \
2615         { .i64 = 0 }, 0, 1, VE }, \
2616     { "realtime", "Hint that encoding should happen in real-time if not faster (e.g. capturing from camera).", \
2617         OFFSET(realtime), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE }, \
2618     { "frames_before", "Other frames will come before the frames in this session. This helps smooth concatenation issues.", \
2619         OFFSET(frames_before), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE }, \
2620     { "frames_after", "Other frames will come after the frames in this session. This helps smooth concatenation issues.", \
2621         OFFSET(frames_after), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE },
2622
2623 #define OFFSET(x) offsetof(VTEncContext, x)
2624 static const AVOption h264_options[] = {
2625     { "profile", "Profile", OFFSET(profile), AV_OPT_TYPE_INT, { .i64 = H264_PROF_AUTO }, H264_PROF_AUTO, H264_PROF_COUNT, VE, "profile" },
2626     { "baseline", "Baseline Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_BASELINE }, INT_MIN, INT_MAX, VE, "profile" },
2627     { "main",     "Main Profile",     0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_MAIN     }, INT_MIN, INT_MAX, VE, "profile" },
2628     { "high",     "High Profile",     0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_HIGH     }, INT_MIN, INT_MAX, VE, "profile" },
2629     { "extended", "Extend Profile",   0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_EXTENDED }, INT_MIN, INT_MAX, VE, "profile" },
2630
2631     { "level", "Level", OFFSET(level), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 52, VE, "level" },
2632     { "1.3", "Level 1.3, only available with Baseline Profile", 0, AV_OPT_TYPE_CONST, { .i64 = 13 }, INT_MIN, INT_MAX, VE, "level" },
2633     { "3.0", "Level 3.0", 0, AV_OPT_TYPE_CONST, { .i64 = 30 }, INT_MIN, INT_MAX, VE, "level" },
2634     { "3.1", "Level 3.1", 0, AV_OPT_TYPE_CONST, { .i64 = 31 }, INT_MIN, INT_MAX, VE, "level" },
2635     { "3.2", "Level 3.2", 0, AV_OPT_TYPE_CONST, { .i64 = 32 }, INT_MIN, INT_MAX, VE, "level" },
2636     { "4.0", "Level 4.0", 0, AV_OPT_TYPE_CONST, { .i64 = 40 }, INT_MIN, INT_MAX, VE, "level" },
2637     { "4.1", "Level 4.1", 0, AV_OPT_TYPE_CONST, { .i64 = 41 }, INT_MIN, INT_MAX, VE, "level" },
2638     { "4.2", "Level 4.2", 0, AV_OPT_TYPE_CONST, { .i64 = 42 }, INT_MIN, INT_MAX, VE, "level" },
2639     { "5.0", "Level 5.0", 0, AV_OPT_TYPE_CONST, { .i64 = 50 }, INT_MIN, INT_MAX, VE, "level" },
2640     { "5.1", "Level 5.1", 0, AV_OPT_TYPE_CONST, { .i64 = 51 }, INT_MIN, INT_MAX, VE, "level" },
2641     { "5.2", "Level 5.2", 0, AV_OPT_TYPE_CONST, { .i64 = 52 }, INT_MIN, INT_MAX, VE, "level" },
2642
2643     { "coder", "Entropy coding", OFFSET(entropy), AV_OPT_TYPE_INT, { .i64 = VT_ENTROPY_NOT_SET }, VT_ENTROPY_NOT_SET, VT_CABAC, VE, "coder" },
2644     { "cavlc", "CAVLC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CAVLC }, INT_MIN, INT_MAX, VE, "coder" },
2645     { "vlc",   "CAVLC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CAVLC }, INT_MIN, INT_MAX, VE, "coder" },
2646     { "cabac", "CABAC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CABAC }, INT_MIN, INT_MAX, VE, "coder" },
2647     { "ac",    "CABAC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CABAC }, INT_MIN, INT_MAX, VE, "coder" },
2648
2649     { "a53cc", "Use A53 Closed Captions (if available)", OFFSET(a53_cc), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, VE },
2650
2651     COMMON_OPTIONS
2652     { NULL },
2653 };
2654
2655 static const AVClass h264_videotoolbox_class = {
2656     .class_name = "h264_videotoolbox",
2657     .item_name  = av_default_item_name,
2658     .option     = h264_options,
2659     .version    = LIBAVUTIL_VERSION_INT,
2660 };
2661
2662 AVCodec ff_h264_videotoolbox_encoder = {
2663     .name             = "h264_videotoolbox",
2664     .long_name        = NULL_IF_CONFIG_SMALL("VideoToolbox H.264 Encoder"),
2665     .type             = AVMEDIA_TYPE_VIDEO,
2666     .id               = AV_CODEC_ID_H264,
2667     .priv_data_size   = sizeof(VTEncContext),
2668     .pix_fmts         = avc_pix_fmts,
2669     .init             = vtenc_init,
2670     .encode2          = vtenc_frame,
2671     .close            = vtenc_close,
2672     .capabilities     = AV_CODEC_CAP_DELAY,
2673     .priv_class       = &h264_videotoolbox_class,
2674     .caps_internal    = FF_CODEC_CAP_INIT_THREADSAFE |
2675                         FF_CODEC_CAP_INIT_CLEANUP,
2676 };
2677
2678 static const AVOption hevc_options[] = {
2679     { "profile", "Profile", OFFSET(profile), AV_OPT_TYPE_INT, { .i64 = HEVC_PROF_AUTO }, HEVC_PROF_AUTO, HEVC_PROF_COUNT, VE, "profile" },
2680     { "main",     "Main Profile",     0, AV_OPT_TYPE_CONST, { .i64 = HEVC_PROF_MAIN   }, INT_MIN, INT_MAX, VE, "profile" },
2681     { "main10",   "Main10 Profile",   0, AV_OPT_TYPE_CONST, { .i64 = HEVC_PROF_MAIN10 }, INT_MIN, INT_MAX, VE, "profile" },
2682
2683     { "alpha_quality", "Compression quality for the alpha channel", OFFSET(alpha_quality), AV_OPT_TYPE_DOUBLE, { .dbl = 0.0 }, 0.0, 1.0, VE },
2684
2685     COMMON_OPTIONS
2686     { NULL },
2687 };
2688
2689 static const AVClass hevc_videotoolbox_class = {
2690     .class_name = "hevc_videotoolbox",
2691     .item_name  = av_default_item_name,
2692     .option     = hevc_options,
2693     .version    = LIBAVUTIL_VERSION_INT,
2694 };
2695
2696 AVCodec ff_hevc_videotoolbox_encoder = {
2697     .name             = "hevc_videotoolbox",
2698     .long_name        = NULL_IF_CONFIG_SMALL("VideoToolbox H.265 Encoder"),
2699     .type             = AVMEDIA_TYPE_VIDEO,
2700     .id               = AV_CODEC_ID_HEVC,
2701     .priv_data_size   = sizeof(VTEncContext),
2702     .pix_fmts         = hevc_pix_fmts,
2703     .init             = vtenc_init,
2704     .encode2          = vtenc_frame,
2705     .close            = vtenc_close,
2706     .capabilities     = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE,
2707     .priv_class       = &hevc_videotoolbox_class,
2708     .caps_internal    = FF_CODEC_CAP_INIT_THREADSAFE |
2709                         FF_CODEC_CAP_INIT_CLEANUP,
2710     .wrapper_name     = "videotoolbox",
2711 };