git.sesse.net Git - ffmpeg/blob - libavcodec/videotoolbox.c

   1 /*
   2  * Videotoolbox hardware acceleration
   3  *
   4  * copyright (c) 2012 Sebastien Zwickert
   5  *
   6  * This file is part of FFmpeg.
   7  *
   8  * FFmpeg is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License as published by the Free Software Foundation; either
  11  * version 2.1 of the License, or (at your option) any later version.
  12  *
  13  * FFmpeg is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16  * Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public
  19  * License along with FFmpeg; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  21  */
  22
  23 #include "config.h"
  24 #if CONFIG_VIDEOTOOLBOX
  25 #  include "videotoolbox.h"
  26 #  include "libavutil/hwcontext_videotoolbox.h"
  27 #else
  28 #  include "vda.h"
  29 #endif
  30 #include "vda_vt_internal.h"
  31 #include "libavutil/avutil.h"
  32 #include "libavutil/hwcontext.h"
  33 #include "bytestream.h"
  34 #include "h264dec.h"
  35 #include "hevcdec.h"
  36 #include "mpegvideo.h"
  37 #include <TargetConditionals.h>
  38
  39 #ifndef kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder
  40 #  define kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder CFSTR("RequireHardwareAcceleratedVideoDecoder")
  41 #endif
  42
  43 #if !HAVE_KCMVIDEOCODECTYPE_HEVC
  44 enum { kCMVideoCodecType_HEVC = 'hvc1' };
  45 #endif
  46
  47 #define VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING  12
  48
  49 static void videotoolbox_buffer_release(void *opaque, uint8_t *data)
  50 {
  51     CVPixelBufferRef cv_buffer = (CVImageBufferRef)data;
  52     CVPixelBufferRelease(cv_buffer);
  53 }
  54
  55 static int videotoolbox_buffer_copy(VTContext *vtctx,
  56                                     const uint8_t *buffer,
  57                                     uint32_t size)
  58 {
  59     void *tmp;
  60
  61     tmp = av_fast_realloc(vtctx->bitstream,
  62                          &vtctx->allocated_size,
  63                          size);
  64
  65     if (!tmp)
  66         return AVERROR(ENOMEM);
  67
  68     vtctx->bitstream = tmp;
  69     memcpy(vtctx->bitstream, buffer, size);
  70     vtctx->bitstream_size = size;
  71
  72     return 0;
  73 }
  74
  75 int ff_videotoolbox_alloc_frame(AVCodecContext *avctx, AVFrame *frame)
  76 {
  77     frame->width  = avctx->width;
  78     frame->height = avctx->height;
  79     frame->format = avctx->pix_fmt;
  80     frame->buf[0] = av_buffer_alloc(1);
  81
  82     if (!frame->buf[0])
  83         return AVERROR(ENOMEM);
  84
  85     return 0;
  86 }
  87
  88 #define AV_W8(p, v) *(p) = (v)
  89
  90 CFDataRef ff_videotoolbox_avcc_extradata_create(AVCodecContext *avctx)
  91 {
  92     H264Context *h     = avctx->priv_data;
  93     CFDataRef data = NULL;
  94     uint8_t *p;
  95     int vt_extradata_size = 6 + 2 + h->ps.sps->data_size + 3 + h->ps.pps->data_size;
  96     uint8_t *vt_extradata = av_malloc(vt_extradata_size);
  97     if (!vt_extradata)
  98         return NULL;
  99
 100     p = vt_extradata;
 101
 102     AV_W8(p + 0, 1); /* version */
 103     AV_W8(p + 1, h->ps.sps->data[1]); /* profile */
 104     AV_W8(p + 2, h->ps.sps->data[2]); /* profile compat */
 105     AV_W8(p + 3, h->ps.sps->data[3]); /* level */
 106     AV_W8(p + 4, 0xff); /* 6 bits reserved (111111) + 2 bits nal size length - 3 (11) */
 107     AV_W8(p + 5, 0xe1); /* 3 bits reserved (111) + 5 bits number of sps (00001) */
 108     AV_WB16(p + 6, h->ps.sps->data_size);
 109     memcpy(p + 8, h->ps.sps->data, h->ps.sps->data_size);
 110     p += 8 + h->ps.sps->data_size;
 111     AV_W8(p + 0, 1); /* number of pps */
 112     AV_WB16(p + 1, h->ps.pps->data_size);
 113     memcpy(p + 3, h->ps.pps->data, h->ps.pps->data_size);
 114
 115     p += 3 + h->ps.pps->data_size;
 116     av_assert0(p - vt_extradata == vt_extradata_size);
 117
 118     data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
 119     av_free(vt_extradata);
 120     return data;
 121 }
 122
 123 CFDataRef ff_videotoolbox_hvcc_extradata_create(AVCodecContext *avctx)
 124 {
 125     HEVCContext *h = avctx->priv_data;
 126     const HEVCVPS *vps = (const HEVCVPS *)h->ps.vps_list[0]->data;
 127     const HEVCSPS *sps = (const HEVCSPS *)h->ps.sps_list[0]->data;
 128     int i, num_pps = 0;
 129     const HEVCPPS *pps = h->ps.pps;
 130     PTLCommon ptlc = vps->ptl.general_ptl;
 131     VUI vui = sps->vui;
 132     uint8_t parallelismType;
 133     CFDataRef data = NULL;
 134     uint8_t *p;
 135     int vt_extradata_size = 23 + 5 + vps->data_size + 5 + sps->data_size + 3;
 136     uint8_t *vt_extradata;
 137
 138     for (i = 0; i < MAX_PPS_COUNT; i++) {
 139         if (h->ps.pps_list[i]) {
 140             const HEVCPPS *pps = (const HEVCPPS *)h->ps.pps_list[i]->data;
 141             vt_extradata_size += 2 + pps->data_size;
 142             num_pps++;
 143         }
 144     }
 145
 146     vt_extradata = av_malloc(vt_extradata_size);
 147     if (!vt_extradata)
 148         return NULL;
 149     p = vt_extradata;
 150
 151     /* unsigned int(8) configurationVersion = 1; */
 152     AV_W8(p + 0, 1);
 153
 154     /*
 155      * unsigned int(2) general_profile_space;
 156      * unsigned int(1) general_tier_flag;
 157      * unsigned int(5) general_profile_idc;
 158      */
 159     AV_W8(p + 1, ptlc.profile_space << 6 |
 160                  ptlc.tier_flag     << 5 |
 161                  ptlc.profile_idc);
 162
 163     /* unsigned int(32) general_profile_compatibility_flags; */
 164     memcpy(p + 2, ptlc.profile_compatibility_flag, 4);
 165
 166     /* unsigned int(48) general_constraint_indicator_flags; */
 167     AV_W8(p + 6, ptlc.progressive_source_flag    << 7 |
 168                  ptlc.interlaced_source_flag     << 6 |
 169                  ptlc.non_packed_constraint_flag << 5 |
 170                  ptlc.frame_only_constraint_flag << 4);
 171     AV_W8(p + 7, 0);
 172     AV_WN32(p + 8, 0);
 173
 174     /* unsigned int(8) general_level_idc; */
 175     AV_W8(p + 12, ptlc.level_idc);
 176
 177     /*
 178      * bit(4) reserved = ‘1111’b;
 179      * unsigned int(12) min_spatial_segmentation_idc;
 180      */
 181     AV_W8(p + 13, 0xf0 | (vui.min_spatial_segmentation_idc >> 4));
 182     AV_W8(p + 14, vui.min_spatial_segmentation_idc & 0xff);
 183
 184     /*
 185      * bit(6) reserved = ‘111111’b;
 186      * unsigned int(2) parallelismType;
 187      */
 188     if (!vui.min_spatial_segmentation_idc)
 189         parallelismType = 0;
 190     else if (pps->entropy_coding_sync_enabled_flag && pps->tiles_enabled_flag)
 191         parallelismType = 0;
 192     else if (pps->entropy_coding_sync_enabled_flag)
 193         parallelismType = 3;
 194     else if (pps->tiles_enabled_flag)
 195         parallelismType = 2;
 196     else
 197         parallelismType = 1;
 198     AV_W8(p + 15, 0xfc | parallelismType);
 199
 200     /*
 201      * bit(6) reserved = ‘111111’b;
 202      * unsigned int(2) chromaFormat;
 203      */
 204     AV_W8(p + 16, sps->chroma_format_idc | 0xfc);
 205
 206     /*
 207      * bit(5) reserved = ‘11111’b;
 208      * unsigned int(3) bitDepthLumaMinus8;
 209      */
 210     AV_W8(p + 17, (sps->bit_depth - 8) | 0xfc);
 211
 212     /*
 213      * bit(5) reserved = ‘11111’b;
 214      * unsigned int(3) bitDepthChromaMinus8;
 215      */
 216     AV_W8(p + 18, (sps->bit_depth_chroma - 8) | 0xfc);
 217
 218     /* bit(16) avgFrameRate; */
 219     AV_WB16(p + 19, 0);
 220
 221     /*
 222      * bit(2) constantFrameRate;
 223      * bit(3) numTemporalLayers;
 224      * bit(1) temporalIdNested;
 225      * unsigned int(2) lengthSizeMinusOne;
 226      */
 227     AV_W8(p + 21, 0                             << 6 |
 228                   sps->max_sub_layers           << 3 |
 229                   sps->temporal_id_nesting_flag << 2 |
 230                   3);
 231
 232     /* unsigned int(8) numOfArrays; */
 233     AV_W8(p + 22, 3);
 234
 235     p += 23;
 236     /* vps */
 237     /*
 238      * bit(1) array_completeness;
 239      * unsigned int(1) reserved = 0;
 240      * unsigned int(6) NAL_unit_type;
 241      */
 242     AV_W8(p, 1 << 7 |
 243              HEVC_NAL_VPS & 0x3f);
 244     /* unsigned int(16) numNalus; */
 245     AV_WB16(p + 1, 1);
 246     /* unsigned int(16) nalUnitLength; */
 247     AV_WB16(p + 3, vps->data_size);
 248     /* bit(8*nalUnitLength) nalUnit; */
 249     memcpy(p + 5, vps->data, vps->data_size);
 250     p += 5 + vps->data_size;
 251
 252     /* sps */
 253     AV_W8(p, 1 << 7 |
 254              HEVC_NAL_SPS & 0x3f);
 255     AV_WB16(p + 1, 1);
 256     AV_WB16(p + 3, sps->data_size);
 257     memcpy(p + 5, sps->data, sps->data_size);
 258     p += 5 + sps->data_size;
 259
 260     /* pps */
 261     AV_W8(p, 1 << 7 |
 262              HEVC_NAL_PPS & 0x3f);
 263     AV_WB16(p + 1, num_pps);
 264     p += 3;
 265     for (i = 0; i < MAX_PPS_COUNT; i++) {
 266         if (h->ps.pps_list[i]) {
 267             const HEVCPPS *pps = (const HEVCPPS *)h->ps.pps_list[i]->data;
 268             AV_WB16(p, pps->data_size);
 269             memcpy(p + 2, pps->data, pps->data_size);
 270             p += 2 + pps->data_size;
 271         }
 272     }
 273
 274     av_assert0(p - vt_extradata == vt_extradata_size);
 275
 276     data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
 277     av_free(vt_extradata);
 278     return data;
 279 }
 280
 281 int ff_videotoolbox_buffer_create(VTContext *vtctx, AVFrame *frame)
 282 {
 283     av_buffer_unref(&frame->buf[0]);
 284
 285     frame->buf[0] = av_buffer_create((uint8_t*)vtctx->frame,
 286                                      sizeof(vtctx->frame),
 287                                      videotoolbox_buffer_release,
 288                                      NULL,
 289                                      AV_BUFFER_FLAG_READONLY);
 290     if (!frame->buf[0]) {
 291         return AVERROR(ENOMEM);
 292     }
 293
 294     frame->data[3] = (uint8_t*)vtctx->frame;
 295     vtctx->frame = NULL;
 296
 297     return 0;
 298 }
 299
 300 int ff_videotoolbox_h264_start_frame(AVCodecContext *avctx,
 301                                      const uint8_t *buffer,
 302                                      uint32_t size)
 303 {
 304     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
 305     H264Context *h  = avctx->priv_data;
 306
 307     vtctx->bitstream_size = 0;
 308
 309     if (h->is_avc == 1) {
 310         return videotoolbox_buffer_copy(vtctx, buffer, size);
 311     }
 312
 313     return 0;
 314 }
 315
 316 int ff_videotoolbox_h264_decode_slice(AVCodecContext *avctx,
 317                                       const uint8_t *buffer,
 318                                       uint32_t size)
 319 {
 320     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
 321     H264Context *h  = avctx->priv_data;
 322     void *tmp;
 323
 324     if (h->is_avc == 1)
 325         return 0;
 326
 327     tmp = av_fast_realloc(vtctx->bitstream,
 328                           &vtctx->allocated_size,
 329                           vtctx->bitstream_size+size+4);
 330     if (!tmp)
 331         return AVERROR(ENOMEM);
 332
 333     vtctx->bitstream = tmp;
 334
 335     AV_WB32(vtctx->bitstream + vtctx->bitstream_size, size);
 336     memcpy(vtctx->bitstream + vtctx->bitstream_size + 4, buffer, size);
 337
 338     vtctx->bitstream_size += size + 4;
 339
 340     return 0;
 341 }
 342
 343 int ff_videotoolbox_uninit(AVCodecContext *avctx)
 344 {
 345     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
 346     if (vtctx) {
 347         av_freep(&vtctx->bitstream);
 348         if (vtctx->frame)
 349             CVPixelBufferRelease(vtctx->frame);
 350     }
 351
 352     return 0;
 353 }
 354
 355 #if CONFIG_VIDEOTOOLBOX
 356 // Return the AVVideotoolboxContext that matters currently. Where it comes from
 357 // depends on the API used.
 358 static AVVideotoolboxContext *videotoolbox_get_context(AVCodecContext *avctx)
 359 {
 360     // Somewhat tricky because the user can call av_videotoolbox_default_free()
 361     // at any time, even when the codec is closed.
 362     if (avctx->internal && avctx->internal->hwaccel_priv_data) {
 363         VTContext *vtctx = avctx->internal->hwaccel_priv_data;
 364         if (vtctx->vt_ctx)
 365             return vtctx->vt_ctx;
 366     }
 367     return avctx->hwaccel_context;
 368 }
 369
 370 static int videotoolbox_buffer_create(AVCodecContext *avctx, AVFrame *frame)
 371 {
 372     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
 373     CVPixelBufferRef pixbuf = (CVPixelBufferRef)vtctx->frame;
 374     OSType pixel_format = CVPixelBufferGetPixelFormatType(pixbuf);
 375     enum AVPixelFormat sw_format = av_map_videotoolbox_format_to_pixfmt(pixel_format);
 376     int width = CVPixelBufferGetWidth(pixbuf);
 377     int height = CVPixelBufferGetHeight(pixbuf);
 378     AVHWFramesContext *cached_frames;
 379     int ret;
 380
 381     ret = ff_videotoolbox_buffer_create(vtctx, frame);
 382     if (ret < 0)
 383         return ret;
 384
 385     // Old API code path.
 386     if (!vtctx->cached_hw_frames_ctx)
 387         return 0;
 388
 389     cached_frames = (AVHWFramesContext*)vtctx->cached_hw_frames_ctx->data;
 390
 391     if (cached_frames->sw_format != sw_format ||
 392         cached_frames->width != width ||
 393         cached_frames->height != height) {
 394         AVBufferRef *hw_frames_ctx = av_hwframe_ctx_alloc(cached_frames->device_ref);
 395         AVHWFramesContext *hw_frames;
 396         if (!hw_frames_ctx)
 397             return AVERROR(ENOMEM);
 398
 399         hw_frames = (AVHWFramesContext*)hw_frames_ctx->data;
 400         hw_frames->format = cached_frames->format;
 401         hw_frames->sw_format = sw_format;
 402         hw_frames->width = width;
 403         hw_frames->height = height;
 404
 405         ret = av_hwframe_ctx_init(hw_frames_ctx);
 406         if (ret < 0) {
 407             av_buffer_unref(&hw_frames_ctx);
 408             return ret;
 409         }
 410
 411         av_buffer_unref(&vtctx->cached_hw_frames_ctx);
 412         vtctx->cached_hw_frames_ctx = hw_frames_ctx;
 413     }
 414
 415     av_buffer_unref(&frame->hw_frames_ctx);
 416     frame->hw_frames_ctx = av_buffer_ref(vtctx->cached_hw_frames_ctx);
 417     if (!frame->hw_frames_ctx)
 418         return AVERROR(ENOMEM);
 419
 420     return 0;
 421 }
 422
 423 static void videotoolbox_write_mp4_descr_length(PutByteContext *pb, int length)
 424 {
 425     int i;
 426     uint8_t b;
 427
 428     for (i = 3; i >= 0; i--) {
 429         b = (length >> (i * 7)) & 0x7F;
 430         if (i != 0)
 431             b |= 0x80;
 432
 433         bytestream2_put_byteu(pb, b);
 434     }
 435 }
 436
 437 static CFDataRef videotoolbox_esds_extradata_create(AVCodecContext *avctx)
 438 {
 439     CFDataRef data;
 440     uint8_t *rw_extradata;
 441     PutByteContext pb;
 442     int full_size = 3 + 5 + 13 + 5 + avctx->extradata_size + 3;
 443     // ES_DescrTag data + DecoderConfigDescrTag + data + DecSpecificInfoTag + size + SLConfigDescriptor
 444     int config_size = 13 + 5 + avctx->extradata_size;
 445     int s;
 446
 447     if (!(rw_extradata = av_mallocz(full_size + VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING)))
 448         return NULL;
 449
 450     bytestream2_init_writer(&pb, rw_extradata, full_size + VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING);
 451     bytestream2_put_byteu(&pb, 0);        // version
 452     bytestream2_put_ne24(&pb, 0);         // flags
 453
 454     // elementary stream descriptor
 455     bytestream2_put_byteu(&pb, 0x03);     // ES_DescrTag
 456     videotoolbox_write_mp4_descr_length(&pb, full_size);
 457     bytestream2_put_ne16(&pb, 0);         // esid
 458     bytestream2_put_byteu(&pb, 0);        // stream priority (0-32)
 459
 460     // decoder configuration descriptor
 461     bytestream2_put_byteu(&pb, 0x04);     // DecoderConfigDescrTag
 462     videotoolbox_write_mp4_descr_length(&pb, config_size);
 463     bytestream2_put_byteu(&pb, 32);       // object type indication. 32 = AV_CODEC_ID_MPEG4
 464     bytestream2_put_byteu(&pb, 0x11);     // stream type
 465     bytestream2_put_ne24(&pb, 0);         // buffer size
 466     bytestream2_put_ne32(&pb, 0);         // max bitrate
 467     bytestream2_put_ne32(&pb, 0);         // avg bitrate
 468
 469     // decoder specific descriptor
 470     bytestream2_put_byteu(&pb, 0x05);     ///< DecSpecificInfoTag
 471     videotoolbox_write_mp4_descr_length(&pb, avctx->extradata_size);
 472
 473     bytestream2_put_buffer(&pb, avctx->extradata, avctx->extradata_size);
 474
 475     // SLConfigDescriptor
 476     bytestream2_put_byteu(&pb, 0x06);     // SLConfigDescrTag
 477     bytestream2_put_byteu(&pb, 0x01);     // length
 478     bytestream2_put_byteu(&pb, 0x02);     //
 479
 480     s = bytestream2_size_p(&pb);
 481
 482     data = CFDataCreate(kCFAllocatorDefault, rw_extradata, s);
 483
 484     av_freep(&rw_extradata);
 485     return data;
 486 }
 487
 488 static CMSampleBufferRef videotoolbox_sample_buffer_create(CMFormatDescriptionRef fmt_desc,
 489                                                            void *buffer,
 490                                                            int size)
 491 {
 492     OSStatus status;
 493     CMBlockBufferRef  block_buf;
 494     CMSampleBufferRef sample_buf;
 495
 496     block_buf  = NULL;
 497     sample_buf = NULL;
 498
 499     status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault,// structureAllocator
 500                                                 buffer,             // memoryBlock
 501                                                 size,               // blockLength
 502                                                 kCFAllocatorNull,   // blockAllocator
 503                                                 NULL,               // customBlockSource
 504                                                 0,                  // offsetToData
 505                                                 size,               // dataLength
 506                                                 0,                  // flags
 507                                                 &block_buf);
 508
 509     if (!status) {
 510         status = CMSampleBufferCreate(kCFAllocatorDefault,  // allocator
 511                                       block_buf,            // dataBuffer
 512                                       TRUE,                 // dataReady
 513                                       0,                    // makeDataReadyCallback
 514                                       0,                    // makeDataReadyRefcon
 515                                       fmt_desc,             // formatDescription
 516                                       1,                    // numSamples
 517                                       0,                    // numSampleTimingEntries
 518                                       NULL,                 // sampleTimingArray
 519                                       0,                    // numSampleSizeEntries
 520                                       NULL,                 // sampleSizeArray
 521                                       &sample_buf);
 522     }
 523
 524     if (block_buf)
 525         CFRelease(block_buf);
 526
 527     return sample_buf;
 528 }
 529
 530 static void videotoolbox_decoder_callback(void *opaque,
 531                                           void *sourceFrameRefCon,
 532                                           OSStatus status,
 533                                           VTDecodeInfoFlags flags,
 534                                           CVImageBufferRef image_buffer,
 535                                           CMTime pts,
 536                                           CMTime duration)
 537 {
 538     AVCodecContext *avctx = opaque;
 539     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
 540
 541     if (vtctx->frame) {
 542         CVPixelBufferRelease(vtctx->frame);
 543         vtctx->frame = NULL;
 544     }
 545
 546     if (!image_buffer) {
 547         av_log(NULL, AV_LOG_DEBUG, "vt decoder cb: output image buffer is null\n");
 548         return;
 549     }
 550
 551     vtctx->frame = CVPixelBufferRetain(image_buffer);
 552 }
 553
 554 static OSStatus videotoolbox_session_decode_frame(AVCodecContext *avctx)
 555 {
 556     OSStatus status;
 557     CMSampleBufferRef sample_buf;
 558     AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx);
 559     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
 560
 561     sample_buf = videotoolbox_sample_buffer_create(videotoolbox->cm_fmt_desc,
 562                                                    vtctx->bitstream,
 563                                                    vtctx->bitstream_size);
 564
 565     if (!sample_buf)
 566         return -1;
 567
 568     status = VTDecompressionSessionDecodeFrame(videotoolbox->session,
 569                                                sample_buf,
 570                                                0,       // decodeFlags
 571                                                NULL,    // sourceFrameRefCon
 572                                                0);      // infoFlagsOut
 573     if (status == noErr)
 574         status = VTDecompressionSessionWaitForAsynchronousFrames(videotoolbox->session);
 575
 576     CFRelease(sample_buf);
 577
 578     return status;
 579 }
 580
 581 static int videotoolbox_common_end_frame(AVCodecContext *avctx, AVFrame *frame)
 582 {
 583     int status;
 584     AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx);
 585     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
 586
 587     if (!videotoolbox->session || !vtctx->bitstream)
 588         return AVERROR_INVALIDDATA;
 589
 590     status = videotoolbox_session_decode_frame(avctx);
 591
 592     if (status) {
 593         av_log(avctx, AV_LOG_ERROR, "Failed to decode frame (%d)\n", status);
 594         return AVERROR_UNKNOWN;
 595     }
 596
 597     if (!vtctx->frame)
 598         return AVERROR_UNKNOWN;
 599
 600     return videotoolbox_buffer_create(avctx, frame);
 601 }
 602
 603 static int videotoolbox_h264_end_frame(AVCodecContext *avctx)
 604 {
 605     H264Context *h = avctx->priv_data;
 606     AVFrame *frame = h->cur_pic_ptr->f;
 607
 608     return videotoolbox_common_end_frame(avctx, frame);
 609 }
 610
 611 static int videotoolbox_hevc_end_frame(AVCodecContext *avctx)
 612 {
 613     HEVCContext *h = avctx->priv_data;
 614     AVFrame *frame = h->ref->frame;
 615     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
 616     int ret;
 617
 618     ret = videotoolbox_common_end_frame(avctx, frame);
 619     vtctx->bitstream_size = 0;
 620     return ret;
 621 }
 622
 623 static int videotoolbox_mpeg_start_frame(AVCodecContext *avctx,
 624                                          const uint8_t *buffer,
 625                                          uint32_t size)
 626 {
 627     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
 628
 629     return videotoolbox_buffer_copy(vtctx, buffer, size);
 630 }
 631
 632 static int videotoolbox_mpeg_decode_slice(AVCodecContext *avctx,
 633                                           const uint8_t *buffer,
 634                                           uint32_t size)
 635 {
 636     return 0;
 637 }
 638
 639 static int videotoolbox_mpeg_end_frame(AVCodecContext *avctx)
 640 {
 641     MpegEncContext *s = avctx->priv_data;
 642     AVFrame *frame = s->current_picture_ptr->f;
 643
 644     return videotoolbox_common_end_frame(avctx, frame);
 645 }
 646
 647 static CFDictionaryRef videotoolbox_decoder_config_create(CMVideoCodecType codec_type,
 648                                                           AVCodecContext *avctx)
 649 {
 650     CFMutableDictionaryRef config_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
 651                                                                    0,
 652                                                                    &kCFTypeDictionaryKeyCallBacks,
 653                                                                    &kCFTypeDictionaryValueCallBacks);
 654
 655     CFDictionarySetValue(config_info,
 656                          kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder,
 657                          kCFBooleanTrue);
 658
 659     if (avctx->extradata_size) {
 660         CFMutableDictionaryRef avc_info;
 661         CFDataRef data = NULL;
 662
 663         avc_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
 664                                              1,
 665                                              &kCFTypeDictionaryKeyCallBacks,
 666                                              &kCFTypeDictionaryValueCallBacks);
 667
 668         switch (codec_type) {
 669         case kCMVideoCodecType_MPEG4Video :
 670             data = videotoolbox_esds_extradata_create(avctx);
 671             if (data)
 672                 CFDictionarySetValue(avc_info, CFSTR("esds"), data);
 673             break;
 674         case kCMVideoCodecType_H264 :
 675             data = ff_videotoolbox_avcc_extradata_create(avctx);
 676             if (data)
 677                 CFDictionarySetValue(avc_info, CFSTR("avcC"), data);
 678             break;
 679         case kCMVideoCodecType_HEVC :
 680             data = ff_videotoolbox_hvcc_extradata_create(avctx);
 681             if (data)
 682                 CFDictionarySetValue(avc_info, CFSTR("hvcC"), data);
 683             break;
 684         default:
 685             break;
 686         }
 687
 688         CFDictionarySetValue(config_info,
 689                 kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms,
 690                 avc_info);
 691
 692         if (data)
 693             CFRelease(data);
 694
 695         CFRelease(avc_info);
 696     }
 697     return config_info;
 698 }
 699
 700 static CFDictionaryRef videotoolbox_buffer_attributes_create(int width,
 701                                                              int height,
 702                                                              OSType pix_fmt)
 703 {
 704     CFMutableDictionaryRef buffer_attributes;
 705     CFMutableDictionaryRef io_surface_properties;
 706     CFNumberRef cv_pix_fmt;
 707     CFNumberRef w;
 708     CFNumberRef h;
 709
 710     w = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &width);
 711     h = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &height);
 712     cv_pix_fmt = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pix_fmt);
 713
 714     buffer_attributes = CFDictionaryCreateMutable(kCFAllocatorDefault,
 715                                                   4,
 716                                                   &kCFTypeDictionaryKeyCallBacks,
 717                                                   &kCFTypeDictionaryValueCallBacks);
 718     io_surface_properties = CFDictionaryCreateMutable(kCFAllocatorDefault,
 719                                                       0,
 720                                                       &kCFTypeDictionaryKeyCallBacks,
 721                                                       &kCFTypeDictionaryValueCallBacks);
 722
 723     if (pix_fmt)
 724         CFDictionarySetValue(buffer_attributes, kCVPixelBufferPixelFormatTypeKey, cv_pix_fmt);
 725     CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfacePropertiesKey, io_surface_properties);
 726     CFDictionarySetValue(buffer_attributes, kCVPixelBufferWidthKey, w);
 727     CFDictionarySetValue(buffer_attributes, kCVPixelBufferHeightKey, h);
 728 #if TARGET_OS_IPHONE
 729     CFDictionarySetValue(buffer_attributes, kCVPixelBufferOpenGLESCompatibilityKey, kCFBooleanTrue);
 730 #else
 731     CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfaceOpenGLTextureCompatibilityKey, kCFBooleanTrue);
 732 #endif
 733
 734     CFRelease(io_surface_properties);
 735     CFRelease(cv_pix_fmt);
 736     CFRelease(w);
 737     CFRelease(h);
 738
 739     return buffer_attributes;
 740 }
 741
 742 static CMVideoFormatDescriptionRef videotoolbox_format_desc_create(CMVideoCodecType codec_type,
 743                                                                    CFDictionaryRef decoder_spec,
 744                                                                    int width,
 745                                                                    int height)
 746 {
 747     CMFormatDescriptionRef cm_fmt_desc;
 748     OSStatus status;
 749
 750     status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault,
 751                                             codec_type,
 752                                             width,
 753                                             height,
 754                                             decoder_spec, // Dictionary of extension
 755                                             &cm_fmt_desc);
 756
 757     if (status)
 758         return NULL;
 759
 760     return cm_fmt_desc;
 761 }
 762
 763 static int videotoolbox_default_init(AVCodecContext *avctx)
 764 {
 765     AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx);
 766     OSStatus status;
 767     VTDecompressionOutputCallbackRecord decoder_cb;
 768     CFDictionaryRef decoder_spec;
 769     CFDictionaryRef buf_attr;
 770
 771     if (!videotoolbox) {
 772         av_log(avctx, AV_LOG_ERROR, "hwaccel context is not set\n");
 773         return -1;
 774     }
 775
 776     switch( avctx->codec_id ) {
 777     case AV_CODEC_ID_H263 :
 778         videotoolbox->cm_codec_type = kCMVideoCodecType_H263;
 779         break;
 780     case AV_CODEC_ID_H264 :
 781         videotoolbox->cm_codec_type = kCMVideoCodecType_H264;
 782         break;
 783     case AV_CODEC_ID_HEVC :
 784         videotoolbox->cm_codec_type = kCMVideoCodecType_HEVC;
 785         break;
 786     case AV_CODEC_ID_MPEG1VIDEO :
 787         videotoolbox->cm_codec_type = kCMVideoCodecType_MPEG1Video;
 788         break;
 789     case AV_CODEC_ID_MPEG2VIDEO :
 790         videotoolbox->cm_codec_type = kCMVideoCodecType_MPEG2Video;
 791         break;
 792     case AV_CODEC_ID_MPEG4 :
 793         videotoolbox->cm_codec_type = kCMVideoCodecType_MPEG4Video;
 794         break;
 795     default :
 796         break;
 797     }
 798
 799     decoder_spec = videotoolbox_decoder_config_create(videotoolbox->cm_codec_type, avctx);
 800
 801     videotoolbox->cm_fmt_desc = videotoolbox_format_desc_create(videotoolbox->cm_codec_type,
 802                                                                 decoder_spec,
 803                                                                 avctx->width,
 804                                                                 avctx->height);
 805     if (!videotoolbox->cm_fmt_desc) {
 806         if (decoder_spec)
 807             CFRelease(decoder_spec);
 808
 809         av_log(avctx, AV_LOG_ERROR, "format description creation failed\n");
 810         return -1;
 811     }
 812
 813     buf_attr = videotoolbox_buffer_attributes_create(avctx->width,
 814                                                      avctx->height,
 815                                                      videotoolbox->cv_pix_fmt_type);
 816
 817     decoder_cb.decompressionOutputCallback = videotoolbox_decoder_callback;
 818     decoder_cb.decompressionOutputRefCon   = avctx;
 819
 820     status = VTDecompressionSessionCreate(NULL,                      // allocator
 821                                           videotoolbox->cm_fmt_desc, // videoFormatDescription
 822                                           decoder_spec,              // videoDecoderSpecification
 823                                           buf_attr,                  // destinationImageBufferAttributes
 824                                           &decoder_cb,               // outputCallback
 825                                           &videotoolbox->session);   // decompressionSessionOut
 826
 827     if (decoder_spec)
 828         CFRelease(decoder_spec);
 829     if (buf_attr)
 830         CFRelease(buf_attr);
 831
 832     switch (status) {
 833     case kVTVideoDecoderNotAvailableNowErr:
 834         av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox session not available.\n");
 835         return AVERROR(ENOSYS);
 836     case kVTVideoDecoderUnsupportedDataFormatErr:
 837         av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox does not support this format.\n");
 838         return AVERROR(ENOSYS);
 839     case kVTVideoDecoderMalfunctionErr:
 840         av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox malfunction.\n");
 841         return AVERROR(EINVAL);
 842     case kVTVideoDecoderBadDataErr :
 843         av_log(avctx, AV_LOG_VERBOSE, "VideoToolbox reported invalid data.\n");
 844         return AVERROR_INVALIDDATA;
 845     case 0:
 846         return 0;
 847     default:
 848         av_log(avctx, AV_LOG_VERBOSE, "Unknown VideoToolbox session creation error %u\n", (unsigned)status);
 849         return AVERROR_UNKNOWN;
 850     }
 851 }
 852
 853 static void videotoolbox_default_free(AVCodecContext *avctx)
 854 {
 855     AVVideotoolboxContext *videotoolbox = videotoolbox_get_context(avctx);
 856
 857     if (videotoolbox) {
 858         if (videotoolbox->cm_fmt_desc)
 859             CFRelease(videotoolbox->cm_fmt_desc);
 860
 861         if (videotoolbox->session) {
 862             VTDecompressionSessionInvalidate(videotoolbox->session);
 863             CFRelease(videotoolbox->session);
 864         }
 865     }
 866 }
 867
 868 static int videotoolbox_uninit(AVCodecContext *avctx)
 869 {
 870     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
 871     if (!vtctx)
 872         return 0;
 873
 874     ff_videotoolbox_uninit(avctx);
 875
 876     if (vtctx->vt_ctx)
 877         videotoolbox_default_free(avctx);
 878
 879     av_buffer_unref(&vtctx->cached_hw_frames_ctx);
 880     av_freep(&vtctx->vt_ctx);
 881
 882     return 0;
 883 }
 884
 885 static int videotoolbox_common_init(AVCodecContext *avctx)
 886 {
 887     VTContext *vtctx = avctx->internal->hwaccel_priv_data;
 888     AVHWFramesContext *hw_frames;
 889     int err;
 890
 891     // Old API - do nothing.
 892     if (avctx->hwaccel_context)
 893         return 0;
 894
 895     if (!avctx->hw_frames_ctx && !avctx->hw_device_ctx) {
 896         av_log(avctx, AV_LOG_ERROR,
 897                "Either hw_frames_ctx or hw_device_ctx must be set.\n");
 898         return AVERROR(EINVAL);
 899     }
 900
 901     vtctx->vt_ctx = av_videotoolbox_alloc_context();
 902     if (!vtctx->vt_ctx) {
 903         err = AVERROR(ENOMEM);
 904         goto fail;
 905     }
 906
 907     if (avctx->hw_frames_ctx) {
 908         hw_frames = (AVHWFramesContext*)avctx->hw_frames_ctx->data;
 909     } else {
 910         avctx->hw_frames_ctx = av_hwframe_ctx_alloc(avctx->hw_device_ctx);
 911         if (!avctx->hw_frames_ctx) {
 912             err = AVERROR(ENOMEM);
 913             goto fail;
 914         }
 915
 916         hw_frames = (AVHWFramesContext*)avctx->hw_frames_ctx->data;
 917         hw_frames->format = AV_PIX_FMT_VIDEOTOOLBOX;
 918         hw_frames->sw_format = AV_PIX_FMT_NV12; // same as av_videotoolbox_alloc_context()
 919         hw_frames->width = avctx->width;
 920         hw_frames->height = avctx->height;
 921
 922         err = av_hwframe_ctx_init(avctx->hw_frames_ctx);
 923         if (err < 0) {
 924             av_buffer_unref(&avctx->hw_frames_ctx);
 925             goto fail;
 926         }
 927     }
 928
 929     vtctx->cached_hw_frames_ctx = av_buffer_ref(avctx->hw_frames_ctx);
 930     if (!vtctx->cached_hw_frames_ctx) {
 931         err = AVERROR(ENOMEM);
 932         goto fail;
 933     }
 934
 935     vtctx->vt_ctx->cv_pix_fmt_type =
 936         av_map_videotoolbox_format_from_pixfmt(hw_frames->sw_format);
 937     if (!vtctx->vt_ctx->cv_pix_fmt_type) {
 938         av_log(avctx, AV_LOG_ERROR, "Unknown sw_format.\n");
 939         err = AVERROR(EINVAL);
 940         goto fail;
 941     }
 942
 943     err = videotoolbox_default_init(avctx);
 944     if (err < 0)
 945         goto fail;
 946
 947     return 0;
 948
 949 fail:
 950     videotoolbox_uninit(avctx);
 951     return err;
 952 }
 953
 954 AVHWAccel ff_h263_videotoolbox_hwaccel = {
 955     .name           = "h263_videotoolbox",
 956     .type           = AVMEDIA_TYPE_VIDEO,
 957     .id             = AV_CODEC_ID_H263,
 958     .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
 959     .alloc_frame    = ff_videotoolbox_alloc_frame,
 960     .start_frame    = videotoolbox_mpeg_start_frame,
 961     .decode_slice   = videotoolbox_mpeg_decode_slice,
 962     .end_frame      = videotoolbox_mpeg_end_frame,
 963     .init           = videotoolbox_common_init,
 964     .uninit         = videotoolbox_uninit,
 965     .priv_data_size = sizeof(VTContext),
 966 };
 967
 968 AVHWAccel ff_hevc_videotoolbox_hwaccel = {
 969     .name           = "hevc_videotoolbox",
 970     .type           = AVMEDIA_TYPE_VIDEO,
 971     .id             = AV_CODEC_ID_HEVC,
 972     .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
 973     .alloc_frame    = ff_videotoolbox_alloc_frame,
 974     .start_frame    = ff_videotoolbox_h264_start_frame,
 975     .decode_slice   = ff_videotoolbox_h264_decode_slice,
 976     .end_frame      = videotoolbox_hevc_end_frame,
 977     .init           = videotoolbox_common_init,
 978     .uninit         = ff_videotoolbox_uninit,
 979     .priv_data_size = sizeof(VTContext),
 980 };
 981
 982 AVHWAccel ff_h264_videotoolbox_hwaccel = {
 983     .name           = "h264_videotoolbox",
 984     .type           = AVMEDIA_TYPE_VIDEO,
 985     .id             = AV_CODEC_ID_H264,
 986     .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
 987     .alloc_frame    = ff_videotoolbox_alloc_frame,
 988     .start_frame    = ff_videotoolbox_h264_start_frame,
 989     .decode_slice   = ff_videotoolbox_h264_decode_slice,
 990     .end_frame      = videotoolbox_h264_end_frame,
 991     .init           = videotoolbox_common_init,
 992     .uninit         = videotoolbox_uninit,
 993     .priv_data_size = sizeof(VTContext),
 994 };
 995
 996 AVHWAccel ff_mpeg1_videotoolbox_hwaccel = {
 997     .name           = "mpeg1_videotoolbox",
 998     .type           = AVMEDIA_TYPE_VIDEO,
 999     .id             = AV_CODEC_ID_MPEG1VIDEO,
1000     .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
1001     .alloc_frame    = ff_videotoolbox_alloc_frame,
1002     .start_frame    = videotoolbox_mpeg_start_frame,
1003     .decode_slice   = videotoolbox_mpeg_decode_slice,
1004     .end_frame      = videotoolbox_mpeg_end_frame,
1005     .init           = videotoolbox_common_init,
1006     .uninit         = videotoolbox_uninit,
1007     .priv_data_size = sizeof(VTContext),
1008 };
1009
1010 AVHWAccel ff_mpeg2_videotoolbox_hwaccel = {
1011     .name           = "mpeg2_videotoolbox",
1012     .type           = AVMEDIA_TYPE_VIDEO,
1013     .id             = AV_CODEC_ID_MPEG2VIDEO,
1014     .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
1015     .alloc_frame    = ff_videotoolbox_alloc_frame,
1016     .start_frame    = videotoolbox_mpeg_start_frame,
1017     .decode_slice   = videotoolbox_mpeg_decode_slice,
1018     .end_frame      = videotoolbox_mpeg_end_frame,
1019     .init           = videotoolbox_common_init,
1020     .uninit         = videotoolbox_uninit,
1021     .priv_data_size = sizeof(VTContext),
1022 };
1023
1024 AVHWAccel ff_mpeg4_videotoolbox_hwaccel = {
1025     .name           = "mpeg4_videotoolbox",
1026     .type           = AVMEDIA_TYPE_VIDEO,
1027     .id             = AV_CODEC_ID_MPEG4,
1028     .pix_fmt        = AV_PIX_FMT_VIDEOTOOLBOX,
1029     .alloc_frame    = ff_videotoolbox_alloc_frame,
1030     .start_frame    = videotoolbox_mpeg_start_frame,
1031     .decode_slice   = videotoolbox_mpeg_decode_slice,
1032     .end_frame      = videotoolbox_mpeg_end_frame,
1033     .init           = videotoolbox_common_init,
1034     .uninit         = videotoolbox_uninit,
1035     .priv_data_size = sizeof(VTContext),
1036 };
1037
1038 AVVideotoolboxContext *av_videotoolbox_alloc_context(void)
1039 {
1040     AVVideotoolboxContext *ret = av_mallocz(sizeof(*ret));
1041
1042     if (ret) {
1043         ret->output_callback = videotoolbox_decoder_callback;
1044         ret->cv_pix_fmt_type = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
1045     }
1046
1047     return ret;
1048 }
1049
1050 int av_videotoolbox_default_init(AVCodecContext *avctx)
1051 {
1052     return av_videotoolbox_default_init2(avctx, NULL);
1053 }
1054
1055 int av_videotoolbox_default_init2(AVCodecContext *avctx, AVVideotoolboxContext *vtctx)
1056 {
1057     avctx->hwaccel_context = vtctx ?: av_videotoolbox_alloc_context();
1058     if (!avctx->hwaccel_context)
1059         return AVERROR(ENOMEM);
1060     return videotoolbox_default_init(avctx);
1061 }
1062
1063 void av_videotoolbox_default_free(AVCodecContext *avctx)
1064 {
1065
1066     videotoolbox_default_free(avctx);
1067     av_freep(&avctx->hwaccel_context);
1068 }
1069 #endif /* CONFIG_VIDEOTOOLBOX */