2 * copyright (c) 2015 Rick Kern <kernrj@gmail.com>
4 * This file is part of FFmpeg.
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 #include <VideoToolbox/VideoToolbox.h>
22 #include <CoreVideo/CoreVideo.h>
23 #include <CoreMedia/CoreMedia.h>
24 #include <TargetConditionals.h>
25 #include <Availability.h>
27 #include "libavutil/opt.h"
28 #include "libavutil/avassert.h"
29 #include "libavutil/atomic.h"
30 #include "libavutil/avstring.h"
31 #include "libavcodec/avcodec.h"
32 #include "libavutil/pixdesc.h"
37 # define kCVImageBufferColorPrimaries_ITU_R_2020 CFSTR("ITU_R_2020")
38 # define kCVImageBufferTransferFunction_ITU_R_2020 CFSTR("ITU_R_2020")
39 # define kCVImageBufferYCbCrMatrix_ITU_R_2020 CFSTR("ITU_R_2020")
42 typedef enum VT_H264Profile {
50 typedef enum VTH264Entropy{
56 static const uint8_t start_code[] = { 0, 0, 0, 1 };
58 typedef struct BufNode {
59 CMSampleBufferRef cm_buffer;
64 typedef struct VTEncContext {
66 VTCompressionSessionRef session;
67 CFStringRef ycbcr_matrix;
68 CFStringRef color_primaries;
69 CFStringRef transfer_function;
72 pthread_cond_t cv_sample_sent;
89 int64_t frames_before;
96 bool warned_color_range;
99 static int vtenc_populate_extradata(AVCodecContext *avctx,
100 CMVideoCodecType codec_type,
101 CFStringRef profile_level,
102 CFNumberRef gamma_level,
103 CFDictionaryRef enc_info,
104 CFDictionaryRef pixel_buffer_info);
107 * NULL-safe release of *refPtr, and sets value to NULL.
109 static void vt_release_num(CFNumberRef* refPtr){
118 static void set_async_error(VTEncContext *vtctx, int err)
122 pthread_mutex_lock(&vtctx->lock);
124 vtctx->async_error = err;
126 info = vtctx->q_head;
127 vtctx->q_head = vtctx->q_tail = NULL;
130 BufNode *next = info->next;
131 CFRelease(info->cm_buffer);
136 pthread_mutex_unlock(&vtctx->lock);
139 static int vtenc_q_pop(VTEncContext *vtctx, bool wait, CMSampleBufferRef *buf)
143 pthread_mutex_lock(&vtctx->lock);
145 if (vtctx->async_error) {
146 pthread_mutex_unlock(&vtctx->lock);
147 return vtctx->async_error;
150 if (vtctx->flushing && vtctx->frame_ct_in == vtctx->frame_ct_out) {
153 pthread_mutex_unlock(&vtctx->lock);
157 while (!vtctx->q_head && !vtctx->async_error && wait) {
158 pthread_cond_wait(&vtctx->cv_sample_sent, &vtctx->lock);
161 if (!vtctx->q_head) {
162 pthread_mutex_unlock(&vtctx->lock);
167 info = vtctx->q_head;
168 vtctx->q_head = vtctx->q_head->next;
169 if (!vtctx->q_head) {
170 vtctx->q_tail = NULL;
173 pthread_mutex_unlock(&vtctx->lock);
175 *buf = info->cm_buffer;
178 vtctx->frame_ct_out++;
183 static void vtenc_q_push(VTEncContext *vtctx, CMSampleBufferRef buffer)
185 BufNode *info = av_malloc(sizeof(BufNode));
187 set_async_error(vtctx, AVERROR(ENOMEM));
192 info->cm_buffer = buffer;
195 pthread_mutex_lock(&vtctx->lock);
196 pthread_cond_signal(&vtctx->cv_sample_sent);
198 if (!vtctx->q_head) {
199 vtctx->q_head = info;
201 vtctx->q_tail->next = info;
204 vtctx->q_tail = info;
206 pthread_mutex_unlock(&vtctx->lock);
209 static int count_nalus(size_t length_code_size,
210 CMSampleBufferRef sample_buffer,
217 size_t src_size = CMSampleBufferGetTotalSampleSize(sample_buffer);
218 CMBlockBufferRef block = CMSampleBufferGetDataBuffer(sample_buffer);
220 if (length_code_size > 4)
221 return AVERROR_INVALIDDATA;
223 while (offset < src_size) {
228 status = CMBlockBufferCopyDataBytes(block,
233 for (i = 0; i < length_code_size; i++) {
235 box_len |= size_buf[i];
238 curr_src_len = box_len + length_code_size;
239 offset += curr_src_len;
248 static CMVideoCodecType get_cm_codec_type(enum AVCodecID id)
251 case AV_CODEC_ID_H264: return kCMVideoCodecType_H264;
257 * Get the parameter sets from a CMSampleBufferRef.
258 * @param dst If *dst isn't NULL, the parameters are copied into existing
259 * memory. *dst_size must be set accordingly when *dst != NULL.
260 * If *dst is NULL, it will be allocated.
261 * In all cases, *dst_size is set to the number of bytes used starting
264 static int get_params_size(
265 AVCodecContext *avctx,
266 CMVideoFormatDescriptionRef vid_fmt,
269 size_t total_size = 0;
271 int is_count_bad = 0;
274 status = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(vid_fmt,
286 for (i = 0; i < ps_count || is_count_bad; i++) {
289 status = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(vid_fmt,
297 * When ps_count is invalid, status != 0 ends the loop normally
298 * unless we didn't get any parameter sets.
300 if (i > 0 && is_count_bad) status = 0;
305 total_size += ps_size + sizeof(start_code);
309 av_log(avctx, AV_LOG_ERROR, "Error getting parameter set sizes: %d\n", status);
310 return AVERROR_EXTERNAL;
317 static int copy_param_sets(
318 AVCodecContext *avctx,
319 CMVideoFormatDescriptionRef vid_fmt,
324 int is_count_bad = 0;
329 status = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(vid_fmt,
342 for (i = 0; i < ps_count || is_count_bad; i++) {
347 status = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(vid_fmt,
354 if (i > 0 && is_count_bad) status = 0;
359 next_offset = offset + sizeof(start_code) + ps_size;
360 if (dst_size < next_offset) {
361 av_log(avctx, AV_LOG_ERROR, "Error: buffer too small for parameter sets.\n");
362 return AVERROR_BUFFER_TOO_SMALL;
365 memcpy(dst + offset, start_code, sizeof(start_code));
366 offset += sizeof(start_code);
368 memcpy(dst + offset, ps, ps_size);
369 offset = next_offset;
373 av_log(avctx, AV_LOG_ERROR, "Error getting parameter set data: %d\n", status);
374 return AVERROR_EXTERNAL;
380 static int set_extradata(AVCodecContext *avctx, CMSampleBufferRef sample_buffer)
382 CMVideoFormatDescriptionRef vid_fmt;
386 vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer);
388 av_log(avctx, AV_LOG_ERROR, "No video format.\n");
389 return AVERROR_EXTERNAL;
392 status = get_params_size(avctx, vid_fmt, &total_size);
394 av_log(avctx, AV_LOG_ERROR, "Could not get parameter sets.\n");
398 avctx->extradata = av_mallocz(total_size + AV_INPUT_BUFFER_PADDING_SIZE);
399 if (!avctx->extradata) {
400 return AVERROR(ENOMEM);
402 avctx->extradata_size = total_size;
404 status = copy_param_sets(avctx, vid_fmt, avctx->extradata, total_size);
407 av_log(avctx, AV_LOG_ERROR, "Could not copy param sets.\n");
414 static void vtenc_output_callback(
416 void *sourceFrameCtx,
418 VTEncodeInfoFlags flags,
419 CMSampleBufferRef sample_buffer)
421 AVCodecContext *avctx = ctx;
422 VTEncContext *vtctx = avctx->priv_data;
424 if (vtctx->async_error) {
425 if(sample_buffer) CFRelease(sample_buffer);
429 if (status || !sample_buffer) {
430 av_log(avctx, AV_LOG_ERROR, "Error encoding frame: %d\n", (int)status);
431 set_async_error(vtctx, AVERROR_EXTERNAL);
435 if (!avctx->extradata && (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER)) {
436 int set_status = set_extradata(avctx, sample_buffer);
438 set_async_error(vtctx, set_status);
443 vtenc_q_push(vtctx, sample_buffer);
446 static int get_length_code_size(
447 AVCodecContext *avctx,
448 CMSampleBufferRef sample_buffer,
451 CMVideoFormatDescriptionRef vid_fmt;
455 vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer);
457 av_log(avctx, AV_LOG_ERROR, "Error getting buffer format description.\n");
458 return AVERROR_EXTERNAL;
461 status = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(vid_fmt,
468 av_log(avctx, AV_LOG_ERROR, "Error getting length code size: %d\n", status);
469 return AVERROR_EXTERNAL;
477 * Returns true on success.
479 * If profile_level_val is NULL and this method returns true, don't specify the
480 * profile/level to the encoder.
482 static bool get_vt_profile_level(AVCodecContext *avctx,
483 CFStringRef *profile_level_val)
485 VTEncContext *vtctx = avctx->priv_data;
486 int64_t profile = vtctx->profile;
488 if (profile == H264_PROF_AUTO && vtctx->level) {
489 //Need to pick a profile if level is not auto-selected.
490 profile = vtctx->has_b_frames ? H264_PROF_MAIN : H264_PROF_BASELINE;
493 *profile_level_val = NULL;
499 case H264_PROF_BASELINE:
500 switch (vtctx->level) {
501 case 0: *profile_level_val = kVTProfileLevel_H264_Baseline_AutoLevel; break;
502 case 13: *profile_level_val = kVTProfileLevel_H264_Baseline_1_3; break;
503 case 30: *profile_level_val = kVTProfileLevel_H264_Baseline_3_0; break;
504 case 31: *profile_level_val = kVTProfileLevel_H264_Baseline_3_1; break;
505 case 32: *profile_level_val = kVTProfileLevel_H264_Baseline_3_2; break;
506 case 40: *profile_level_val = kVTProfileLevel_H264_Baseline_4_0; break;
507 case 41: *profile_level_val = kVTProfileLevel_H264_Baseline_4_1; break;
508 case 42: *profile_level_val = kVTProfileLevel_H264_Baseline_4_2; break;
509 case 50: *profile_level_val = kVTProfileLevel_H264_Baseline_5_0; break;
510 case 51: *profile_level_val = kVTProfileLevel_H264_Baseline_5_1; break;
511 case 52: *profile_level_val = kVTProfileLevel_H264_Baseline_5_2; break;
516 switch (vtctx->level) {
517 case 0: *profile_level_val = kVTProfileLevel_H264_Main_AutoLevel; break;
518 case 30: *profile_level_val = kVTProfileLevel_H264_Main_3_0; break;
519 case 31: *profile_level_val = kVTProfileLevel_H264_Main_3_1; break;
520 case 32: *profile_level_val = kVTProfileLevel_H264_Main_3_2; break;
521 case 40: *profile_level_val = kVTProfileLevel_H264_Main_4_0; break;
522 case 41: *profile_level_val = kVTProfileLevel_H264_Main_4_1; break;
523 case 42: *profile_level_val = kVTProfileLevel_H264_Main_4_2; break;
524 case 50: *profile_level_val = kVTProfileLevel_H264_Main_5_0; break;
525 case 51: *profile_level_val = kVTProfileLevel_H264_Main_5_1; break;
526 case 52: *profile_level_val = kVTProfileLevel_H264_Main_5_2; break;
531 switch (vtctx->level) {
532 case 0: *profile_level_val = kVTProfileLevel_H264_High_AutoLevel; break;
533 case 30: *profile_level_val = kVTProfileLevel_H264_High_3_0; break;
534 case 31: *profile_level_val = kVTProfileLevel_H264_High_3_1; break;
535 case 32: *profile_level_val = kVTProfileLevel_H264_High_3_2; break;
536 case 40: *profile_level_val = kVTProfileLevel_H264_High_4_0; break;
537 case 41: *profile_level_val = kVTProfileLevel_H264_High_4_1; break;
538 case 42: *profile_level_val = kVTProfileLevel_H264_High_4_2; break;
539 case 50: *profile_level_val = kVTProfileLevel_H264_High_5_0; break;
540 case 51: *profile_level_val = kVTProfileLevel_H264_High_5_1; break;
541 case 52: *profile_level_val = kVTProfileLevel_H264_High_5_2; break;
546 if (!*profile_level_val) {
547 av_log(avctx, AV_LOG_ERROR, "Invalid Profile/Level.\n");
554 static int get_cv_pixel_format(AVCodecContext* avctx,
555 enum AVPixelFormat fmt,
556 enum AVColorRange range,
557 int* av_pixel_format,
560 if (range_guessed) *range_guessed = range != AVCOL_RANGE_MPEG &&
561 range != AVCOL_RANGE_JPEG;
563 //MPEG range is used when no range is set
564 if (fmt == AV_PIX_FMT_NV12) {
565 *av_pixel_format = range == AVCOL_RANGE_JPEG ?
566 kCVPixelFormatType_420YpCbCr8BiPlanarFullRange :
567 kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
568 } else if (fmt == AV_PIX_FMT_YUV420P) {
569 *av_pixel_format = range == AVCOL_RANGE_JPEG ?
570 kCVPixelFormatType_420YpCbCr8PlanarFullRange :
571 kCVPixelFormatType_420YpCbCr8Planar;
573 return AVERROR(EINVAL);
579 static void add_color_attr(AVCodecContext *avctx, CFMutableDictionaryRef dict) {
580 VTEncContext *vtctx = avctx->priv_data;
582 if (vtctx->color_primaries) {
583 CFDictionarySetValue(dict,
584 kCVImageBufferColorPrimariesKey,
585 vtctx->color_primaries);
588 if (vtctx->transfer_function) {
589 CFDictionarySetValue(dict,
590 kCVImageBufferTransferFunctionKey,
591 vtctx->transfer_function);
594 if (vtctx->ycbcr_matrix) {
595 CFDictionarySetValue(dict,
596 kCVImageBufferYCbCrMatrixKey,
597 vtctx->ycbcr_matrix);
601 static int create_cv_pixel_buffer_info(AVCodecContext* avctx,
602 CFMutableDictionaryRef* dict)
604 CFNumberRef cv_color_format_num = NULL;
605 CFNumberRef width_num = NULL;
606 CFNumberRef height_num = NULL;
607 CFMutableDictionaryRef pixel_buffer_info = NULL;
609 int status = get_cv_pixel_format(avctx,
614 if (status) return status;
616 pixel_buffer_info = CFDictionaryCreateMutable(
619 &kCFCopyStringDictionaryKeyCallBacks,
620 &kCFTypeDictionaryValueCallBacks);
622 if (!pixel_buffer_info) goto pbinfo_nomem;
624 cv_color_format_num = CFNumberCreate(kCFAllocatorDefault,
627 if (!cv_color_format_num) goto pbinfo_nomem;
629 CFDictionarySetValue(pixel_buffer_info,
630 kCVPixelBufferPixelFormatTypeKey,
631 cv_color_format_num);
632 vt_release_num(&cv_color_format_num);
634 width_num = CFNumberCreate(kCFAllocatorDefault,
637 if (!width_num) return AVERROR(ENOMEM);
639 CFDictionarySetValue(pixel_buffer_info,
640 kCVPixelBufferWidthKey,
642 vt_release_num(&width_num);
644 height_num = CFNumberCreate(kCFAllocatorDefault,
647 if (!height_num) goto pbinfo_nomem;
649 CFDictionarySetValue(pixel_buffer_info,
650 kCVPixelBufferHeightKey,
652 vt_release_num(&height_num);
654 add_color_attr(avctx, pixel_buffer_info);
656 *dict = pixel_buffer_info;
660 vt_release_num(&cv_color_format_num);
661 vt_release_num(&width_num);
662 vt_release_num(&height_num);
663 if (pixel_buffer_info) CFRelease(pixel_buffer_info);
665 return AVERROR(ENOMEM);
668 static int get_cv_color_primaries(AVCodecContext *avctx,
669 CFStringRef *primaries)
671 enum AVColorPrimaries pri = avctx->color_primaries;
673 case AVCOL_PRI_UNSPECIFIED:
677 case AVCOL_PRI_BT709:
678 *primaries = kCVImageBufferColorPrimaries_ITU_R_709_2;
681 case AVCOL_PRI_BT2020:
682 *primaries = kCVImageBufferColorPrimaries_ITU_R_2020;
686 av_log(avctx, AV_LOG_ERROR, "Color primaries %s is not supported.\n", av_color_primaries_name(pri));
694 static int get_cv_transfer_function(AVCodecContext *avctx,
695 CFStringRef *transfer_fnc,
696 CFNumberRef *gamma_level)
698 enum AVColorTransferCharacteristic trc = avctx->color_trc;
703 case AVCOL_TRC_UNSPECIFIED:
704 *transfer_fnc = NULL;
707 case AVCOL_TRC_BT709:
708 *transfer_fnc = kCVImageBufferTransferFunction_ITU_R_709_2;
711 case AVCOL_TRC_SMPTE240M:
712 *transfer_fnc = kCVImageBufferTransferFunction_SMPTE_240M_1995;
715 case AVCOL_TRC_GAMMA22:
717 *transfer_fnc = kCVImageBufferTransferFunction_UseGamma;
718 *gamma_level = CFNumberCreate(NULL, kCFNumberFloat32Type, &gamma);
721 case AVCOL_TRC_GAMMA28:
723 *transfer_fnc = kCVImageBufferTransferFunction_UseGamma;
724 *gamma_level = CFNumberCreate(NULL, kCFNumberFloat32Type, &gamma);
727 case AVCOL_TRC_BT2020_10:
728 case AVCOL_TRC_BT2020_12:
729 *transfer_fnc = kCVImageBufferTransferFunction_ITU_R_2020;
733 av_log(avctx, AV_LOG_ERROR, "Transfer function %s is not supported.\n", av_color_transfer_name(trc));
740 static int get_cv_ycbcr_matrix(AVCodecContext *avctx, CFStringRef *matrix) {
741 switch(avctx->colorspace) {
742 case AVCOL_SPC_BT709:
743 *matrix = kCVImageBufferYCbCrMatrix_ITU_R_709_2;
746 case AVCOL_SPC_UNSPECIFIED:
750 case AVCOL_SPC_BT470BG:
751 case AVCOL_SPC_SMPTE170M:
752 *matrix = kCVImageBufferYCbCrMatrix_ITU_R_601_4;
755 case AVCOL_SPC_SMPTE240M:
756 *matrix = kCVImageBufferYCbCrMatrix_SMPTE_240M_1995;
759 case AVCOL_SPC_BT2020_NCL:
760 *matrix = kCVImageBufferYCbCrMatrix_ITU_R_2020;
764 av_log(avctx, AV_LOG_ERROR, "Color space %s is not supported.\n", av_color_space_name(avctx->colorspace));
771 static int vtenc_create_encoder(AVCodecContext *avctx,
772 CMVideoCodecType codec_type,
773 CFStringRef profile_level,
774 CFNumberRef gamma_level,
775 CFDictionaryRef enc_info,
776 CFDictionaryRef pixel_buffer_info,
777 VTCompressionSessionRef *session)
779 VTEncContext *vtctx = avctx->priv_data;
780 SInt32 bit_rate = avctx->bit_rate;
781 CFNumberRef bit_rate_num;
783 int status = VTCompressionSessionCreate(kCFAllocatorDefault,
790 vtenc_output_callback,
794 if (status || !vtctx->session) {
795 av_log(avctx, AV_LOG_ERROR, "Error: cannot create compression session: %d\n", status);
797 #if !TARGET_OS_IPHONE
798 if (!vtctx->allow_sw) {
799 av_log(avctx, AV_LOG_ERROR, "Try -allow_sw 1. The hardware encoder may be busy, or not supported.\n");
803 return AVERROR_EXTERNAL;
806 bit_rate_num = CFNumberCreate(kCFAllocatorDefault,
809 if (!bit_rate_num) return AVERROR(ENOMEM);
811 status = VTSessionSetProperty(vtctx->session,
812 kVTCompressionPropertyKey_AverageBitRate,
814 CFRelease(bit_rate_num);
817 av_log(avctx, AV_LOG_ERROR, "Error setting bitrate property: %d\n", status);
818 return AVERROR_EXTERNAL;
822 status = VTSessionSetProperty(vtctx->session,
823 kVTCompressionPropertyKey_ProfileLevel,
826 av_log(avctx, AV_LOG_ERROR, "Error setting profile/level property: %d\n", status);
827 return AVERROR_EXTERNAL;
831 if (avctx->gop_size > 0) {
832 CFNumberRef interval = CFNumberCreate(kCFAllocatorDefault,
836 return AVERROR(ENOMEM);
839 status = VTSessionSetProperty(vtctx->session,
840 kVTCompressionPropertyKey_MaxKeyFrameInterval,
845 av_log(avctx, AV_LOG_ERROR, "Error setting 'max key-frame interval' property: %d\n", status);
846 return AVERROR_EXTERNAL;
850 if (vtctx->frames_before) {
851 status = VTSessionSetProperty(vtctx->session,
852 kVTCompressionPropertyKey_MoreFramesBeforeStart,
855 if (status == kVTPropertyNotSupportedErr) {
856 av_log(avctx, AV_LOG_WARNING, "frames_before property is not supported on this device. Ignoring.\n");
858 av_log(avctx, AV_LOG_ERROR, "Error setting frames_before property: %d\n", status);
862 if (vtctx->frames_after) {
863 status = VTSessionSetProperty(vtctx->session,
864 kVTCompressionPropertyKey_MoreFramesAfterEnd,
867 if (status == kVTPropertyNotSupportedErr) {
868 av_log(avctx, AV_LOG_WARNING, "frames_after property is not supported on this device. Ignoring.\n");
870 av_log(avctx, AV_LOG_ERROR, "Error setting frames_after property: %d\n", status);
874 if (avctx->sample_aspect_ratio.num != 0) {
877 CFMutableDictionaryRef par;
878 AVRational *avpar = &avctx->sample_aspect_ratio;
880 av_reduce(&avpar->num, &avpar->den,
881 avpar->num, avpar->den,
884 num = CFNumberCreate(kCFAllocatorDefault,
888 den = CFNumberCreate(kCFAllocatorDefault,
894 par = CFDictionaryCreateMutable(kCFAllocatorDefault,
896 &kCFCopyStringDictionaryKeyCallBacks,
897 &kCFTypeDictionaryValueCallBacks);
899 if (!par || !num || !den) {
900 if (par) CFRelease(par);
901 if (num) CFRelease(num);
902 if (den) CFRelease(den);
904 return AVERROR(ENOMEM);
907 CFDictionarySetValue(
909 kCMFormatDescriptionKey_PixelAspectRatioHorizontalSpacing,
912 CFDictionarySetValue(
914 kCMFormatDescriptionKey_PixelAspectRatioVerticalSpacing,
917 status = VTSessionSetProperty(vtctx->session,
918 kVTCompressionPropertyKey_PixelAspectRatio,
928 "Error setting pixel aspect ratio to %d:%d: %d.\n",
929 avctx->sample_aspect_ratio.num,
930 avctx->sample_aspect_ratio.den,
933 return AVERROR_EXTERNAL;
938 if (vtctx->transfer_function) {
939 status = VTSessionSetProperty(vtctx->session,
940 kVTCompressionPropertyKey_TransferFunction,
941 vtctx->transfer_function);
944 av_log(avctx, AV_LOG_WARNING, "Could not set transfer function: %d\n", status);
949 if (vtctx->ycbcr_matrix) {
950 status = VTSessionSetProperty(vtctx->session,
951 kVTCompressionPropertyKey_YCbCrMatrix,
952 vtctx->ycbcr_matrix);
955 av_log(avctx, AV_LOG_WARNING, "Could not set ycbcr matrix: %d\n", status);
960 if (vtctx->color_primaries) {
961 status = VTSessionSetProperty(vtctx->session,
962 kVTCompressionPropertyKey_ColorPrimaries,
963 vtctx->color_primaries);
966 av_log(avctx, AV_LOG_WARNING, "Could not set color primaries: %d\n", status);
971 status = VTSessionSetProperty(vtctx->session,
972 kCVImageBufferGammaLevelKey,
976 av_log(avctx, AV_LOG_WARNING, "Could not set gamma level: %d\n", status);
980 if (!vtctx->has_b_frames) {
981 status = VTSessionSetProperty(vtctx->session,
982 kVTCompressionPropertyKey_AllowFrameReordering,
986 av_log(avctx, AV_LOG_ERROR, "Error setting 'allow frame reordering' property: %d\n", status);
987 return AVERROR_EXTERNAL;
991 if (vtctx->entropy != VT_ENTROPY_NOT_SET) {
992 CFStringRef entropy = vtctx->entropy == VT_CABAC ?
993 kVTH264EntropyMode_CABAC:
994 kVTH264EntropyMode_CAVLC;
996 status = VTSessionSetProperty(vtctx->session,
997 kVTCompressionPropertyKey_H264EntropyMode,
1001 av_log(avctx, AV_LOG_ERROR, "Error setting entropy property: %d\n", status);
1002 return AVERROR_EXTERNAL;
1006 if (vtctx->realtime) {
1007 status = VTSessionSetProperty(vtctx->session,
1008 kVTCompressionPropertyKey_RealTime,
1012 av_log(avctx, AV_LOG_ERROR, "Error setting realtime property: %d\n", status);
1016 status = VTCompressionSessionPrepareToEncodeFrames(vtctx->session);
1018 av_log(avctx, AV_LOG_ERROR, "Error: cannot prepare encoder: %d\n", status);
1019 return AVERROR_EXTERNAL;
1025 static av_cold int vtenc_init(AVCodecContext *avctx)
1027 CFMutableDictionaryRef enc_info;
1028 CFMutableDictionaryRef pixel_buffer_info;
1029 CMVideoCodecType codec_type;
1030 VTEncContext *vtctx = avctx->priv_data;
1031 CFStringRef profile_level;
1032 CFBooleanRef has_b_frames_cfbool;
1033 CFNumberRef gamma_level = NULL;
1036 codec_type = get_cm_codec_type(avctx->codec_id);
1038 av_log(avctx, AV_LOG_ERROR, "Error: no mapping for AVCodecID %d\n", avctx->codec_id);
1039 return AVERROR(EINVAL);
1042 vtctx->has_b_frames = avctx->max_b_frames > 0;
1043 if(vtctx->has_b_frames && vtctx->profile == H264_PROF_BASELINE){
1044 av_log(avctx, AV_LOG_WARNING, "Cannot use B-frames with baseline profile. Output will not contain B-frames.\n");
1045 vtctx->has_b_frames = false;
1048 if (vtctx->entropy == VT_CABAC && vtctx->profile == H264_PROF_BASELINE) {
1049 av_log(avctx, AV_LOG_WARNING, "CABAC entropy requires 'main' or 'high' profile, but baseline was requested. Encode will not use CABAC entropy.\n");
1050 vtctx->entropy = VT_ENTROPY_NOT_SET;
1053 if (!get_vt_profile_level(avctx, &profile_level)) return AVERROR(EINVAL);
1055 vtctx->session = NULL;
1057 enc_info = CFDictionaryCreateMutable(
1058 kCFAllocatorDefault,
1060 &kCFCopyStringDictionaryKeyCallBacks,
1061 &kCFTypeDictionaryValueCallBacks
1064 if (!enc_info) return AVERROR(ENOMEM);
1066 #if !TARGET_OS_IPHONE
1067 if (!vtctx->allow_sw) {
1068 CFDictionarySetValue(enc_info, kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder, kCFBooleanTrue);
1070 CFDictionarySetValue(enc_info, kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder, kCFBooleanTrue);
1074 if (avctx->pix_fmt != AV_PIX_FMT_VIDEOTOOLBOX) {
1075 status = create_cv_pixel_buffer_info(avctx, &pixel_buffer_info);
1079 pixel_buffer_info = NULL;
1082 pthread_mutex_init(&vtctx->lock, NULL);
1083 pthread_cond_init(&vtctx->cv_sample_sent, NULL);
1084 vtctx->dts_delta = vtctx->has_b_frames ? -1 : 0;
1086 get_cv_transfer_function(avctx, &vtctx->transfer_function, &gamma_level);
1087 get_cv_ycbcr_matrix(avctx, &vtctx->ycbcr_matrix);
1088 get_cv_color_primaries(avctx, &vtctx->color_primaries);
1091 if (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER) {
1092 status = vtenc_populate_extradata(avctx,
1102 status = vtenc_create_encoder(avctx,
1113 status = VTSessionCopyProperty(vtctx->session,
1114 kVTCompressionPropertyKey_AllowFrameReordering,
1115 kCFAllocatorDefault,
1116 &has_b_frames_cfbool);
1119 //Some devices don't output B-frames for main profile, even if requested.
1120 vtctx->has_b_frames = CFBooleanGetValue(has_b_frames_cfbool);
1121 CFRelease(has_b_frames_cfbool);
1123 avctx->has_b_frames = vtctx->has_b_frames;
1127 CFRelease(gamma_level);
1129 if (pixel_buffer_info)
1130 CFRelease(pixel_buffer_info);
1132 CFRelease(enc_info);
1137 static void vtenc_get_frame_info(CMSampleBufferRef buffer, bool *is_key_frame)
1139 CFArrayRef attachments;
1140 CFDictionaryRef attachment;
1141 CFBooleanRef not_sync;
1144 attachments = CMSampleBufferGetSampleAttachmentsArray(buffer, false);
1145 len = !attachments ? 0 : CFArrayGetCount(attachments);
1148 *is_key_frame = true;
1152 attachment = CFArrayGetValueAtIndex(attachments, 0);
1154 if (CFDictionaryGetValueIfPresent(attachment,
1155 kCMSampleAttachmentKey_NotSync,
1156 (const void **)¬_sync))
1158 *is_key_frame = !CFBooleanGetValue(not_sync);
1160 *is_key_frame = true;
1165 * Copies NAL units and replaces length codes with
1166 * H.264 Annex B start codes. On failure, the contents of
1167 * dst_data may have been modified.
1169 * @param length_code_size Byte length of each length code
1170 * @param src_data NAL units prefixed with length codes.
1171 * @param src_size Length of buffer, excluding any padding.
1172 * @param dst_data Must be zeroed before calling this function.
1173 * Contains the copied NAL units prefixed with
1174 * start codes when the function returns
1176 * @param dst_size Length of dst_data
1177 * @return 0 on success
1178 * AVERROR_INVALIDDATA if length_code_size is invalid
1179 * AVERROR_BUFFER_TOO_SMALL if dst_data is too small
1180 * or if a length_code in src_data specifies data beyond
1181 * the end of its buffer.
1183 static int copy_replace_length_codes(
1184 AVCodecContext *avctx,
1185 size_t length_code_size,
1186 CMSampleBufferRef sample_buffer,
1190 size_t src_size = CMSampleBufferGetTotalSampleSize(sample_buffer);
1191 size_t remaining_src_size = src_size;
1192 size_t remaining_dst_size = dst_size;
1193 size_t src_offset = 0;
1195 uint8_t size_buf[4];
1196 CMBlockBufferRef block = CMSampleBufferGetDataBuffer(sample_buffer);
1198 if (length_code_size > 4) {
1199 return AVERROR_INVALIDDATA;
1202 while (remaining_src_size > 0) {
1203 size_t curr_src_len;
1204 size_t curr_dst_len;
1210 status = CMBlockBufferCopyDataBytes(block,
1215 av_log(avctx, AV_LOG_ERROR, "Cannot copy length: %d\n", status);
1216 return AVERROR_EXTERNAL;
1219 for (i = 0; i < length_code_size; i++) {
1221 box_len |= size_buf[i];
1224 curr_src_len = box_len + length_code_size;
1225 curr_dst_len = box_len + sizeof(start_code);
1227 if (remaining_src_size < curr_src_len) {
1228 return AVERROR_BUFFER_TOO_SMALL;
1231 if (remaining_dst_size < curr_dst_len) {
1232 return AVERROR_BUFFER_TOO_SMALL;
1235 dst_box = dst_data + sizeof(start_code);
1237 memcpy(dst_data, start_code, sizeof(start_code));
1238 status = CMBlockBufferCopyDataBytes(block,
1239 src_offset + length_code_size,
1244 av_log(avctx, AV_LOG_ERROR, "Cannot copy data: %d\n", status);
1245 return AVERROR_EXTERNAL;
1248 src_offset += curr_src_len;
1249 dst_data += curr_dst_len;
1251 remaining_src_size -= curr_src_len;
1252 remaining_dst_size -= curr_dst_len;
1258 static int vtenc_cm_to_avpacket(
1259 AVCodecContext *avctx,
1260 CMSampleBufferRef sample_buffer,
1263 VTEncContext *vtctx = avctx->priv_data;
1268 size_t length_code_size;
1269 size_t header_size = 0;
1271 size_t out_buf_size;
1273 int64_t time_base_num;
1277 CMVideoFormatDescriptionRef vid_fmt;
1280 vtenc_get_frame_info(sample_buffer, &is_key_frame);
1281 status = get_length_code_size(avctx, sample_buffer, &length_code_size);
1282 if (status) return status;
1284 add_header = is_key_frame && !(avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER);
1287 vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer);
1289 av_log(avctx, AV_LOG_ERROR, "Cannot get format description.\n");
1290 return AVERROR_EXTERNAL;
1293 int status = get_params_size(avctx, vid_fmt, &header_size);
1294 if (status) return status;
1297 status = count_nalus(length_code_size, sample_buffer, &nalu_count);
1301 in_buf_size = CMSampleBufferGetTotalSampleSize(sample_buffer);
1302 out_buf_size = header_size +
1304 nalu_count * ((int)sizeof(start_code) - (int)length_code_size);
1306 status = ff_alloc_packet2(avctx, pkt, out_buf_size, out_buf_size);
1311 status = copy_param_sets(avctx, vid_fmt, pkt->data, out_buf_size);
1312 if(status) return status;
1315 status = copy_replace_length_codes(
1319 pkt->data + header_size,
1320 pkt->size - header_size
1324 av_log(avctx, AV_LOG_ERROR, "Error copying packet data: %d", status);
1329 pkt->flags |= AV_PKT_FLAG_KEY;
1332 pts = CMSampleBufferGetPresentationTimeStamp(sample_buffer);
1333 dts = CMSampleBufferGetDecodeTimeStamp (sample_buffer);
1335 if (CMTIME_IS_INVALID(dts)) {
1336 if (!vtctx->has_b_frames) {
1339 av_log(avctx, AV_LOG_ERROR, "DTS is invalid.\n");
1340 return AVERROR_EXTERNAL;
1344 dts_delta = vtctx->dts_delta >= 0 ? vtctx->dts_delta : 0;
1345 time_base_num = avctx->time_base.num;
1346 pkt->pts = pts.value / time_base_num;
1347 pkt->dts = dts.value / time_base_num - dts_delta;
1348 pkt->size = out_buf_size;
1354 * contiguous_buf_size is 0 if not contiguous, and the size of the buffer
1355 * containing all planes if so.
1357 static int get_cv_pixel_info(
1358 AVCodecContext *avctx,
1359 const AVFrame *frame,
1365 size_t *contiguous_buf_size)
1367 VTEncContext *vtctx = avctx->priv_data;
1368 int av_format = frame->format;
1369 int av_color_range = av_frame_get_color_range(frame);
1374 status = get_cv_pixel_format(avctx, av_format, av_color_range, color, &range_guessed);
1378 "Could not get pixel format for color format '%s' range '%s'.\n",
1379 av_get_pix_fmt_name(av_format),
1380 av_color_range > AVCOL_RANGE_UNSPECIFIED &&
1381 av_color_range < AVCOL_RANGE_NB ?
1382 av_color_range_name(av_color_range) :
1385 return AVERROR(EINVAL);
1388 if (range_guessed) {
1389 if (!vtctx->warned_color_range) {
1390 vtctx->warned_color_range = true;
1393 "Color range not set for %s. Using MPEG range.\n",
1394 av_get_pix_fmt_name(av_format));
1397 av_log(avctx, AV_LOG_WARNING, "");
1400 switch (av_format) {
1401 case AV_PIX_FMT_NV12:
1404 widths [0] = avctx->width;
1405 heights[0] = avctx->height;
1406 strides[0] = frame ? frame->linesize[0] : avctx->width;
1408 widths [1] = (avctx->width + 1) / 2;
1409 heights[1] = (avctx->height + 1) / 2;
1410 strides[1] = frame ? frame->linesize[1] : (avctx->width + 1) & -2;
1413 case AV_PIX_FMT_YUV420P:
1416 widths [0] = avctx->width;
1417 heights[0] = avctx->height;
1418 strides[0] = frame ? frame->linesize[0] : avctx->width;
1420 widths [1] = (avctx->width + 1) / 2;
1421 heights[1] = (avctx->height + 1) / 2;
1422 strides[1] = frame ? frame->linesize[1] : (avctx->width + 1) / 2;
1424 widths [2] = (avctx->width + 1) / 2;
1425 heights[2] = (avctx->height + 1) / 2;
1426 strides[2] = frame ? frame->linesize[2] : (avctx->width + 1) / 2;
1433 "Could not get frame format info for color %d range %d.\n",
1437 return AVERROR(EINVAL);
1440 *contiguous_buf_size = 0;
1441 for (i = 0; i < *plane_count; i++) {
1442 if (i < *plane_count - 1 &&
1443 frame->data[i] + strides[i] * heights[i] != frame->data[i + 1]) {
1444 *contiguous_buf_size = 0;
1448 *contiguous_buf_size += strides[i] * heights[i];
1454 #if !TARGET_OS_IPHONE
1455 //Not used on iOS - frame is always copied.
1456 static void free_avframe(
1461 const void *plane_addresses[])
1463 AVFrame *frame = release_ctx;
1464 av_frame_free(&frame);
1467 //Not used on OSX - frame is never copied.
1468 static int copy_avframe_to_pixel_buffer(AVCodecContext *avctx,
1469 const AVFrame *frame,
1470 CVPixelBufferRef cv_img,
1471 const size_t *plane_strides,
1472 const size_t *plane_rows)
1484 status = CVPixelBufferLockBaseAddress(cv_img, 0);
1489 "Error: Could not lock base address of CVPixelBuffer: %d.\n",
1494 if (CVPixelBufferIsPlanar(cv_img)) {
1495 plane_count = CVPixelBufferGetPlaneCount(cv_img);
1496 for (i = 0; frame->data[i]; i++) {
1497 if (i == plane_count) {
1498 CVPixelBufferUnlockBaseAddress(cv_img, 0);
1501 "Error: different number of planes in AVFrame and CVPixelBuffer.\n"
1504 return AVERROR_EXTERNAL;
1507 dst_addr = (uint8_t*)CVPixelBufferGetBaseAddressOfPlane(cv_img, i);
1508 src_addr = (uint8_t*)frame->data[i];
1509 dst_stride = CVPixelBufferGetBytesPerRowOfPlane(cv_img, i);
1510 src_stride = plane_strides[i];
1511 rows = plane_rows[i];
1513 if (dst_stride == src_stride) {
1514 memcpy(dst_addr, src_addr, src_stride * rows);
1516 copy_bytes = dst_stride < src_stride ? dst_stride : src_stride;
1518 for (j = 0; j < rows; j++) {
1519 memcpy(dst_addr + j * dst_stride, src_addr + j * src_stride, copy_bytes);
1524 if (frame->data[1]) {
1525 CVPixelBufferUnlockBaseAddress(cv_img, 0);
1528 "Error: different number of planes in AVFrame and non-planar CVPixelBuffer.\n"
1531 return AVERROR_EXTERNAL;
1534 dst_addr = (uint8_t*)CVPixelBufferGetBaseAddress(cv_img);
1535 src_addr = (uint8_t*)frame->data[0];
1536 dst_stride = CVPixelBufferGetBytesPerRow(cv_img);
1537 src_stride = plane_strides[0];
1538 rows = plane_rows[0];
1540 if (dst_stride == src_stride) {
1541 memcpy(dst_addr, src_addr, src_stride * rows);
1543 copy_bytes = dst_stride < src_stride ? dst_stride : src_stride;
1545 for (j = 0; j < rows; j++) {
1546 memcpy(dst_addr + j * dst_stride, src_addr + j * src_stride, copy_bytes);
1551 status = CVPixelBufferUnlockBaseAddress(cv_img, 0);
1553 av_log(avctx, AV_LOG_ERROR, "Error: Could not unlock CVPixelBuffer base address: %d.\n", status);
1554 return AVERROR_EXTERNAL;
1559 #endif //!TARGET_OS_IPHONE
1561 static int create_cv_pixel_buffer(AVCodecContext *avctx,
1562 const AVFrame *frame,
1563 CVPixelBufferRef *cv_img)
1567 size_t widths [AV_NUM_DATA_POINTERS];
1568 size_t heights[AV_NUM_DATA_POINTERS];
1569 size_t strides[AV_NUM_DATA_POINTERS];
1571 size_t contiguous_buf_size;
1572 #if TARGET_OS_IPHONE
1573 CVPixelBufferPoolRef pix_buf_pool;
1574 VTEncContext* vtctx = avctx->priv_data;
1576 CFMutableDictionaryRef pix_buf_attachments = CFDictionaryCreateMutable(
1577 kCFAllocatorDefault,
1579 &kCFCopyStringDictionaryKeyCallBacks,
1580 &kCFTypeDictionaryValueCallBacks);
1582 if (!pix_buf_attachments) return AVERROR(ENOMEM);
1585 if (avctx->pix_fmt == AV_PIX_FMT_VIDEOTOOLBOX) {
1586 av_assert0(frame->format == AV_PIX_FMT_VIDEOTOOLBOX);
1588 *cv_img = (CVPixelBufferRef)frame->data[3];
1589 av_assert0(*cv_img);
1595 memset(widths, 0, sizeof(widths));
1596 memset(heights, 0, sizeof(heights));
1597 memset(strides, 0, sizeof(strides));
1599 status = get_cv_pixel_info(
1607 &contiguous_buf_size
1614 "Error: Cannot convert format %d color_range %d: %d\n",
1616 av_frame_get_color_range(frame),
1620 return AVERROR_EXTERNAL;
1623 #if TARGET_OS_IPHONE
1624 pix_buf_pool = VTCompressionSessionGetPixelBufferPool(vtctx->session);
1625 if (!pix_buf_pool) {
1626 av_log(avctx, AV_LOG_ERROR, "Could not get pixel buffer pool.\n");
1627 return AVERROR_EXTERNAL;
1630 status = CVPixelBufferPoolCreatePixelBuffer(NULL,
1636 av_log(avctx, AV_LOG_ERROR, "Could not create pixel buffer from pool: %d.\n", status);
1637 return AVERROR_EXTERNAL;
1640 status = copy_avframe_to_pixel_buffer(avctx, frame, *cv_img, strides, heights);
1647 AVFrame *enc_frame = av_frame_alloc();
1648 if (!enc_frame) return AVERROR(ENOMEM);
1650 status = av_frame_ref(enc_frame, frame);
1652 av_frame_free(&enc_frame);
1656 status = CVPixelBufferCreateWithPlanarBytes(
1657 kCFAllocatorDefault,
1662 contiguous_buf_size,
1664 (void **)enc_frame->data,
1674 add_color_attr(avctx, pix_buf_attachments);
1675 CVBufferSetAttachments(*cv_img, pix_buf_attachments, kCVAttachmentMode_ShouldPropagate);
1676 CFRelease(pix_buf_attachments);
1679 av_log(avctx, AV_LOG_ERROR, "Error: Could not create CVPixelBuffer: %d\n", status);
1680 return AVERROR_EXTERNAL;
1687 static int create_encoder_dict_h264(const AVFrame *frame,
1688 CFDictionaryRef* dict_out)
1690 CFDictionaryRef dict = NULL;
1691 if (frame->pict_type == AV_PICTURE_TYPE_I) {
1692 const void *keys[] = { kVTEncodeFrameOptionKey_ForceKeyFrame };
1693 const void *vals[] = { kCFBooleanTrue };
1695 dict = CFDictionaryCreate(NULL, keys, vals, 1, NULL, NULL);
1696 if(!dict) return AVERROR(ENOMEM);
1703 static int vtenc_send_frame(AVCodecContext *avctx,
1704 VTEncContext *vtctx,
1705 const AVFrame *frame)
1708 CFDictionaryRef frame_dict;
1709 CVPixelBufferRef cv_img = NULL;
1710 int status = create_cv_pixel_buffer(avctx, frame, &cv_img);
1712 if (status) return status;
1714 status = create_encoder_dict_h264(frame, &frame_dict);
1720 time = CMTimeMake(frame->pts * avctx->time_base.num, avctx->time_base.den);
1721 status = VTCompressionSessionEncodeFrame(
1731 if (frame_dict) CFRelease(frame_dict);
1735 av_log(avctx, AV_LOG_ERROR, "Error: cannot encode frame: %d\n", status);
1736 return AVERROR_EXTERNAL;
1742 static av_cold int vtenc_frame(
1743 AVCodecContext *avctx,
1745 const AVFrame *frame,
1748 VTEncContext *vtctx = avctx->priv_data;
1751 CMSampleBufferRef buf = NULL;
1754 status = vtenc_send_frame(avctx, vtctx, frame);
1757 status = AVERROR_EXTERNAL;
1761 if (vtctx->frame_ct_in == 0) {
1762 vtctx->first_pts = frame->pts;
1763 } else if(vtctx->frame_ct_in == 1 && vtctx->has_b_frames) {
1764 vtctx->dts_delta = frame->pts - vtctx->first_pts;
1767 vtctx->frame_ct_in++;
1768 } else if(!vtctx->flushing) {
1769 vtctx->flushing = true;
1771 status = VTCompressionSessionCompleteFrames(vtctx->session,
1775 av_log(avctx, AV_LOG_ERROR, "Error flushing frames: %d\n", status);
1776 status = AVERROR_EXTERNAL;
1782 get_frame = vtctx->dts_delta >= 0 || !frame;
1788 status = vtenc_q_pop(vtctx, !frame, &buf);
1789 if (status) goto end_nopkt;
1790 if (!buf) goto end_nopkt;
1792 status = vtenc_cm_to_avpacket(avctx, buf, pkt);
1794 if (status) goto end_nopkt;
1800 av_packet_unref(pkt);
1804 static int vtenc_populate_extradata(AVCodecContext *avctx,
1805 CMVideoCodecType codec_type,
1806 CFStringRef profile_level,
1807 CFNumberRef gamma_level,
1808 CFDictionaryRef enc_info,
1809 CFDictionaryRef pixel_buffer_info)
1811 VTEncContext *vtctx = avctx->priv_data;
1812 AVFrame *frame = av_frame_alloc();
1813 int y_size = avctx->width * avctx->height;
1814 int chroma_size = (avctx->width / 2) * (avctx->height / 2);
1815 CMSampleBufferRef buf = NULL;
1819 return AVERROR(ENOMEM);
1821 frame->buf[0] = av_buffer_alloc(y_size + 2 * chroma_size);
1824 status = AVERROR(ENOMEM);
1828 status = vtenc_create_encoder(avctx,
1838 frame->data[0] = frame->buf[0]->data;
1839 memset(frame->data[0], 0, y_size);
1841 frame->data[1] = frame->buf[0]->data + y_size;
1842 memset(frame->data[1], 128, chroma_size);
1845 if (avctx->pix_fmt == AV_PIX_FMT_YUV420P) {
1846 frame->data[2] = frame->buf[0]->data + y_size + chroma_size;
1847 memset(frame->data[2], 128, chroma_size);
1850 frame->linesize[0] = avctx->width;
1852 if (avctx->pix_fmt == AV_PIX_FMT_YUV420P) {
1853 frame->linesize[1] =
1854 frame->linesize[2] = (avctx->width + 1) / 2;
1856 frame->linesize[1] = (avctx->width + 1) / 2;
1859 frame->format = avctx->pix_fmt;
1860 frame->width = avctx->width;
1861 frame->height = avctx->height;
1862 av_frame_set_colorspace(frame, avctx->colorspace);
1863 av_frame_set_color_range(frame, avctx->color_range);
1864 frame->color_trc = avctx->color_trc;
1865 frame->color_primaries = avctx->color_primaries;
1868 status = vtenc_send_frame(avctx, vtctx, frame);
1870 av_log(avctx, AV_LOG_ERROR, "Error sending frame: %d\n", status);
1874 av_log(avctx, AV_LOG_INFO, "Completing\n");
1876 //Populates extradata - output frames are flushed and param sets are available.
1877 status = VTCompressionSessionCompleteFrames(vtctx->session,
1881 av_log(avctx, AV_LOG_INFO, "Completed: %d\n", status);
1885 status = vtenc_q_pop(vtctx, 0, &buf);
1887 av_log(avctx, AV_LOG_ERROR, "popping: %d\n", status);
1897 CFRelease(vtctx->session);
1899 vtctx->session = NULL;
1900 vtctx->frame_ct_out = 0;
1902 av_frame_unref(frame);
1903 av_frame_free(&frame);
1905 av_log(avctx, AV_LOG_INFO, "status %d ed %p size %d\n", status, avctx->extradata, avctx->extradata_size);
1907 av_assert0(status != 0 || (avctx->extradata && avctx->extradata_size > 0));
1912 static av_cold int vtenc_close(AVCodecContext *avctx)
1914 VTEncContext *vtctx = avctx->priv_data;
1916 if(!vtctx->session) return 0;
1918 pthread_cond_destroy(&vtctx->cv_sample_sent);
1919 pthread_mutex_destroy(&vtctx->lock);
1920 CFRelease(vtctx->session);
1921 vtctx->session = NULL;
1923 if (vtctx->color_primaries) {
1924 CFRelease(vtctx->color_primaries);
1925 vtctx->color_primaries = NULL;
1928 if (vtctx->transfer_function) {
1929 CFRelease(vtctx->transfer_function);
1930 vtctx->transfer_function = NULL;
1933 if (vtctx->ycbcr_matrix) {
1934 CFRelease(vtctx->ycbcr_matrix);
1935 vtctx->ycbcr_matrix = NULL;
1941 static const enum AVPixelFormat pix_fmts[] = {
1942 AV_PIX_FMT_VIDEOTOOLBOX,
1948 #define OFFSET(x) offsetof(VTEncContext, x)
1949 #define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM
1950 static const AVOption options[] = {
1951 { "profile", "Profile", OFFSET(profile), AV_OPT_TYPE_INT, { .i64 = H264_PROF_AUTO }, H264_PROF_AUTO, H264_PROF_COUNT, VE, "profile" },
1952 { "baseline", "Baseline Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_BASELINE }, INT_MIN, INT_MAX, VE, "profile" },
1953 { "main", "Main Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_MAIN }, INT_MIN, INT_MAX, VE, "profile" },
1954 { "high", "High Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_HIGH }, INT_MIN, INT_MAX, VE, "profile" },
1956 { "level", "Level", OFFSET(level), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 52, VE, "level" },
1957 { "1.3", "Level 1.3, only available with Baseline Profile", 0, AV_OPT_TYPE_CONST, { .i64 = 13 }, INT_MIN, INT_MAX, VE, "level" },
1958 { "3.0", "Level 3.0", 0, AV_OPT_TYPE_CONST, { .i64 = 30 }, INT_MIN, INT_MAX, VE, "level" },
1959 { "3.1", "Level 3.1", 0, AV_OPT_TYPE_CONST, { .i64 = 31 }, INT_MIN, INT_MAX, VE, "level" },
1960 { "3.2", "Level 3.2", 0, AV_OPT_TYPE_CONST, { .i64 = 32 }, INT_MIN, INT_MAX, VE, "level" },
1961 { "4.0", "Level 4.0", 0, AV_OPT_TYPE_CONST, { .i64 = 40 }, INT_MIN, INT_MAX, VE, "level" },
1962 { "4.1", "Level 4.1", 0, AV_OPT_TYPE_CONST, { .i64 = 41 }, INT_MIN, INT_MAX, VE, "level" },
1963 { "4.2", "Level 4.2", 0, AV_OPT_TYPE_CONST, { .i64 = 42 }, INT_MIN, INT_MAX, VE, "level" },
1964 { "5.0", "Level 5.0", 0, AV_OPT_TYPE_CONST, { .i64 = 50 }, INT_MIN, INT_MAX, VE, "level" },
1965 { "5.1", "Level 5.1", 0, AV_OPT_TYPE_CONST, { .i64 = 51 }, INT_MIN, INT_MAX, VE, "level" },
1966 { "5.2", "Level 5.2", 0, AV_OPT_TYPE_CONST, { .i64 = 52 }, INT_MIN, INT_MAX, VE, "level" },
1968 { "allow_sw", "Allow software encoding", OFFSET(allow_sw), AV_OPT_TYPE_BOOL,
1969 { .i64 = 0 }, 0, 1, VE },
1971 { "coder", "Entropy coding", OFFSET(entropy), AV_OPT_TYPE_INT, { .i64 = VT_ENTROPY_NOT_SET }, VT_ENTROPY_NOT_SET, VT_CABAC, VE, "coder" },
1972 { "cavlc", "CAVLC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CAVLC }, INT_MIN, INT_MAX, VE, "coder" },
1973 { "vlc", "CAVLC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CAVLC }, INT_MIN, INT_MAX, VE, "coder" },
1974 { "cabac", "CABAC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CABAC }, INT_MIN, INT_MAX, VE, "coder" },
1975 { "ac", "CABAC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CABAC }, INT_MIN, INT_MAX, VE, "coder" },
1977 { "realtime", "Hint that encoding should happen in real-time if not faster (e.g. capturing from camera).",
1978 OFFSET(realtime), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE },
1980 { "frames_before", "Other frames will come before the frames in this session. This helps smooth concatenation issues.",
1981 OFFSET(frames_before), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE },
1982 { "frames_after", "Other frames will come after the frames in this session. This helps smooth concatenation issues.",
1983 OFFSET(frames_after), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE },
1988 static const AVClass h264_videotoolbox_class = {
1989 .class_name = "h264_videotoolbox",
1990 .item_name = av_default_item_name,
1992 .version = LIBAVUTIL_VERSION_INT,
1995 AVCodec ff_h264_videotoolbox_encoder = {
1996 .name = "h264_videotoolbox",
1997 .long_name = NULL_IF_CONFIG_SMALL("VideoToolbox H.264 Encoder"),
1998 .type = AVMEDIA_TYPE_VIDEO,
1999 .id = AV_CODEC_ID_H264,
2000 .priv_data_size = sizeof(VTEncContext),
2001 .pix_fmts = pix_fmts,
2003 .encode2 = vtenc_frame,
2004 .close = vtenc_close,
2005 .capabilities = AV_CODEC_CAP_DELAY,
2006 .priv_class = &h264_videotoolbox_class,
2007 .caps_internal = FF_CODEC_CAP_INIT_THREADSAFE |
2008 FF_CODEC_CAP_INIT_CLEANUP,