avcodec/proresenc_aw : simplify frame flags

[ffmpeg] / libavcodec / videotoolboxenc.c

diff --git a/libavcodec/videotoolboxenc.c b/libavcodec/videotoolboxenc.c
index 1060055ba5ceee759d36e8b002c76556a994d09b..50aba2d87f9c6bd8ca6e27d5c8bfca7d1940586c 100644 (file)
--- a/libavcodec/videotoolboxenc.c
+++ b/libavcodec/videotoolboxenc.c
@@ -41,7 +41,7 @@ enum { kCMVideoCodecType_HEVC = 'hvc1' };
 
 typedef OSStatus (*getParameterSetAtIndex)(CMFormatDescriptionRef videoDesc,
                                            size_t parameterSetIndex,
-                                           const uint8_t * _Nullable *parameterSetPointerOut,
+                                           const uint8_t **parameterSetPointerOut,
                                            size_t *parameterSetSizeOut,
                                            size_t *parameterSetCountOut,
                                            int *NALUnitHeaderLengthOut);
@@ -1017,7 +1017,7 @@ static int vtenc_create_encoder(AVCodecContext   *avctx,
         return AVERROR_EXTERNAL;
     }
 
-    if (vtctx->codec_id == AV_CODEC_ID_H264) {
+    if (vtctx->codec_id == AV_CODEC_ID_H264 && max_rate > 0) {
         // kVTCompressionPropertyKey_DataRateLimits is not available for HEVC
         bytes_per_second_value = max_rate >> 3;
         bytes_per_second = CFNumberCreate(kCFAllocatorDefault,
@@ -1058,7 +1058,10 @@ static int vtenc_create_encoder(AVCodecContext   *avctx,
             av_log(avctx, AV_LOG_ERROR, "Error setting max bitrate property: %d\n", status);
             return AVERROR_EXTERNAL;
         }
+    }
 
+    if (vtctx->codec_id == AV_CODEC_ID_H264) {
+        // kVTCompressionPropertyKey_ProfileLevel is not available for HEVC
         if (profile_level) {
             status = VTSessionSetProperty(vtctx->session,
                                         kVTCompressionPropertyKey_ProfileLevel,
@@ -2175,8 +2178,27 @@ static int create_cv_pixel_buffer(AVCodecContext   *avctx,
 #if TARGET_OS_IPHONE
     pix_buf_pool = VTCompressionSessionGetPixelBufferPool(vtctx->session);
     if (!pix_buf_pool) {
-        av_log(avctx, AV_LOG_ERROR, "Could not get pixel buffer pool.\n");
-        return AVERROR_EXTERNAL;
+        /* On iOS, the VT session is invalidated when the APP switches from
+         * foreground to background and vice versa. Fetch the actual error code
+         * of the VT session to detect that case and restart the VT session
+         * accordingly. */
+        OSStatus vtstatus;
+
+        vtstatus = VTCompressionSessionPrepareToEncodeFrames(vtctx->session);
+        if (vtstatus == kVTInvalidSessionErr) {
+            CFRelease(vtctx->session);
+            vtctx->session = NULL;
+            status = vtenc_configure_encoder(avctx);
+            if (status == 0)
+                pix_buf_pool = VTCompressionSessionGetPixelBufferPool(vtctx->session);
+        }
+        if (!pix_buf_pool) {
+            av_log(avctx, AV_LOG_ERROR, "Could not get pixel buffer pool.\n");
+            return AVERROR_EXTERNAL;
+        }
+        else
+            av_log(avctx, AV_LOG_WARNING, "VT session restarted because of a "
+                   "kVTInvalidSessionErr error.\n");
     }
 
     status = CVPixelBufferPoolCreatePixelBuffer(NULL,