]> git.sesse.net Git - ffmpeg/blobdiff - libavcodec/vdpau.c
mpeg12dec: avoid signed overflow in bitrate calculation
[ffmpeg] / libavcodec / vdpau.c
index 1d68e70b478e589c07820076d86767d60706efd7..d3cb188d7422dc0c61890d4d766dde37eff9d9bb 100644 (file)
  */
 
 #include <limits.h>
-#include "libavutil/avassert.h"
+
 #include "avcodec.h"
 #include "internal.h"
-#include "h264.h"
+#include "h264dec.h"
 #include "vc1.h"
-
-#undef NDEBUG
-#include <assert.h>
-
 #include "vdpau.h"
 #include "vdpau_internal.h"
 
@@ -64,6 +60,46 @@ static int vdpau_error(VdpStatus status)
     }
 }
 
+int av_vdpau_get_surface_parameters(AVCodecContext *avctx,
+                                    VdpChromaType *type,
+                                    uint32_t *width, uint32_t *height)
+{
+    VdpChromaType t;
+    uint32_t w = avctx->coded_width;
+    uint32_t h = avctx->coded_height;
+
+    /* See <vdpau/vdpau.h> for per-type alignment constraints. */
+    switch (avctx->sw_pix_fmt) {
+    case AV_PIX_FMT_YUV420P:
+    case AV_PIX_FMT_YUVJ420P:
+        t = VDP_CHROMA_TYPE_420;
+        w = (w + 1) & ~1;
+        h = (h + 3) & ~3;
+        break;
+    case AV_PIX_FMT_YUV422P:
+    case AV_PIX_FMT_YUVJ422P:
+        t = VDP_CHROMA_TYPE_422;
+        w = (w + 1) & ~1;
+        h = (h + 1) & ~1;
+        break;
+    case AV_PIX_FMT_YUV444P:
+    case AV_PIX_FMT_YUVJ444P:
+        t = VDP_CHROMA_TYPE_444;
+        h = (h + 1) & ~1;
+        break;
+    default:
+        return AVERROR(ENOSYS);
+    }
+
+    if (type)
+        *type = t;
+    if (width)
+        *width = w;
+    if (height)
+        *height = h;
+    return 0;
+}
+
 int ff_vdpau_common_init(AVCodecContext *avctx, VdpDecoderProfile profile,
                          int level)
 {
@@ -76,9 +112,9 @@ int ff_vdpau_common_init(AVCodecContext *avctx, VdpDecoderProfile profile,
     VdpStatus status;
     VdpBool supported;
     uint32_t max_level, max_mb, max_width, max_height;
-    /* See vdpau/vdpau.h for alignment constraints. */
-    uint32_t width  = (avctx->coded_width + 1) & ~1;
-    uint32_t height = (avctx->coded_height + 3) & ~3;
+    VdpChromaType type;
+    uint32_t width;
+    uint32_t height;
 
     vdctx->width            = UINT32_MAX;
     vdctx->height           = UINT32_MAX;
@@ -99,6 +135,13 @@ int ff_vdpau_common_init(AVCodecContext *avctx, VdpDecoderProfile profile,
     else if (level < 0)
         return AVERROR(ENOTSUP);
 
+    if (av_vdpau_get_surface_parameters(avctx, &type, &width, &height))
+        return AVERROR(ENOSYS);
+
+    if (!(hwctx->flags & AV_HWACCEL_FLAG_ALLOW_HIGH_DEPTH) &&
+        type != VDP_CHROMA_TYPE_420)
+        return AVERROR(ENOSYS);
+
     status = vdctx->get_proc_address(vdctx->device,
                                      VDP_FUNC_ID_VIDEO_SURFACE_QUERY_CAPABILITIES,
                                      &func);
@@ -107,7 +150,7 @@ int ff_vdpau_common_init(AVCodecContext *avctx, VdpDecoderProfile profile,
     else
         surface_query_caps = func;
 
-    status = surface_query_caps(vdctx->device, VDP_CHROMA_TYPE_420, &supported,
+    status = surface_query_caps(vdctx->device, type, &supported,
                                 &max_width, &max_height);
     if (status != VDP_STATUS_OK)
         return vdpau_error(status);
@@ -125,6 +168,14 @@ int ff_vdpau_common_init(AVCodecContext *avctx, VdpDecoderProfile profile,
 
     status = decoder_query_caps(vdctx->device, profile, &supported, &max_level,
                                 &max_mb, &max_width, &max_height);
+#ifdef VDP_DECODER_PROFILE_H264_CONSTRAINED_BASELINE
+    if ((status != VDP_STATUS_OK || supported != VDP_TRUE) && profile == VDP_DECODER_PROFILE_H264_CONSTRAINED_BASELINE) {
+        profile = VDP_DECODER_PROFILE_H264_MAIN;
+        status = decoder_query_caps(vdctx->device, profile, &supported,
+                                    &max_level, &max_mb,
+                                    &max_width, &max_height);
+    }
+#endif
     if (status != VDP_STATUS_OK)
         return vdpau_error(status);
 
@@ -216,7 +267,7 @@ int ff_vdpau_common_end_frame(AVCodecContext *avctx, AVFrame *frame,
     if (val < 0)
         return val;
 
-    status = vdctx->render(vdctx->decoder, surf, (void *)&pic_ctx->info,
+    status = vdctx->render(vdctx->decoder, surf, &pic_ctx->info,
                            pic_ctx->bitstream_buffers_used,
                            pic_ctx->bitstream_buffers);
 
@@ -224,7 +275,7 @@ int ff_vdpau_common_end_frame(AVCodecContext *avctx, AVFrame *frame,
     return vdpau_error(status);
 }
 
-#if CONFIG_H263_VDPAU_HWACCEL  || CONFIG_MPEG1_VDPAU_HWACCEL || \
+#if CONFIG_MPEG1_VDPAU_HWACCEL || \
     CONFIG_MPEG2_VDPAU_HWACCEL || CONFIG_MPEG4_VDPAU_HWACCEL || \
     CONFIG_VC1_VDPAU_HWACCEL   || CONFIG_WMV3_VDPAU_HWACCEL
 int ff_vdpau_mpeg_end_frame(AVCodecContext *avctx)
@@ -262,48 +313,54 @@ int ff_vdpau_add_buffer(struct vdpau_picture_context *pic_ctx,
     return 0;
 }
 
+#if FF_API_VDPAU_PROFILE
 int av_vdpau_get_profile(AVCodecContext *avctx, VdpDecoderProfile *profile)
 {
-#define PROFILE(prof)       \
-do {                        \
-    *profile = prof;        \
-    return 0;               \
+#define PROFILE(prof)                      \
+do {                                       \
+    *profile = VDP_DECODER_PROFILE_##prof; \
+    return 0;                              \
 } while (0)
 
     switch (avctx->codec_id) {
-    case AV_CODEC_ID_MPEG1VIDEO:               PROFILE(VDP_DECODER_PROFILE_MPEG1);
+    case AV_CODEC_ID_MPEG1VIDEO:               PROFILE(MPEG1);
     case AV_CODEC_ID_MPEG2VIDEO:
         switch (avctx->profile) {
-        case FF_PROFILE_MPEG2_MAIN:            PROFILE(VDP_DECODER_PROFILE_MPEG2_MAIN);
-        case FF_PROFILE_MPEG2_SIMPLE:          PROFILE(VDP_DECODER_PROFILE_MPEG2_SIMPLE);
+        case FF_PROFILE_MPEG2_MAIN:            PROFILE(MPEG2_MAIN);
+        case FF_PROFILE_MPEG2_SIMPLE:          PROFILE(MPEG2_SIMPLE);
         default:                               return AVERROR(EINVAL);
         }
-    case AV_CODEC_ID_H263:                     PROFILE(VDP_DECODER_PROFILE_MPEG4_PART2_ASP);
+    case AV_CODEC_ID_H263:                     PROFILE(MPEG4_PART2_ASP);
     case AV_CODEC_ID_MPEG4:
         switch (avctx->profile) {
-        case FF_PROFILE_MPEG4_SIMPLE:          PROFILE(VDP_DECODER_PROFILE_MPEG4_PART2_SP);
-        case FF_PROFILE_MPEG4_ADVANCED_SIMPLE: PROFILE(VDP_DECODER_PROFILE_MPEG4_PART2_ASP);
+        case FF_PROFILE_MPEG4_SIMPLE:          PROFILE(MPEG4_PART2_SP);
+        case FF_PROFILE_MPEG4_ADVANCED_SIMPLE: PROFILE(MPEG4_PART2_ASP);
         default:                               return AVERROR(EINVAL);
         }
     case AV_CODEC_ID_H264:
         switch (avctx->profile & ~FF_PROFILE_H264_INTRA) {
+        case FF_PROFILE_H264_BASELINE:         PROFILE(H264_BASELINE);
         case FF_PROFILE_H264_CONSTRAINED_BASELINE:
-        case FF_PROFILE_H264_BASELINE:         PROFILE(VDP_DECODER_PROFILE_H264_BASELINE);
-        case FF_PROFILE_H264_MAIN:             PROFILE(VDP_DECODER_PROFILE_H264_MAIN);
-        case FF_PROFILE_H264_HIGH:             PROFILE(VDP_DECODER_PROFILE_H264_HIGH);
+        case FF_PROFILE_H264_MAIN:             PROFILE(H264_MAIN);
+        case FF_PROFILE_H264_HIGH:             PROFILE(H264_HIGH);
+#ifdef VDP_DECODER_PROFILE_H264_EXTENDED
+        case FF_PROFILE_H264_EXTENDED:         PROFILE(H264_EXTENDED);
+#endif
         default:                               return AVERROR(EINVAL);
         }
     case AV_CODEC_ID_WMV3:
     case AV_CODEC_ID_VC1:
         switch (avctx->profile) {
-        case FF_PROFILE_VC1_SIMPLE:            PROFILE(VDP_DECODER_PROFILE_VC1_SIMPLE);
-        case FF_PROFILE_VC1_MAIN:              PROFILE(VDP_DECODER_PROFILE_VC1_MAIN);
-        case FF_PROFILE_VC1_ADVANCED:          PROFILE(VDP_DECODER_PROFILE_VC1_ADVANCED);
+        case FF_PROFILE_VC1_SIMPLE:            PROFILE(VC1_SIMPLE);
+        case FF_PROFILE_VC1_MAIN:              PROFILE(VC1_MAIN);
+        case FF_PROFILE_VC1_ADVANCED:          PROFILE(VC1_ADVANCED);
         default:                               return AVERROR(EINVAL);
         }
     }
     return AVERROR(EINVAL);
+#undef PROFILE
 }
+#endif /* FF_API_VDPAU_PROFILE */
 
 AVVDPAUContext *av_vdpau_alloc_context(void)
 {
@@ -315,7 +372,7 @@ int av_vdpau_bind_context(AVCodecContext *avctx, VdpDevice device,
 {
     VDPAUHWContext *hwctx;
 
-    if (flags & ~AV_HWACCEL_FLAG_IGNORE_LEVEL)
+    if (flags & ~(AV_HWACCEL_FLAG_IGNORE_LEVEL|AV_HWACCEL_FLAG_ALLOW_HIGH_DEPTH))
         return AVERROR(EINVAL);
 
     if (av_reallocp(&avctx->hwaccel_context, sizeof(*hwctx)))