hwcontext_vulkan: correctly download and upload flipped images

We derive the destination buffer stride from the input stride,
which meant if the image was flipped with a negative stride,
we'd be FFALIGNING a negative number which ends up being huge,
thus making the Vulkan buffer allocation fail and the whole
image transfer fail.

Only found out about this as OpenGL compositors can copy an entire
image with a single call if its flipped, rather than iterate over
each line.

diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c
index e4546f67ca0826611875be1555b80c3a9f60116e..fa53d9d1215efb8eafb4582756b4a8be23610bb7 100644 (file)
--- a/libavutil/hwcontext_vulkan.c
+++ b/libavutil/hwcontext_vulkan.c
@@ -2638,7 +2638,7 @@ static int vulkan_transfer_data_from_mem(AVHWFramesContext *hwfc, AVFrame *dst,
         int h = src->height;
         int p_height = i > 0 ? AV_CEIL_RSHIFT(h, log2_chroma) : h;
 
-        tmp.linesize[i] = src->linesize[i];
+        tmp.linesize[i] = FFABS(src->linesize[i]);
         err = create_buf(dev_ctx, &buf[i], p_height,
                          &tmp.linesize[i], VK_BUFFER_USAGE_TRANSFER_SRC_BIT,
                          VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, NULL, NULL);
@@ -2793,7 +2793,7 @@ static int vulkan_transfer_data_to_mem(AVHWFramesContext *hwfc, AVFrame *dst,
         int h = dst->height;
         int p_height = i > 0 ? AV_CEIL_RSHIFT(h, log2_chroma) : h;
 
-        tmp.linesize[i] = dst->linesize[i];
+        tmp.linesize[i] = FFABS(dst->linesize[i]);
         err = create_buf(dev_ctx, &buf[i], p_height,
                          &tmp.linesize[i], VK_BUFFER_USAGE_TRANSFER_DST_BIT,
                          VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, NULL, NULL);