git.sesse.net Git - ffmpeg/blob - libavfilter/vf_overlay_vulkan.c

   1 /*
   2  * This file is part of FFmpeg.
   3  *
   4  * FFmpeg is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU Lesser General Public
   6  * License as published by the Free Software Foundation; either
   7  * version 2.1 of the License, or (at your option) any later version.
   8  *
   9  * FFmpeg is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * Lesser General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU Lesser General Public
  15  * License along with FFmpeg; if not, write to the Free Software
  16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  17  */
  18
  19 #include "libavutil/opt.h"
  20 #include "vulkan.h"
  21 #include "internal.h"
  22 #include "framesync.h"
  23
  24 #define CGROUPS (int [3]){ 32, 32, 1 }
  25
  26 typedef struct OverlayVulkanContext {
  27     VulkanFilterContext vkctx;
  28
  29     int initialized;
  30     VulkanPipeline *pl;
  31     FFVkExecContext *exec;
  32     FFFrameSync fs;
  33     FFVkBuffer params_buf;
  34
  35     /* Shader updators, must be in the main filter struct */
  36     VkDescriptorImageInfo main_images[3];
  37     VkDescriptorImageInfo overlay_images[3];
  38     VkDescriptorImageInfo output_images[3];
  39     VkDescriptorBufferInfo params_desc;
  40
  41     int overlay_x;
  42     int overlay_y;
  43     int overlay_w;
  44     int overlay_h;
  45 } OverlayVulkanContext;
  46
  47 static const char overlay_noalpha[] = {
  48     C(0, void overlay_noalpha(int i, ivec2 pos)                                )
  49     C(0, {                                                                     )
  50     C(1,     if ((o_offset[i].x <= pos.x) && (o_offset[i].y <= pos.y) &&
  51                  (pos.x < (o_offset[i].x + o_size[i].x)) &&
  52                  (pos.y < (o_offset[i].y + o_size[i].y))) {                    )
  53     C(2,         vec4 res = texture(overlay_img[i], pos - o_offset[i]);        )
  54     C(2,         imageStore(output_img[i], pos, res);                          )
  55     C(1,     } else {                                                          )
  56     C(2,         vec4 res = texture(main_img[i], pos);                         )
  57     C(2,         imageStore(output_img[i], pos, res);                          )
  58     C(1,     }                                                                 )
  59     C(0, }                                                                     )
  60 };
  61
  62 static av_cold int init_filter(AVFilterContext *ctx)
  63 {
  64     int err;
  65     OverlayVulkanContext *s = ctx->priv;
  66     VkSampler *sampler = ff_vk_init_sampler(ctx, 1, VK_FILTER_LINEAR);
  67     if (!sampler)
  68         return AVERROR_EXTERNAL;
  69
  70     s->pl = ff_vk_create_pipeline(ctx);
  71     if (!s->pl)
  72         return AVERROR(ENOMEM);
  73
  74     { /* Create the shader */
  75         const int planes = av_pix_fmt_count_planes(s->vkctx.output_format);
  76
  77         VulkanDescriptorSetBinding desc_i[3] = {
  78             {
  79                 .name       = "main_img",
  80                 .type       = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
  81                 .dimensions = 2,
  82                 .elems      = planes,
  83                 .stages     = VK_SHADER_STAGE_COMPUTE_BIT,
  84                 .updater    = s->main_images,
  85                 .samplers   = DUP_SAMPLER_ARRAY4(*sampler),
  86             },
  87             {
  88                 .name       = "overlay_img",
  89                 .type       = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
  90                 .dimensions = 2,
  91                 .elems      = planes,
  92                 .stages     = VK_SHADER_STAGE_COMPUTE_BIT,
  93                 .updater    = s->overlay_images,
  94                 .samplers   = DUP_SAMPLER_ARRAY4(*sampler),
  95             },
  96             {
  97                 .name       = "output_img",
  98                 .type       = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
  99                 .mem_layout = ff_vk_shader_rep_fmt(s->vkctx.output_format),
 100                 .mem_quali  = "writeonly",
 101                 .dimensions = 2,
 102                 .elems      = planes,
 103                 .stages     = VK_SHADER_STAGE_COMPUTE_BIT,
 104                 .updater    = s->output_images,
 105             },
 106         };
 107
 108         VulkanDescriptorSetBinding desc_b = {
 109             .name        = "params",
 110             .type        = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
 111             .mem_quali   = "readonly",
 112             .mem_layout  = "std430",
 113             .stages      = VK_SHADER_STAGE_COMPUTE_BIT,
 114             .updater     = &s->params_desc,
 115             .buf_content = "ivec2 o_offset[3], o_size[3];",
 116         };
 117
 118         SPIRVShader *shd = ff_vk_init_shader(ctx, s->pl, "overlay_compute",
 119                                              VK_SHADER_STAGE_COMPUTE_BIT);
 120         if (!shd)
 121             return AVERROR(ENOMEM);
 122
 123         ff_vk_set_compute_shader_sizes(ctx, shd, CGROUPS);
 124
 125         RET(ff_vk_add_descriptor_set(ctx, s->pl, shd,  desc_i, 3, 0)); /* set 0 */
 126         RET(ff_vk_add_descriptor_set(ctx, s->pl, shd, &desc_b, 1, 0)); /* set 1 */
 127
 128         GLSLD(   overlay_noalpha                                              );
 129         GLSLC(0, void main()                                                  );
 130         GLSLC(0, {                                                            );
 131         GLSLC(1,     ivec2 pos = ivec2(gl_GlobalInvocationID.xy);             );
 132         GLSLF(1,     int planes = %i;                                  ,planes);
 133         GLSLC(1,     for (int i = 0; i < planes; i++) {                       );
 134         GLSLC(2,         overlay_noalpha(i, pos);                             );
 135         GLSLC(1,     }                                                        );
 136         GLSLC(0, }                                                            );
 137
 138         RET(ff_vk_compile_shader(ctx, shd, "main"));
 139     }
 140
 141     RET(ff_vk_init_pipeline_layout(ctx, s->pl));
 142     RET(ff_vk_init_compute_pipeline(ctx, s->pl));
 143
 144     { /* Create and update buffer */
 145         const AVPixFmtDescriptor *desc;
 146
 147         /* NOTE: std430 requires the same identical struct layout, padding and
 148          * alignment as C, so we're allowed to do this, as this will map
 149          * exactly to what the shader recieves */
 150         struct {
 151             int32_t o_offset[2*3];
 152             int32_t o_size[2*3];
 153         } *par;
 154
 155         err = ff_vk_create_buf(ctx, &s->params_buf,
 156                                sizeof(*par),
 157                                VK_BUFFER_USAGE_STORAGE_BUFFER_BIT,
 158                                VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT);
 159         if (err)
 160             return err;
 161
 162         err = ff_vk_map_buffers(ctx, &s->params_buf, (uint8_t **)&par, 1, 0);
 163         if (err)
 164             return err;
 165
 166         desc = av_pix_fmt_desc_get(s->vkctx.output_format);
 167
 168         par->o_offset[0] = s->overlay_x;
 169         par->o_offset[1] = s->overlay_y;
 170         par->o_offset[2] = par->o_offset[0] >> desc->log2_chroma_w;
 171         par->o_offset[3] = par->o_offset[1] >> desc->log2_chroma_h;
 172         par->o_offset[4] = par->o_offset[0] >> desc->log2_chroma_w;
 173         par->o_offset[5] = par->o_offset[1] >> desc->log2_chroma_h;
 174
 175         par->o_size[0] = s->overlay_w;
 176         par->o_size[1] = s->overlay_h;
 177         par->o_size[2] = par->o_size[0] >> desc->log2_chroma_w;
 178         par->o_size[3] = par->o_size[1] >> desc->log2_chroma_h;
 179         par->o_size[4] = par->o_size[0] >> desc->log2_chroma_w;
 180         par->o_size[5] = par->o_size[1] >> desc->log2_chroma_h;
 181
 182         err = ff_vk_unmap_buffers(ctx, &s->params_buf, 1, 1);
 183         if (err)
 184             return err;
 185
 186         s->params_desc.buffer = s->params_buf.buf;
 187         s->params_desc.range  = VK_WHOLE_SIZE;
 188
 189         ff_vk_update_descriptor_set(ctx, s->pl, 1);
 190     }
 191
 192     /* Execution context */
 193     RET(ff_vk_create_exec_ctx(ctx, &s->exec,
 194                               s->vkctx.hwctx->queue_family_comp_index));
 195
 196     s->initialized = 1;
 197
 198     return 0;
 199
 200 fail:
 201     return err;
 202 }
 203
 204 static int process_frames(AVFilterContext *avctx, AVFrame *out_f,
 205                           AVFrame *main_f, AVFrame *overlay_f)
 206 {
 207     int err;
 208     OverlayVulkanContext *s = avctx->priv;
 209     int planes = av_pix_fmt_count_planes(s->vkctx.output_format);
 210
 211     AVVkFrame *out     = (AVVkFrame *)out_f->data[0];
 212     AVVkFrame *main    = (AVVkFrame *)main_f->data[0];
 213     AVVkFrame *overlay = (AVVkFrame *)overlay_f->data[0];
 214
 215     AVHWFramesContext *main_fc = (AVHWFramesContext*)main_f->hw_frames_ctx->data;
 216     AVHWFramesContext *overlay_fc = (AVHWFramesContext*)overlay_f->hw_frames_ctx->data;
 217
 218     for (int i = 0; i < planes; i++) {
 219         RET(ff_vk_create_imageview(avctx, &s->main_images[i].imageView, main->img[i],
 220                                    av_vkfmt_from_pixfmt(main_fc->sw_format)[i],
 221                                    ff_comp_identity_map));
 222
 223         RET(ff_vk_create_imageview(avctx, &s->overlay_images[i].imageView, overlay->img[i],
 224                                    av_vkfmt_from_pixfmt(overlay_fc->sw_format)[i],
 225                                    ff_comp_identity_map));
 226
 227         RET(ff_vk_create_imageview(avctx, &s->output_images[i].imageView, out->img[i],
 228                                    av_vkfmt_from_pixfmt(s->vkctx.output_format)[i],
 229                                    ff_comp_identity_map));
 230
 231         s->main_images[i].imageLayout    = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
 232         s->overlay_images[i].imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
 233         s->output_images[i].imageLayout  = VK_IMAGE_LAYOUT_GENERAL;
 234     }
 235
 236     ff_vk_update_descriptor_set(avctx, s->pl, 0);
 237
 238     ff_vk_start_exec_recording(avctx, s->exec);
 239
 240     for (int i = 0; i < planes; i++) {
 241         VkImageMemoryBarrier bar[3] = {
 242             {
 243                 .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
 244                 .srcAccessMask = 0,
 245                 .dstAccessMask = VK_ACCESS_SHADER_READ_BIT,
 246                 .oldLayout = main->layout[i],
 247                 .newLayout = s->main_images[i].imageLayout,
 248                 .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
 249                 .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
 250                 .image = main->img[i],
 251                 .subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
 252                 .subresourceRange.levelCount = 1,
 253                 .subresourceRange.layerCount = 1,
 254             },
 255             {
 256                 .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
 257                 .srcAccessMask = 0,
 258                 .dstAccessMask = VK_ACCESS_SHADER_READ_BIT,
 259                 .oldLayout = overlay->layout[i],
 260                 .newLayout = s->overlay_images[i].imageLayout,
 261                 .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
 262                 .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
 263                 .image = overlay->img[i],
 264                 .subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
 265                 .subresourceRange.levelCount = 1,
 266                 .subresourceRange.layerCount = 1,
 267             },
 268             {
 269                 .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
 270                 .srcAccessMask = 0,
 271                 .dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
 272                 .oldLayout = out->layout[i],
 273                 .newLayout = s->output_images[i].imageLayout,
 274                 .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
 275                 .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
 276                 .image = out->img[i],
 277                 .subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
 278                 .subresourceRange.levelCount = 1,
 279                 .subresourceRange.layerCount = 1,
 280             },
 281         };
 282
 283         vkCmdPipelineBarrier(s->exec->buf, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
 284                              VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0,
 285                              0, NULL, 0, NULL, FF_ARRAY_ELEMS(bar), bar);
 286
 287         main->layout[i]    = bar[0].newLayout;
 288         main->access[i]    = bar[0].dstAccessMask;
 289
 290         overlay->layout[i] = bar[1].newLayout;
 291         overlay->access[i] = bar[1].dstAccessMask;
 292
 293         out->layout[i]     = bar[2].newLayout;
 294         out->access[i]     = bar[2].dstAccessMask;
 295     }
 296
 297     ff_vk_bind_pipeline_exec(avctx, s->exec, s->pl);
 298
 299     vkCmdDispatch(s->exec->buf,
 300                   FFALIGN(s->vkctx.output_width,  CGROUPS[0])/CGROUPS[0],
 301                   FFALIGN(s->vkctx.output_height, CGROUPS[1])/CGROUPS[1], 1);
 302
 303     ff_vk_add_exec_dep(avctx, s->exec, main_f, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT);
 304     ff_vk_add_exec_dep(avctx, s->exec, overlay_f, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT);
 305     ff_vk_add_exec_dep(avctx, s->exec, out_f, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT);
 306
 307     err = ff_vk_submit_exec_queue(avctx, s->exec);
 308     if (err)
 309         return err;
 310
 311 fail:
 312
 313     for (int i = 0; i < planes; i++) {
 314         ff_vk_destroy_imageview(avctx, &s->main_images[i].imageView);
 315         ff_vk_destroy_imageview(avctx, &s->overlay_images[i].imageView);
 316         ff_vk_destroy_imageview(avctx, &s->output_images[i].imageView);
 317     }
 318
 319     return err;
 320 }
 321
 322 static int overlay_vulkan_blend(FFFrameSync *fs)
 323 {
 324     int err;
 325     AVFilterContext *ctx = fs->parent;
 326     OverlayVulkanContext *s = ctx->priv;
 327     AVFilterLink *outlink = ctx->outputs[0];
 328     AVFrame *input_main, *input_overlay, *out;
 329
 330     err = ff_framesync_get_frame(fs, 0, &input_main, 0);
 331     if (err < 0)
 332         goto fail;
 333     err = ff_framesync_get_frame(fs, 1, &input_overlay, 0);
 334     if (err < 0)
 335         goto fail;
 336
 337     if (!input_main || !input_overlay)
 338         return 0;
 339
 340     if (!s->initialized) {
 341         AVHWFramesContext *main_fc = (AVHWFramesContext*)input_main->hw_frames_ctx->data;
 342         AVHWFramesContext *overlay_fc = (AVHWFramesContext*)input_overlay->hw_frames_ctx->data;
 343         if (main_fc->sw_format != overlay_fc->sw_format) {
 344             av_log(ctx, AV_LOG_ERROR, "Mismatching sw formats!\n");
 345             return AVERROR(EINVAL);
 346         }
 347
 348         s->overlay_w = input_overlay->width;
 349         s->overlay_h = input_overlay->height;
 350
 351         RET(init_filter(ctx));
 352     }
 353
 354     out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
 355     if (!out) {
 356         err = AVERROR(ENOMEM);
 357         goto fail;
 358     }
 359
 360     RET(process_frames(ctx, out, input_main, input_overlay));
 361
 362     err = av_frame_copy_props(out, input_main);
 363     if (err < 0)
 364         goto fail;
 365
 366     return ff_filter_frame(outlink, out);
 367
 368 fail:
 369     av_frame_free(&out);
 370     return err;
 371 }
 372
 373 static int overlay_vulkan_config_output(AVFilterLink *outlink)
 374 {
 375     int err;
 376     AVFilterContext *avctx = outlink->src;
 377     OverlayVulkanContext *s = avctx->priv;
 378
 379     err = ff_vk_filter_config_output(outlink);
 380     if (err < 0)
 381         return err;
 382
 383     err = ff_framesync_init_dualinput(&s->fs, avctx);
 384     if (err < 0)
 385         return err;
 386
 387     return ff_framesync_configure(&s->fs);
 388 }
 389
 390 static int overlay_vulkan_activate(AVFilterContext *avctx)
 391 {
 392     OverlayVulkanContext *s = avctx->priv;
 393
 394     return ff_framesync_activate(&s->fs);
 395 }
 396
 397 static av_cold int overlay_vulkan_init(AVFilterContext *avctx)
 398 {
 399     OverlayVulkanContext *s = avctx->priv;
 400
 401     s->fs.on_event = &overlay_vulkan_blend;
 402
 403     return ff_vk_filter_init(avctx);
 404 }
 405
 406 static void overlay_vulkan_uninit(AVFilterContext *avctx)
 407 {
 408     OverlayVulkanContext *s = avctx->priv;
 409
 410     ff_vk_filter_uninit(avctx);
 411     ff_framesync_uninit(&s->fs);
 412
 413     ff_vk_free_buf(avctx, &s->params_buf);
 414
 415     s->initialized = 0;
 416 }
 417
 418 #define OFFSET(x) offsetof(OverlayVulkanContext, x)
 419 #define FLAGS (AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM)
 420 static const AVOption overlay_vulkan_options[] = {
 421     { "x", "Set horizontal offset", OFFSET(overlay_x), AV_OPT_TYPE_INT, {.i64 = 0}, 0, INT_MAX, .flags = FLAGS },
 422     { "y", "Set vertical offset",   OFFSET(overlay_y), AV_OPT_TYPE_INT, {.i64 = 0}, 0, INT_MAX, .flags = FLAGS },
 423     { NULL },
 424 };
 425
 426 AVFILTER_DEFINE_CLASS(overlay_vulkan);
 427
 428 static const AVFilterPad overlay_vulkan_inputs[] = {
 429     {
 430         .name         = "main",
 431         .type         = AVMEDIA_TYPE_VIDEO,
 432         .config_props = &ff_vk_filter_config_input,
 433     },
 434     {
 435         .name         = "overlay",
 436         .type         = AVMEDIA_TYPE_VIDEO,
 437         .config_props = &ff_vk_filter_config_input,
 438     },
 439     { NULL }
 440 };
 441
 442 static const AVFilterPad overlay_vulkan_outputs[] = {
 443     {
 444         .name = "default",
 445         .type = AVMEDIA_TYPE_VIDEO,
 446         .config_props = &overlay_vulkan_config_output,
 447     },
 448     { NULL }
 449 };
 450
 451 AVFilter ff_vf_overlay_vulkan = {
 452     .name           = "overlay_vulkan",
 453     .description    = NULL_IF_CONFIG_SMALL("Overlay a source on top of another"),
 454     .priv_size      = sizeof(OverlayVulkanContext),
 455     .init           = &overlay_vulkan_init,
 456     .uninit         = &overlay_vulkan_uninit,
 457     .query_formats  = &ff_vk_filter_query_formats,
 458     .activate       = &overlay_vulkan_activate,
 459     .inputs         = overlay_vulkan_inputs,
 460     .outputs        = overlay_vulkan_outputs,
 461     .priv_class     = &overlay_vulkan_class,
 462     .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
 463 };