git.sesse.net Git - ffmpeg/blob - libavfilter/vf_overlay_vulkan.c

   1 /*
   2  * This file is part of FFmpeg.
   3  *
   4  * FFmpeg is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU Lesser General Public
   6  * License as published by the Free Software Foundation; either
   7  * version 2.1 of the License, or (at your option) any later version.
   8  *
   9  * FFmpeg is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * Lesser General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU Lesser General Public
  15  * License along with FFmpeg; if not, write to the Free Software
  16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  17  */
  18
  19 #include "libavutil/random_seed.h"
  20 #include "libavutil/opt.h"
  21 #include "vulkan.h"
  22 #include "internal.h"
  23 #include "framesync.h"
  24
  25 #define CGROUPS (int [3]){ 32, 32, 1 }
  26
  27 typedef struct OverlayVulkanContext {
  28     VulkanFilterContext vkctx;
  29
  30     int initialized;
  31     VulkanPipeline *pl;
  32     FFVkExecContext *exec;
  33     FFFrameSync fs;
  34     FFVkBuffer params_buf;
  35
  36     /* Shader updators, must be in the main filter struct */
  37     VkDescriptorImageInfo main_images[3];
  38     VkDescriptorImageInfo overlay_images[3];
  39     VkDescriptorImageInfo output_images[3];
  40     VkDescriptorBufferInfo params_desc;
  41
  42     int overlay_x;
  43     int overlay_y;
  44     int overlay_w;
  45     int overlay_h;
  46 } OverlayVulkanContext;
  47
  48 static const char overlay_noalpha[] = {
  49     C(0, void overlay_noalpha(int i, ivec2 pos)                                )
  50     C(0, {                                                                     )
  51     C(1,     if ((o_offset[i].x <= pos.x) && (o_offset[i].y <= pos.y) &&
  52                  (pos.x < (o_offset[i].x + o_size[i].x)) &&
  53                  (pos.y < (o_offset[i].y + o_size[i].y))) {                    )
  54     C(2,         vec4 res = texture(overlay_img[i], pos - o_offset[i]);        )
  55     C(2,         imageStore(output_img[i], pos, res);                          )
  56     C(1,     } else {                                                          )
  57     C(2,         vec4 res = texture(main_img[i], pos);                         )
  58     C(2,         imageStore(output_img[i], pos, res);                          )
  59     C(1,     }                                                                 )
  60     C(0, }                                                                     )
  61 };
  62
  63 static const char overlay_alpha[] = {
  64     C(0, void overlay_alpha_opaque(int i, ivec2 pos)                           )
  65     C(0, {                                                                     )
  66     C(1,     vec4 res = texture(main_img[i], pos);                             )
  67     C(1,     if ((o_offset[i].x <= pos.x) && (o_offset[i].y <= pos.y) &&
  68                  (pos.x < (o_offset[i].x + o_size[i].x)) &&
  69                  (pos.y < (o_offset[i].y + o_size[i].y))) {                    )
  70     C(2,         vec4 ovr = texture(overlay_img[i], pos - o_offset[i]);        )
  71     C(2,         res = ovr * ovr.a + res * (1.0f - ovr.a);                     )
  72     C(2,         res.a = 1.0f;                                                 )
  73     C(2,         imageStore(output_img[i], pos, res);                          )
  74     C(1,     }                                                                 )
  75     C(1,     imageStore(output_img[i], pos, res);                              )
  76     C(0, }                                                                     )
  77 };
  78
  79 static av_cold int init_filter(AVFilterContext *ctx)
  80 {
  81     int err;
  82     OverlayVulkanContext *s = ctx->priv;
  83     VkSampler *sampler = ff_vk_init_sampler(ctx, 1, VK_FILTER_NEAREST);
  84     if (!sampler)
  85         return AVERROR_EXTERNAL;
  86
  87     s->pl = ff_vk_create_pipeline(ctx);
  88     if (!s->pl)
  89         return AVERROR(ENOMEM);
  90
  91     s->vkctx.queue_family_idx = s->vkctx.hwctx->queue_family_comp_index;
  92     s->vkctx.queue_count = GET_QUEUE_COUNT(s->vkctx.hwctx, 0, 1, 0);
  93     s->vkctx.cur_queue_idx = av_get_random_seed() % s->vkctx.queue_count;
  94
  95     { /* Create the shader */
  96         const int planes = av_pix_fmt_count_planes(s->vkctx.output_format);
  97         const int ialpha = av_pix_fmt_desc_get(s->vkctx.input_format)->flags & AV_PIX_FMT_FLAG_ALPHA;
  98
  99         VulkanDescriptorSetBinding desc_i[3] = {
 100             {
 101                 .name       = "main_img",
 102                 .type       = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
 103                 .dimensions = 2,
 104                 .elems      = planes,
 105                 .stages     = VK_SHADER_STAGE_COMPUTE_BIT,
 106                 .updater    = s->main_images,
 107                 .samplers   = DUP_SAMPLER_ARRAY4(*sampler),
 108             },
 109             {
 110                 .name       = "overlay_img",
 111                 .type       = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
 112                 .dimensions = 2,
 113                 .elems      = planes,
 114                 .stages     = VK_SHADER_STAGE_COMPUTE_BIT,
 115                 .updater    = s->overlay_images,
 116                 .samplers   = DUP_SAMPLER_ARRAY4(*sampler),
 117             },
 118             {
 119                 .name       = "output_img",
 120                 .type       = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
 121                 .mem_layout = ff_vk_shader_rep_fmt(s->vkctx.output_format),
 122                 .mem_quali  = "writeonly",
 123                 .dimensions = 2,
 124                 .elems      = planes,
 125                 .stages     = VK_SHADER_STAGE_COMPUTE_BIT,
 126                 .updater    = s->output_images,
 127             },
 128         };
 129
 130         VulkanDescriptorSetBinding desc_b = {
 131             .name        = "params",
 132             .type        = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
 133             .mem_quali   = "readonly",
 134             .mem_layout  = "std430",
 135             .stages      = VK_SHADER_STAGE_COMPUTE_BIT,
 136             .updater     = &s->params_desc,
 137             .buf_content = "ivec2 o_offset[3], o_size[3];",
 138         };
 139
 140         SPIRVShader *shd = ff_vk_init_shader(ctx, s->pl, "overlay_compute",
 141                                              VK_SHADER_STAGE_COMPUTE_BIT);
 142         if (!shd)
 143             return AVERROR(ENOMEM);
 144
 145         ff_vk_set_compute_shader_sizes(ctx, shd, CGROUPS);
 146
 147         RET(ff_vk_add_descriptor_set(ctx, s->pl, shd,  desc_i, 3, 0)); /* set 0 */
 148         RET(ff_vk_add_descriptor_set(ctx, s->pl, shd, &desc_b, 1, 0)); /* set 1 */
 149
 150         GLSLD(   overlay_noalpha                                              );
 151         GLSLD(   overlay_alpha                                                );
 152         GLSLC(0, void main()                                                  );
 153         GLSLC(0, {                                                            );
 154         GLSLC(1,     ivec2 pos = ivec2(gl_GlobalInvocationID.xy);             );
 155         GLSLF(1,     int planes = %i;                                  ,planes);
 156         GLSLC(1,     for (int i = 0; i < planes; i++) {                       );
 157         if (ialpha)
 158             GLSLC(2,         overlay_alpha_opaque(i, pos);                    );
 159         else
 160             GLSLC(2,         overlay_noalpha(i, pos);                         );
 161         GLSLC(1,     }                                                        );
 162         GLSLC(0, }                                                            );
 163
 164         RET(ff_vk_compile_shader(ctx, shd, "main"));
 165     }
 166
 167     RET(ff_vk_init_pipeline_layout(ctx, s->pl));
 168     RET(ff_vk_init_compute_pipeline(ctx, s->pl));
 169
 170     { /* Create and update buffer */
 171         const AVPixFmtDescriptor *desc;
 172
 173         /* NOTE: std430 requires the same identical struct layout, padding and
 174          * alignment as C, so we're allowed to do this, as this will map
 175          * exactly to what the shader recieves */
 176         struct {
 177             int32_t o_offset[2*3];
 178             int32_t o_size[2*3];
 179         } *par;
 180
 181         err = ff_vk_create_buf(ctx, &s->params_buf,
 182                                sizeof(*par),
 183                                VK_BUFFER_USAGE_STORAGE_BUFFER_BIT,
 184                                VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT);
 185         if (err)
 186             return err;
 187
 188         err = ff_vk_map_buffers(ctx, &s->params_buf, (uint8_t **)&par, 1, 0);
 189         if (err)
 190             return err;
 191
 192         desc = av_pix_fmt_desc_get(s->vkctx.output_format);
 193
 194         par->o_offset[0] = s->overlay_x;
 195         par->o_offset[1] = s->overlay_y;
 196         par->o_offset[2] = par->o_offset[0] >> desc->log2_chroma_w;
 197         par->o_offset[3] = par->o_offset[1] >> desc->log2_chroma_h;
 198         par->o_offset[4] = par->o_offset[0] >> desc->log2_chroma_w;
 199         par->o_offset[5] = par->o_offset[1] >> desc->log2_chroma_h;
 200
 201         par->o_size[0] = s->overlay_w;
 202         par->o_size[1] = s->overlay_h;
 203         par->o_size[2] = par->o_size[0] >> desc->log2_chroma_w;
 204         par->o_size[3] = par->o_size[1] >> desc->log2_chroma_h;
 205         par->o_size[4] = par->o_size[0] >> desc->log2_chroma_w;
 206         par->o_size[5] = par->o_size[1] >> desc->log2_chroma_h;
 207
 208         err = ff_vk_unmap_buffers(ctx, &s->params_buf, 1, 1);
 209         if (err)
 210             return err;
 211
 212         s->params_desc.buffer = s->params_buf.buf;
 213         s->params_desc.range  = VK_WHOLE_SIZE;
 214
 215         ff_vk_update_descriptor_set(ctx, s->pl, 1);
 216     }
 217
 218     /* Execution context */
 219     RET(ff_vk_create_exec_ctx(ctx, &s->exec));
 220
 221     s->initialized = 1;
 222
 223     return 0;
 224
 225 fail:
 226     return err;
 227 }
 228
 229 static int process_frames(AVFilterContext *avctx, AVFrame *out_f,
 230                           AVFrame *main_f, AVFrame *overlay_f)
 231 {
 232     int err;
 233     VkCommandBuffer cmd_buf;
 234     OverlayVulkanContext *s = avctx->priv;
 235     int planes = av_pix_fmt_count_planes(s->vkctx.output_format);
 236
 237     AVVkFrame *out     = (AVVkFrame *)out_f->data[0];
 238     AVVkFrame *main    = (AVVkFrame *)main_f->data[0];
 239     AVVkFrame *overlay = (AVVkFrame *)overlay_f->data[0];
 240
 241     AVHWFramesContext *main_fc = (AVHWFramesContext*)main_f->hw_frames_ctx->data;
 242     AVHWFramesContext *overlay_fc = (AVHWFramesContext*)overlay_f->hw_frames_ctx->data;
 243
 244     /* Update descriptors and init the exec context */
 245     ff_vk_start_exec_recording(avctx, s->exec);
 246     cmd_buf = ff_vk_get_exec_buf(avctx, s->exec);
 247
 248     for (int i = 0; i < planes; i++) {
 249         RET(ff_vk_create_imageview(avctx, s->exec, &s->main_images[i].imageView,
 250                                    main->img[i],
 251                                    av_vkfmt_from_pixfmt(main_fc->sw_format)[i],
 252                                    ff_comp_identity_map));
 253
 254         RET(ff_vk_create_imageview(avctx, s->exec, &s->overlay_images[i].imageView,
 255                                    overlay->img[i],
 256                                    av_vkfmt_from_pixfmt(overlay_fc->sw_format)[i],
 257                                    ff_comp_identity_map));
 258
 259         RET(ff_vk_create_imageview(avctx, s->exec, &s->output_images[i].imageView,
 260                                    out->img[i],
 261                                    av_vkfmt_from_pixfmt(s->vkctx.output_format)[i],
 262                                    ff_comp_identity_map));
 263
 264         s->main_images[i].imageLayout    = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
 265         s->overlay_images[i].imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
 266         s->output_images[i].imageLayout  = VK_IMAGE_LAYOUT_GENERAL;
 267     }
 268
 269     ff_vk_update_descriptor_set(avctx, s->pl, 0);
 270
 271     for (int i = 0; i < planes; i++) {
 272         VkImageMemoryBarrier bar[3] = {
 273             {
 274                 .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
 275                 .srcAccessMask = 0,
 276                 .dstAccessMask = VK_ACCESS_SHADER_READ_BIT,
 277                 .oldLayout = main->layout[i],
 278                 .newLayout = s->main_images[i].imageLayout,
 279                 .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
 280                 .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
 281                 .image = main->img[i],
 282                 .subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
 283                 .subresourceRange.levelCount = 1,
 284                 .subresourceRange.layerCount = 1,
 285             },
 286             {
 287                 .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
 288                 .srcAccessMask = 0,
 289                 .dstAccessMask = VK_ACCESS_SHADER_READ_BIT,
 290                 .oldLayout = overlay->layout[i],
 291                 .newLayout = s->overlay_images[i].imageLayout,
 292                 .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
 293                 .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
 294                 .image = overlay->img[i],
 295                 .subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
 296                 .subresourceRange.levelCount = 1,
 297                 .subresourceRange.layerCount = 1,
 298             },
 299             {
 300                 .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
 301                 .srcAccessMask = 0,
 302                 .dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT,
 303                 .oldLayout = out->layout[i],
 304                 .newLayout = s->output_images[i].imageLayout,
 305                 .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
 306                 .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
 307                 .image = out->img[i],
 308                 .subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
 309                 .subresourceRange.levelCount = 1,
 310                 .subresourceRange.layerCount = 1,
 311             },
 312         };
 313
 314         vkCmdPipelineBarrier(cmd_buf, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
 315                              VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0,
 316                              0, NULL, 0, NULL, FF_ARRAY_ELEMS(bar), bar);
 317
 318         main->layout[i]    = bar[0].newLayout;
 319         main->access[i]    = bar[0].dstAccessMask;
 320
 321         overlay->layout[i] = bar[1].newLayout;
 322         overlay->access[i] = bar[1].dstAccessMask;
 323
 324         out->layout[i]     = bar[2].newLayout;
 325         out->access[i]     = bar[2].dstAccessMask;
 326     }
 327
 328     ff_vk_bind_pipeline_exec(avctx, s->exec, s->pl);
 329
 330     vkCmdDispatch(cmd_buf,
 331                   FFALIGN(s->vkctx.output_width,  CGROUPS[0])/CGROUPS[0],
 332                   FFALIGN(s->vkctx.output_height, CGROUPS[1])/CGROUPS[1], 1);
 333
 334     ff_vk_add_exec_dep(avctx, s->exec, main_f, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT);
 335     ff_vk_add_exec_dep(avctx, s->exec, overlay_f, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT);
 336     ff_vk_add_exec_dep(avctx, s->exec, out_f, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT);
 337
 338     err = ff_vk_submit_exec_queue(avctx, s->exec);
 339     if (err)
 340         return err;
 341
 342     return err;
 343
 344 fail:
 345     ff_vk_discard_exec_deps(avctx, s->exec);
 346     return err;
 347 }
 348
 349 static int overlay_vulkan_blend(FFFrameSync *fs)
 350 {
 351     int err;
 352     AVFilterContext *ctx = fs->parent;
 353     OverlayVulkanContext *s = ctx->priv;
 354     AVFilterLink *outlink = ctx->outputs[0];
 355     AVFrame *input_main, *input_overlay, *out;
 356
 357     err = ff_framesync_get_frame(fs, 0, &input_main, 0);
 358     if (err < 0)
 359         goto fail;
 360     err = ff_framesync_get_frame(fs, 1, &input_overlay, 0);
 361     if (err < 0)
 362         goto fail;
 363
 364     if (!input_main || !input_overlay)
 365         return 0;
 366
 367     if (!s->initialized) {
 368         AVHWFramesContext *main_fc = (AVHWFramesContext*)input_main->hw_frames_ctx->data;
 369         AVHWFramesContext *overlay_fc = (AVHWFramesContext*)input_overlay->hw_frames_ctx->data;
 370         if (main_fc->sw_format != overlay_fc->sw_format) {
 371             av_log(ctx, AV_LOG_ERROR, "Mismatching sw formats!\n");
 372             return AVERROR(EINVAL);
 373         }
 374
 375         s->overlay_w = input_overlay->width;
 376         s->overlay_h = input_overlay->height;
 377
 378         RET(init_filter(ctx));
 379     }
 380
 381     out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
 382     if (!out) {
 383         err = AVERROR(ENOMEM);
 384         goto fail;
 385     }
 386
 387     RET(process_frames(ctx, out, input_main, input_overlay));
 388
 389     err = av_frame_copy_props(out, input_main);
 390     if (err < 0)
 391         goto fail;
 392
 393     return ff_filter_frame(outlink, out);
 394
 395 fail:
 396     av_frame_free(&out);
 397     return err;
 398 }
 399
 400 static int overlay_vulkan_config_output(AVFilterLink *outlink)
 401 {
 402     int err;
 403     AVFilterContext *avctx = outlink->src;
 404     OverlayVulkanContext *s = avctx->priv;
 405
 406     err = ff_vk_filter_config_output(outlink);
 407     if (err < 0)
 408         return err;
 409
 410     err = ff_framesync_init_dualinput(&s->fs, avctx);
 411     if (err < 0)
 412         return err;
 413
 414     return ff_framesync_configure(&s->fs);
 415 }
 416
 417 static int overlay_vulkan_activate(AVFilterContext *avctx)
 418 {
 419     OverlayVulkanContext *s = avctx->priv;
 420
 421     return ff_framesync_activate(&s->fs);
 422 }
 423
 424 static av_cold int overlay_vulkan_init(AVFilterContext *avctx)
 425 {
 426     OverlayVulkanContext *s = avctx->priv;
 427
 428     s->fs.on_event = &overlay_vulkan_blend;
 429
 430     return ff_vk_filter_init(avctx);
 431 }
 432
 433 static void overlay_vulkan_uninit(AVFilterContext *avctx)
 434 {
 435     OverlayVulkanContext *s = avctx->priv;
 436
 437     ff_vk_filter_uninit(avctx);
 438     ff_framesync_uninit(&s->fs);
 439
 440     ff_vk_free_buf(avctx, &s->params_buf);
 441
 442     s->initialized = 0;
 443 }
 444
 445 #define OFFSET(x) offsetof(OverlayVulkanContext, x)
 446 #define FLAGS (AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM)
 447 static const AVOption overlay_vulkan_options[] = {
 448     { "x", "Set horizontal offset", OFFSET(overlay_x), AV_OPT_TYPE_INT, {.i64 = 0}, 0, INT_MAX, .flags = FLAGS },
 449     { "y", "Set vertical offset",   OFFSET(overlay_y), AV_OPT_TYPE_INT, {.i64 = 0}, 0, INT_MAX, .flags = FLAGS },
 450     { NULL },
 451 };
 452
 453 AVFILTER_DEFINE_CLASS(overlay_vulkan);
 454
 455 static const AVFilterPad overlay_vulkan_inputs[] = {
 456     {
 457         .name         = "main",
 458         .type         = AVMEDIA_TYPE_VIDEO,
 459         .config_props = &ff_vk_filter_config_input,
 460     },
 461     {
 462         .name         = "overlay",
 463         .type         = AVMEDIA_TYPE_VIDEO,
 464         .config_props = &ff_vk_filter_config_input,
 465     },
 466     { NULL }
 467 };
 468
 469 static const AVFilterPad overlay_vulkan_outputs[] = {
 470     {
 471         .name = "default",
 472         .type = AVMEDIA_TYPE_VIDEO,
 473         .config_props = &overlay_vulkan_config_output,
 474     },
 475     { NULL }
 476 };
 477
 478 const AVFilter ff_vf_overlay_vulkan = {
 479     .name           = "overlay_vulkan",
 480     .description    = NULL_IF_CONFIG_SMALL("Overlay a source on top of another"),
 481     .priv_size      = sizeof(OverlayVulkanContext),
 482     .init           = &overlay_vulkan_init,
 483     .uninit         = &overlay_vulkan_uninit,
 484     .query_formats  = &ff_vk_filter_query_formats,
 485     .activate       = &overlay_vulkan_activate,
 486     .inputs         = overlay_vulkan_inputs,
 487     .outputs        = overlay_vulkan_outputs,
 488     .priv_class     = &overlay_vulkan_class,
 489     .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
 490 };