git.sesse.net Git - casparcg/blob - accelerator/cpu/image/image_mixer.cpp

   1 /*\r
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>\r
   3 *\r
   4 * This file is part of CasparCG (www.casparcg.com).\r
   5 *\r
   6 * CasparCG is free software: you can redistribute it and/or modify\r
   7 * it under the terms of the GNU General Public License as published by\r
   8 * the Free Software Foundation, either version 3 of the License, or\r
   9 * (at your option) any later version.\r
  10 *\r
  11 * CasparCG is distributed in the hope that it will be useful,\r
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of\r
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\r
  14 * GNU General Public License for more details.\r
  15 *\r
  16 * You should have received a copy of the GNU General Public License\r
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.\r
  18 *\r
  19 * Author: Robert Nagy, ronag89@gmail.com\r
  20 */\r
  21 \r
  22 #include "../../stdafx.h"\r
  23 \r
  24 #include "image_mixer.h"\r
  25 \r
  26 #include "../util/write_frame.h"\r
  27 \r
  28 #include <common/assert.h>\r
  29 #include <common/gl/gl_check.h>\r
  30 #include <common/concurrency/async.h>\r
  31 #include <common/memory/memcpy.h>\r
  32 \r
  33 #include <core/frame/write_frame.h>\r
  34 #include <core/frame/frame_transform.h>\r
  35 #include <core/frame/pixel_format.h>\r
  36 #include <core/video_format.h>\r
  37 \r
  38 #include <asmlib.h>\r
  39 \r
  40 #include <gl/glew.h>\r
  41 \r
  42 #include <tbb/cache_aligned_allocator.h>\r
  43 \r
  44 #include <boost/foreach.hpp>\r
  45 #include <boost/range.hpp>\r
  46 #include <boost/range/algorithm_ext/erase.hpp>\r
  47 #include <boost/thread/future.hpp>\r
  48 \r
  49 #include <algorithm>\r
  50 #include <vector>\r
  51 \r
  52 namespace caspar { namespace accelerator { namespace cpu {\r
  53                 \r
  54 struct item\r
  55 {\r
  56         core::pixel_format_desc                                         pix_desc;\r
  57         std::vector<spl::shared_ptr<host_buffer>>       buffers;\r
  58         core::frame_transform                                           transform;\r
  59 \r
  60         item()\r
  61                 : pix_desc(core::pixel_format::invalid)\r
  62         {\r
  63         }\r
  64 };\r
  65 \r
  66 bool operator==(const item& lhs, const item& rhs)\r
  67 {\r
  68         return lhs.buffers == rhs.buffers && lhs.transform == rhs.transform;\r
  69 }\r
  70 \r
  71 bool operator!=(const item& lhs, const item& rhs)\r
  72 {\r
  73         return !(lhs == rhs);\r
  74 }\r
  75 \r
  76 struct layer\r
  77 {\r
  78         std::vector<item>       items;\r
  79 \r
  80         layer()\r
  81         {\r
  82         }\r
  83 \r
  84         layer(std::vector<item> items)\r
  85                 : items(std::move(items))\r
  86         {\r
  87         }\r
  88 };\r
  89 \r
  90 bool operator==(const layer& lhs, const layer& rhs)\r
  91 {\r
  92         return lhs.items == rhs.items;\r
  93 }\r
  94 \r
  95 bool operator!=(const layer& lhs, const layer& rhs)\r
  96 {\r
  97         return !(lhs == rhs);\r
  98 }\r
  99 \r
 100 class image_renderer\r
 101 {\r
 102         std::pair<std::vector<layer>, boost::shared_future<boost::iterator_range<const uint8_t*>>> last_image_;\r
 103 public:\r
 104         \r
 105         boost::shared_future<boost::iterator_range<const uint8_t*>> operator()(std::vector<layer> layers, const core::video_format_desc& format_desc)\r
 106         {       \r
 107                 if(last_image_.first == layers && last_image_.second.has_value())\r
 108                         return last_image_.second;\r
 109 \r
 110                 auto image      = render(layers, format_desc);\r
 111                 last_image_ = std::make_pair(std::move(layers), image);\r
 112                 return image;\r
 113         }\r
 114 \r
 115 private:        \r
 116         boost::shared_future<boost::iterator_range<const uint8_t*>> render(std::vector<layer> layers, const core::video_format_desc& format_desc)\r
 117         {       \r
 118                 static const auto empty = spl::make_shared<const std::vector<uint8_t, tbb::cache_aligned_allocator<uint8_t>>>(2048*2048*4, 0);\r
 119                 CASPAR_VERIFY(empty->size() >= format_desc.size);\r
 120                 \r
 121                 if(layers.empty())\r
 122                 {\r
 123                         return async(launch_policy::deferred, [=]\r
 124                         {\r
 125                                 return boost::iterator_range<const uint8_t*>(empty->data(), empty->data() + format_desc.size);\r
 126                         });\r
 127                 }\r
 128                 else if(layers.size()                           == 1 &&\r
 129                             layers.at(0).items.size()   == 1 &&\r
 130                             layers.at(0).items.at(0).pix_desc.format            == core::pixel_format::bgra &&\r
 131                             layers.at(0).items.at(0).buffers.at(0)->size() == format_desc.size &&\r
 132                             layers.at(0).items.at(0).transform                          == core::frame_transform())\r
 133                 {\r
 134                         auto buffer = layers.at(0).items.at(0).buffers.at(0);\r
 135                         return async(launch_policy::deferred, [=]\r
 136                         {\r
 137                                 return boost::iterator_range<const uint8_t*>(buffer->data(), buffer->data() + format_desc.size);\r
 138                         });\r
 139                 }\r
 140                 else\r
 141                 {\r
 142                         return async(launch_policy::deferred, [=]\r
 143                         {\r
 144                                 return boost::iterator_range<const uint8_t*>(empty->data(), empty->data() + format_desc.size);\r
 145                         });\r
 146                 }\r
 147                 //else\r
 148                 //{                             \r
 149                         //auto draw_buffer = create_mixer_buffer(4, format_desc);\r
 150 \r
 151                         //if(format_desc.field_mode != core::field_mode::progressive)\r
 152                         //{\r
 153                         //      auto upper = layers;\r
 154                         //      auto lower = std::move(layers);\r
 155 \r
 156                         //      BOOST_FOREACH(auto& layer, upper)\r
 157                         //      {\r
 158                         //              BOOST_FOREACH(auto& item, layer.items)\r
 159                         //                      item.transform.field_mode &= core::field_mode::upper;\r
 160                         //      }\r
 161 \r
 162                         //      BOOST_FOREACH(auto& layer, lower)\r
 163                         //      {\r
 164                         //              BOOST_FOREACH(auto& item, layer.items)\r
 165                         //                      item.transform.field_mode &= core::field_mode::lower;\r
 166                         //      }\r
 167 \r
 168                         //      draw(std::move(upper), draw_buffer, format_desc);\r
 169                         //      draw(std::move(lower), draw_buffer, format_desc);\r
 170                         //}\r
 171                         //else\r
 172                         //{\r
 173                         //      draw(std::move(layers), draw_buffer, format_desc);\r
 174                         //}\r
 175                         //                                              \r
 176                         //return draw_buffer;\r
 177                 //}\r
 178         }\r
 179         \r
 180         //void draw(std::vector<layer>&&                                layers, \r
 181         //                spl::shared_ptr<device_buffer>&       draw_buffer, \r
 182         //                const core::video_format_desc&        format_desc)\r
 183         //{\r
 184         //      std::shared_ptr<device_buffer> layer_key_buffer;\r
 185 \r
 186         //      BOOST_FOREACH(auto& layer, layers)\r
 187         //              draw_layer(std::move(layer), draw_buffer, layer_key_buffer, format_desc);\r
 188         //}\r
 189 \r
 190         //void draw_layer(layer&&                                                               layer, \r
 191         //                              spl::shared_ptr<device_buffer>&         draw_buffer,\r
 192         //                              std::shared_ptr<device_buffer>&         layer_key_buffer,\r
 193         //                              const core::video_format_desc&          format_desc)\r
 194         //{             \r
 195         //      // Remove empty items.\r
 196         //      boost::range::remove_erase_if(layer.items, [](const item& item)\r
 197         //      {\r
 198         //              return item.transform.field_mode == core::field_mode::empty;\r
 199         //      });\r
 200 \r
 201         //      if(layer.items.empty())\r
 202         //              return;\r
 203 \r
 204         //      std::shared_ptr<device_buffer> local_key_buffer;\r
 205         //      std::shared_ptr<device_buffer> local_mix_buffer;\r
 206         //                      \r
 207         //      if(layer.blend_mode != core::blend_mode::normal)\r
 208         //      {\r
 209         //              auto layer_draw_buffer = create_mixer_buffer(4, format_desc);\r
 210 \r
 211         //              BOOST_FOREACH(auto& item, layer.items)\r
 212         //                      draw_item(std::move(item), layer_draw_buffer, layer_key_buffer, local_key_buffer, local_mix_buffer, format_desc);       \r
 213         //      \r
 214         //              draw_mixer_buffer(layer_draw_buffer, std::move(local_mix_buffer), core::blend_mode::normal);                                                    \r
 215         //              draw_mixer_buffer(draw_buffer, std::move(layer_draw_buffer), layer.blend_mode);\r
 216         //      }\r
 217         //      else // fast path\r
 218         //      {\r
 219         //              BOOST_FOREACH(auto& item, layer.items)          \r
 220         //                      draw_item(std::move(item), draw_buffer, layer_key_buffer, local_key_buffer, local_mix_buffer, format_desc);             \r
 221         //                              \r
 222         //              draw_mixer_buffer(draw_buffer, std::move(local_mix_buffer), core::blend_mode::normal);\r
 223         //      }                                       \r
 224 \r
 225         //      layer_key_buffer = std::move(local_key_buffer);\r
 226         //}\r
 227 \r
 228         //void draw_item(item&&                                                 item, \r
 229         //                         spl::shared_ptr<device_buffer>&      draw_buffer, \r
 230         //                         std::shared_ptr<device_buffer>&      layer_key_buffer, \r
 231         //                         std::shared_ptr<device_buffer>&      local_key_buffer, \r
 232         //                         std::shared_ptr<device_buffer>&      local_mix_buffer,\r
 233         //                         const core::video_format_desc&       format_desc)\r
 234         //{                     \r
 235         //      draw_params draw_params;\r
 236         //      draw_params.pix_desc    = std::move(item.pix_desc);\r
 237         //      draw_params.transform   = std::move(item.transform);\r
 238         //      BOOST_FOREACH(auto& future_texture, item.textures)\r
 239         //              draw_params.textures.push_back(future_texture.get());\r
 240 \r
 241         //      if(item.transform.is_key)\r
 242         //      {\r
 243         //              local_key_buffer = local_key_buffer ? local_key_buffer : create_mixer_buffer(1, format_desc);\r
 244 \r
 245         //              draw_params.background                  = local_key_buffer;\r
 246         //              draw_params.local_key                   = nullptr;\r
 247         //              draw_params.layer_key                   = nullptr;\r
 248 \r
 249         //              kernel_.draw(std::move(draw_params));\r
 250         //      }\r
 251         //      else if(item.transform.is_mix)\r
 252         //      {\r
 253         //              local_mix_buffer = local_mix_buffer ? local_mix_buffer : create_mixer_buffer(4, format_desc);\r
 254 \r
 255         //              draw_params.background                  = local_mix_buffer;\r
 256         //              draw_params.local_key                   = std::move(local_key_buffer);\r
 257         //              draw_params.layer_key                   = layer_key_buffer;\r
 258 \r
 259         //              draw_params.keyer                               = keyer::additive;\r
 260 \r
 261         //              kernel_.draw(std::move(draw_params));\r
 262         //      }\r
 263         //      else\r
 264         //      {\r
 265         //              draw_mixer_buffer(draw_buffer, std::move(local_mix_buffer), core::blend_mode::normal);\r
 266         //              \r
 267         //              draw_params.background                  = draw_buffer;\r
 268         //              draw_params.local_key                   = std::move(local_key_buffer);\r
 269         //              draw_params.layer_key                   = layer_key_buffer;\r
 270 \r
 271         //              kernel_.draw(std::move(draw_params));\r
 272         //      }       \r
 273         //}\r
 274 \r
 275         //void draw_mixer_buffer(spl::shared_ptr<device_buffer>&        draw_buffer, \r
 276         //                                         std::shared_ptr<device_buffer>&& source_buffer, \r
 277         //                                         core::blend_mode                                     blend_mode = core::blend_mode::normal)\r
 278         //{\r
 279         //      if(!source_buffer)\r
 280         //              return;\r
 281 \r
 282         //      draw_params draw_params;\r
 283         //      draw_params.pix_desc.format             = core::pixel_format::bgra;\r
 284         //      draw_params.pix_desc.planes             = list_of(core::pixel_format_desc::plane(source_buffer->width(), source_buffer->height(), 4));\r
 285         //      draw_params.textures                    = list_of(source_buffer);\r
 286         //      draw_params.transform                   = core::frame_transform();\r
 287         //      draw_params.blend_mode                  = blend_mode;\r
 288         //      draw_params.background                  = draw_buffer;\r
 289 \r
 290         //      kernel_.draw(std::move(draw_params));\r
 291         //}\r
 292         //              \r
 293         //spl::shared_ptr<device_buffer> create_mixer_buffer(int stride, const core::video_format_desc& format_desc)\r
 294         //{\r
 295         //      auto buffer = ogl_->create_device_buffer(format_desc.width, format_desc.height, stride);\r
 296         //      buffer->clear();\r
 297         //      return buffer;\r
 298         //}\r
 299 };\r
 300                 \r
 301 struct image_mixer::impl : boost::noncopyable\r
 302 {       \r
 303         image_renderer                                          renderer_;\r
 304         std::vector<core::frame_transform>      transform_stack_;\r
 305         std::vector<layer>                                      layers_; // layer/stream/items\r
 306 public:\r
 307         impl() \r
 308                 : transform_stack_(1)   \r
 309         {\r
 310                 CASPAR_LOG(info) << L"Initialized CPU Accelerated Image Mixer";\r
 311         }\r
 312 \r
 313         void begin_layer(core::blend_mode blend_mode)\r
 314         {\r
 315                 layers_.push_back(layer(std::vector<item>()));\r
 316         }\r
 317                 \r
 318         void push(core::frame_transform& transform)\r
 319         {\r
 320                 transform_stack_.push_back(transform_stack_.back()*transform);\r
 321         }\r
 322                 \r
 323         void visit(core::data_frame& frame2)\r
 324         {                       \r
 325                 write_frame* frame = dynamic_cast<write_frame*>(&frame2);\r
 326                 if(frame == nullptr)\r
 327                         return;\r
 328 \r
 329                 if(frame->get_pixel_format_desc().format == core::pixel_format::invalid)\r
 330                         return;\r
 331 \r
 332                 if(frame->get_buffers().empty())\r
 333                         return;\r
 334 \r
 335                 if(transform_stack_.back().field_mode == core::field_mode::empty)\r
 336                         return;\r
 337 \r
 338                 item item;\r
 339                 item.pix_desc                   = frame->get_pixel_format_desc();\r
 340                 item.buffers                    = frame->get_buffers();                         \r
 341                 item.transform                  = transform_stack_.back();\r
 342                 item.transform.volume   = core::frame_transform().volume; // Set volume to default since we don't care about it here.\r
 343 \r
 344                 layers_.back().items.push_back(item);\r
 345         }\r
 346 \r
 347         void pop()\r
 348         {\r
 349                 transform_stack_.pop_back();\r
 350         }\r
 351 \r
 352         void end_layer()\r
 353         {               \r
 354         }\r
 355         \r
 356         boost::shared_future<boost::iterator_range<const uint8_t*>> render(const core::video_format_desc& format_desc)\r
 357         {\r
 358                 // Remove empty layers.\r
 359                 boost::range::remove_erase_if(layers_, [](const layer& layer)\r
 360                 {\r
 361                         return layer.items.empty();\r
 362                 });\r
 363 \r
 364                 return renderer_(std::move(layers_), format_desc);\r
 365         }\r
 366         \r
 367         virtual spl::shared_ptr<cpu::write_frame> create_frame(const void* tag, const core::pixel_format_desc& desc)\r
 368         {\r
 369                 return spl::make_shared<cpu::write_frame>(tag, desc);\r
 370         }\r
 371 };\r
 372 \r
 373 image_mixer::image_mixer() : impl_(new impl()){}\r
 374 void image_mixer::push(core::frame_transform& transform){impl_->push(transform);}\r
 375 void image_mixer::visit(core::data_frame& frame){impl_->visit(frame);}\r
 376 void image_mixer::pop(){impl_->pop();}\r
 377 boost::shared_future<boost::iterator_range<const uint8_t*>> image_mixer::operator()(const core::video_format_desc& format_desc){return impl_->render(format_desc);}\r
 378 void image_mixer::begin_layer(core::blend_mode blend_mode){impl_->begin_layer(blend_mode);}\r
 379 void image_mixer::end_layer(){impl_->end_layer();}\r
 380 spl::shared_ptr<core::write_frame> image_mixer::create_frame(const void* tag, const core::pixel_format_desc& desc) {return impl_->create_frame(tag, desc);}\r
 381 \r
 382 }}}