2.1.0: cpu/image_mixer: Added support for image conversion and blending. Only interla...

git-svn-id: https://casparcg.svn.sourceforge.net/svnroot/casparcg/server/branches/2.1.0@2314 362d55ac-95cf-4e76-9f9a-cbaa9c17b72d

diff --git a/accelerator/cpu/image/image_mixer.cpp b/accelerator/cpu/image/image_mixer.cpp
index e13fc5f2eba7114fab194a8a3a461e1b045cc0e2..8f4133fb72a19ad81d9eac51a69bb05cfc2704fb 100644 (file)
--- a/accelerator/cpu/image/image_mixer.cpp
+++ b/accelerator/cpu/image/image_mixer.cpp
@@ -24,6 +24,7 @@
 #include "image_mixer.h"\r
 \r
 #include "../util/write_frame.h"\r
+#include "../util/blend.h"\r
 \r
 #include <common/assert.h>\r
 #include <common/gl/gl_check.h>\r
@@ -35,12 +36,16 @@
 #include <core/frame/pixel_format.h>\r
 #include <core/video_format.h>\r
 \r
+#include <modules/ffmpeg/producer/util/util.h>\r
+\r
 #include <asmlib.h>\r
 \r
 #include <gl/glew.h>\r
 \r
 #include <tbb/cache_aligned_allocator.h>\r
+#include <tbb/parallel_for_each.h>\r
 \r
+#include <boost/assign.hpp>\r
 #include <boost/foreach.hpp>\r
 #include <boost/range.hpp>\r
 #include <boost/range/algorithm_ext/erase.hpp>\r
@@ -49,6 +54,20 @@
 #include <algorithm>\r
 #include <vector>\r
 \r
+#if defined(_MSC_VER)\r
+#pragma warning (push)\r
+#pragma warning (disable : 4244)\r
+#endif\r
+extern "C" \r
+{\r
+       #include <libswscale/swscale.h>\r
+       #include <libavcodec/avcodec.h>\r
+       #include <libavformat/avformat.h>\r
+}\r
+#if defined(_MSC_VER)\r
+#pragma warning (pop)\r
+#endif\r
+\r
 namespace caspar { namespace accelerator { namespace cpu {\r
                \r
 struct item\r
@@ -99,9 +118,9 @@ bool operator!=(const layer& lhs, const layer& rhs)
 \r
 class image_renderer\r
 {\r
-       std::pair<std::vector<layer>, boost::shared_future<boost::iterator_range<const uint8_t*>>> last_image_;\r
-public:\r
-       \r
+       std::pair<std::vector<layer>, boost::shared_future<boost::iterator_range<const uint8_t*>>>      last_image_;\r
+       std::map<int, std::shared_ptr<SwsContext>>                                                                                                      sws_contexts_;\r
+public:        \r
        boost::shared_future<boost::iterator_range<const uint8_t*>> operator()(std::vector<layer> layers, const core::video_format_desc& format_desc)\r
        {       \r
                if(last_image_.first == layers && last_image_.second.has_value())\r
@@ -112,190 +131,83 @@ public:
                return image;\r
        }\r
 \r
-private:       \r
+private:\r
        boost::shared_future<boost::iterator_range<const uint8_t*>> render(std::vector<layer> layers, const core::video_format_desc& format_desc)\r
-       {       \r
+       {\r
                static const auto empty = spl::make_shared<const std::vector<uint8_t, tbb::cache_aligned_allocator<uint8_t>>>(2048*2048*4, 0);\r
                CASPAR_VERIFY(empty->size() >= format_desc.size);\r
                \r
-               if(layers.empty())\r
+               std::vector<item> items;\r
+               BOOST_FOREACH(auto& layer, layers)\r
+                       items.insert(items.end(), layer.items.begin(), layer.items.end());\r
+\r
+               if(items.empty())\r
                {\r
                        return async(launch_policy::deferred, [=]\r
                        {\r
                                return boost::iterator_range<const uint8_t*>(empty->data(), empty->data() + format_desc.size);\r
-                       });\r
+                       });             \r
                }\r
-               else if(layers.size()                           == 1 &&\r
-                           layers.at(0).items.size()   == 1 &&\r
-                           layers.at(0).items.at(0).pix_desc.format            == core::pixel_format::bgra &&\r
-                           layers.at(0).items.at(0).buffers.at(0)->size() == format_desc.size &&\r
-                           layers.at(0).items.at(0).transform                          == core::frame_transform())\r
+\r
+               convert(items.begin(), items.end(), format_desc);                       \r
+               blend(items.begin(), items.end());\r
+               \r
+               auto buffer = items.front().buffers.at(0);\r
+               return async(launch_policy::deferred, [=]\r
                {\r
-                       auto buffer = layers.at(0).items.at(0).buffers.at(0);\r
-                       return async(launch_policy::deferred, [=]\r
-                       {\r
-                               return boost::iterator_range<const uint8_t*>(buffer->data(), buffer->data() + format_desc.size);\r
-                       });\r
-               }\r
-               else\r
+                       return boost::iterator_range<const uint8_t*>(buffer->data(), buffer->data() + format_desc.size);\r
+               });             \r
+       }\r
+\r
+       template<typename I>\r
+       void blend(I begin, I end)\r
+       {\r
+               for(auto it = begin + 1; it != end; ++it)\r
                {\r
-                       return async(launch_policy::deferred, [=]\r
-                       {\r
-                               return boost::iterator_range<const uint8_t*>(empty->data(), empty->data() + format_desc.size);\r
-                       });\r
+                       auto size    = begin->buffers.at(0)->size();\r
+                       auto dest    = begin->buffers.at(0)->data();\r
+                       auto source2 = it->buffers.at(0)->data();\r
+                       cpu::blend(dest, dest, source2, size);\r
                }\r
-               //else\r
-               //{                             \r
-                       //auto draw_buffer = create_mixer_buffer(4, format_desc);\r
-\r
-                       //if(format_desc.field_mode != core::field_mode::progressive)\r
-                       //{\r
-                       //      auto upper = layers;\r
-                       //      auto lower = std::move(layers);\r
-\r
-                       //      BOOST_FOREACH(auto& layer, upper)\r
-                       //      {\r
-                       //              BOOST_FOREACH(auto& item, layer.items)\r
-                       //                      item.transform.field_mode &= core::field_mode::upper;\r
-                       //      }\r
-\r
-                       //      BOOST_FOREACH(auto& layer, lower)\r
-                       //      {\r
-                       //              BOOST_FOREACH(auto& item, layer.items)\r
-                       //                      item.transform.field_mode &= core::field_mode::lower;\r
-                       //      }\r
-\r
-                       //      draw(std::move(upper), draw_buffer, format_desc);\r
-                       //      draw(std::move(lower), draw_buffer, format_desc);\r
-                       //}\r
-                       //else\r
-                       //{\r
-                       //      draw(std::move(layers), draw_buffer, format_desc);\r
-                       //}\r
-                       //                                              \r
-                       //return draw_buffer;\r
-               //}\r
        }\r
        \r
-       //void draw(std::vector<layer>&&                                layers, \r
-       //                spl::shared_ptr<device_buffer>&       draw_buffer, \r
-       //                const core::video_format_desc&        format_desc)\r
-       //{\r
-       //      std::shared_ptr<device_buffer> layer_key_buffer;\r
-\r
-       //      BOOST_FOREACH(auto& layer, layers)\r
-       //              draw_layer(std::move(layer), draw_buffer, layer_key_buffer, format_desc);\r
-       //}\r
-\r
-       //void draw_layer(layer&&                                                               layer, \r
-       //                              spl::shared_ptr<device_buffer>&         draw_buffer,\r
-       //                              std::shared_ptr<device_buffer>&         layer_key_buffer,\r
-       //                              const core::video_format_desc&          format_desc)\r
-       //{             \r
-       //      // Remove empty items.\r
-       //      boost::range::remove_erase_if(layer.items, [](const item& item)\r
-       //      {\r
-       //              return item.transform.field_mode == core::field_mode::empty;\r
-       //      });\r
-\r
-       //      if(layer.items.empty())\r
-       //              return;\r
-\r
-       //      std::shared_ptr<device_buffer> local_key_buffer;\r
-       //      std::shared_ptr<device_buffer> local_mix_buffer;\r
-       //                      \r
-       //      if(layer.blend_mode != core::blend_mode::normal)\r
-       //      {\r
-       //              auto layer_draw_buffer = create_mixer_buffer(4, format_desc);\r
-\r
-       //              BOOST_FOREACH(auto& item, layer.items)\r
-       //                      draw_item(std::move(item), layer_draw_buffer, layer_key_buffer, local_key_buffer, local_mix_buffer, format_desc);       \r
-       //      \r
-       //              draw_mixer_buffer(layer_draw_buffer, std::move(local_mix_buffer), core::blend_mode::normal);                                                    \r
-       //              draw_mixer_buffer(draw_buffer, std::move(layer_draw_buffer), layer.blend_mode);\r
-       //      }\r
-       //      else // fast path\r
-       //      {\r
-       //              BOOST_FOREACH(auto& item, layer.items)          \r
-       //                      draw_item(std::move(item), draw_buffer, layer_key_buffer, local_key_buffer, local_mix_buffer, format_desc);             \r
-       //                              \r
-       //              draw_mixer_buffer(draw_buffer, std::move(local_mix_buffer), core::blend_mode::normal);\r
-       //      }                                       \r
-\r
-       //      layer_key_buffer = std::move(local_key_buffer);\r
-       //}\r
-\r
-       //void draw_item(item&&                                                 item, \r
-       //                         spl::shared_ptr<device_buffer>&      draw_buffer, \r
-       //                         std::shared_ptr<device_buffer>&      layer_key_buffer, \r
-       //                         std::shared_ptr<device_buffer>&      local_key_buffer, \r
-       //                         std::shared_ptr<device_buffer>&      local_mix_buffer,\r
-       //                         const core::video_format_desc&       format_desc)\r
-       //{                     \r
-       //      draw_params draw_params;\r
-       //      draw_params.pix_desc    = std::move(item.pix_desc);\r
-       //      draw_params.transform   = std::move(item.transform);\r
-       //      BOOST_FOREACH(auto& future_texture, item.textures)\r
-       //              draw_params.textures.push_back(future_texture.get());\r
-\r
-       //      if(item.transform.is_key)\r
-       //      {\r
-       //              local_key_buffer = local_key_buffer ? local_key_buffer : create_mixer_buffer(1, format_desc);\r
-\r
-       //              draw_params.background                  = local_key_buffer;\r
-       //              draw_params.local_key                   = nullptr;\r
-       //              draw_params.layer_key                   = nullptr;\r
-\r
-       //              kernel_.draw(std::move(draw_params));\r
-       //      }\r
-       //      else if(item.transform.is_mix)\r
-       //      {\r
-       //              local_mix_buffer = local_mix_buffer ? local_mix_buffer : create_mixer_buffer(4, format_desc);\r
-\r
-       //              draw_params.background                  = local_mix_buffer;\r
-       //              draw_params.local_key                   = std::move(local_key_buffer);\r
-       //              draw_params.layer_key                   = layer_key_buffer;\r
-\r
-       //              draw_params.keyer                               = keyer::additive;\r
-\r
-       //              kernel_.draw(std::move(draw_params));\r
-       //      }\r
-       //      else\r
-       //      {\r
-       //              draw_mixer_buffer(draw_buffer, std::move(local_mix_buffer), core::blend_mode::normal);\r
-       //              \r
-       //              draw_params.background                  = draw_buffer;\r
-       //              draw_params.local_key                   = std::move(local_key_buffer);\r
-       //              draw_params.layer_key                   = layer_key_buffer;\r
-\r
-       //              kernel_.draw(std::move(draw_params));\r
-       //      }       \r
-       //}\r
-\r
-       //void draw_mixer_buffer(spl::shared_ptr<device_buffer>&        draw_buffer, \r
-       //                                         std::shared_ptr<device_buffer>&& source_buffer, \r
-       //                                         core::blend_mode                                     blend_mode = core::blend_mode::normal)\r
-       //{\r
-       //      if(!source_buffer)\r
-       //              return;\r
-\r
-       //      draw_params draw_params;\r
-       //      draw_params.pix_desc.format             = core::pixel_format::bgra;\r
-       //      draw_params.pix_desc.planes             = list_of(core::pixel_format_desc::plane(source_buffer->width(), source_buffer->height(), 4));\r
-       //      draw_params.textures                    = list_of(source_buffer);\r
-       //      draw_params.transform                   = core::frame_transform();\r
-       //      draw_params.blend_mode                  = blend_mode;\r
-       //      draw_params.background                  = draw_buffer;\r
-\r
-       //      kernel_.draw(std::move(draw_params));\r
-       //}\r
-       //              \r
-       //spl::shared_ptr<device_buffer> create_mixer_buffer(int stride, const core::video_format_desc& format_desc)\r
-       //{\r
-       //      auto buffer = ogl_->create_device_buffer(format_desc.width, format_desc.height, stride);\r
-       //      buffer->clear();\r
-       //      return buffer;\r
-       //}\r
+       template<typename I>\r
+       void convert(I begin, I end, const core::video_format_desc& format_desc)\r
+       {\r
+               tbb::parallel_for_each(begin, end, [&](item& item)\r
+               {\r
+                       if(item.pix_desc.format == core::pixel_format::bgra)\r
+                               return;\r
+\r
+                       auto input_av_frame = ffmpeg::make_av_frame(item.buffers, item.pix_desc);\r
+                                                               \r
+                       int key = ((input_av_frame->width << 22) & 0xFFC00000) | ((input_av_frame->height << 6) & 0x003FC000) | ((input_av_frame->format << 7) & 0x00007F00);\r
+                                                                       \r
+                       auto& sws_context = sws_contexts_[key];\r
+                       if(!sws_context)\r
+                       {\r
+                               double param;\r
+                               sws_context.reset(sws_getContext(input_av_frame->width, input_av_frame->height, static_cast<PixelFormat>(input_av_frame->format), format_desc.width, format_desc.height, PIX_FMT_BGRA, SWS_BILINEAR, nullptr, nullptr, &param), sws_freeContext);\r
+                       }\r
+                       \r
+                       if(!sws_context)                                \r
+                               BOOST_THROW_EXCEPTION(operation_failed() << msg_info("Could not create software scaling context.") << boost::errinfo_api_function("sws_getContext"));                           \r
+               \r
+                       auto dest = spl::make_shared<host_buffer>(format_desc.size);\r
+\r
+                       spl::shared_ptr<AVFrame> av_frame(avcodec_alloc_frame(), av_free);      \r
+                       avcodec_get_frame_defaults(av_frame.get());                     \r
+                       avpicture_fill(reinterpret_cast<AVPicture*>(av_frame.get()), dest->data(), PIX_FMT_BGRA, format_desc.width, format_desc.height);\r
+                               \r
+                       sws_scale(sws_context.get(), input_av_frame->data, input_av_frame->linesize, 0, input_av_frame->height, av_frame->data, av_frame->linesize);    \r
+\r
+                       item.buffers.clear();\r
+                       item.buffers.push_back(dest);\r
+                       item.pix_desc = core::pixel_format_desc(core::pixel_format::bgra);\r
+                       item.pix_desc.planes.clear();\r
+                       item.pix_desc.planes.push_back(core::pixel_format_desc::plane(format_desc.width, format_desc.height, 4));\r
+               });\r
+       }\r
 };\r
                \r
 struct image_mixer::impl : boost::noncopyable\r

diff --git a/accelerator/cpu/util/blend.h b/accelerator/cpu/util/blend.h
index b08a7bfe8501778db17e2dd06d9026f1adafc895..3e749ac2c646f58de6ec736a521918fafe88c8df 100644 (file)
--- a/accelerator/cpu/util/blend.h
+++ b/accelerator/cpu/util/blend.h
@@ -35,7 +35,7 @@ static void blend(uint8_t* dest, const uint8_t* source1, const uint8_t* source2,
     const xmm_epi16 round  = 128;\r
     const xmm_epi16 lomask = 0x00FF;\r
                \r
-       tbb::parallel_for(tbb::blocked_range<int>(0, count/sizeof(xmm_epi8)), [&](const tbb::blocked_range<int>& r)\r
+       tbb::parallel_for(tbb::blocked_range<size_t>(0, count/sizeof(xmm_epi8)), [&](const tbb::blocked_range<size_t>& r)\r
        {\r
                for(auto n = r.begin(); n != r.end(); ++n)    \r
                {\r

diff --git a/modules/ffmpeg/producer/util/util.cpp b/modules/ffmpeg/producer/util/util.cpp
index b4e647767f4de8dc077714309b68ae2df6709a6b..f06e3084655a42edf6813978c37df3149d0c3e01 100644 (file)
--- a/modules/ffmpeg/producer/util/util.cpp
+++ b/modules/ffmpeg/producer/util/util.cpp
@@ -289,6 +289,74 @@ spl::shared_ptr<core::write_frame> make_write_frame(const void* tag, const spl::
        return spl::make_shared_ptr(write);\r
 }\r
 \r
+spl::shared_ptr<AVFrame> make_av_frame(caspar::core::data_frame& frame)\r
+{\r
+       std::array<void*, 4> data = {};\r
+       for(int n = 0; n < frame.get_pixel_format_desc().planes.size(); ++n)\r
+               data[n] = frame.image_data(n).begin();\r
+\r
+       return make_av_frame(data, frame.get_pixel_format_desc());\r
+}\r
+\r
+spl::shared_ptr<AVFrame> make_av_frame(std::array<void*, 4> data, const core::pixel_format_desc& pix_desc)\r
+{\r
+       spl::shared_ptr<AVFrame> av_frame(avcodec_alloc_frame(), av_free);      \r
+       avcodec_get_frame_defaults(av_frame.get());\r
+       \r
+       auto planes              = pix_desc.planes;\r
+       auto format              = pix_desc.format.value();\r
+\r
+       av_frame->width  = planes[0].width;\r
+       av_frame->height = planes[0].height;\r
+       for(int n = 0; n < planes.size(); ++n)  \r
+       {\r
+               av_frame->data[n]         = reinterpret_cast<uint8_t*>(data[n]);\r
+               av_frame->linesize[n] = planes[n].linesize;     \r
+       }\r
+       switch(format)\r
+       {\r
+       case core::pixel_format::rgba:\r
+               av_frame->format = PIX_FMT_RGBA; \r
+               break;\r
+       case core::pixel_format::argb:\r
+               av_frame->format = PIX_FMT_ARGB; \r
+               break;\r
+       case core::pixel_format::bgra:\r
+               av_frame->format = PIX_FMT_BGRA; \r
+               break;\r
+       case core::pixel_format::abgr:\r
+               av_frame->format = PIX_FMT_ABGR; \r
+               break;\r
+       case core::pixel_format::gray:\r
+               av_frame->format = PIX_FMT_GRAY8; \r
+               break;\r
+       case core::pixel_format::ycbcr:\r
+       {\r
+               int y_w = planes[0].width;\r
+               int y_h = planes[0].height;\r
+               int c_w = planes[1].width;\r
+               int c_h = planes[1].height;\r
+\r
+               if(c_h == y_h && c_w == y_w)\r
+                       av_frame->format = PIX_FMT_YUV444P;\r
+               else if(c_h == y_h && c_w*2 == y_w)\r
+                       av_frame->format = PIX_FMT_YUV422P;\r
+               else if(c_h == y_h && c_w*4 == y_w)\r
+                       av_frame->format = PIX_FMT_YUV411P;\r
+               else if(c_h*2 == y_h && c_w*2 == y_w)\r
+                       av_frame->format = PIX_FMT_YUV420P;\r
+               else if(c_h*2 == y_h && c_w*4 == y_w)\r
+                       av_frame->format = PIX_FMT_YUV410P;\r
+\r
+               break;\r
+       }\r
+       case core::pixel_format::ycbcra:\r
+               av_frame->format = PIX_FMT_YUVA420P;\r
+               break;\r
+       }\r
+       return av_frame;\r
+}\r
+\r
 bool is_sane_fps(AVRational time_base)\r
 {\r
        double fps = static_cast<double>(time_base.den) / static_cast<double>(time_base.num);\r

diff --git a/modules/ffmpeg/producer/util/util.h b/modules/ffmpeg/producer/util/util.h
index 815db5876cfcb56d928ed4c9807ab347300741b5..06d843c836d7a28b31d6a6ff2a3da1c95e6bd2a8 100644 (file)
--- a/modules/ffmpeg/producer/util/util.h
+++ b/modules/ffmpeg/producer/util/util.h
@@ -28,6 +28,8 @@
 #include <core/frame/pixel_format.h>\r
 #include <core/mixer/audio/audio_mixer.h>\r
 \r
+#include <array>\r
+\r
 enum PixelFormat;\r
 struct AVFrame;\r
 struct AVFormatContext;\r
@@ -37,6 +39,7 @@ struct AVCodecContext;
 \r
 FORWARD2(caspar, core, struct pixel_format_desc);\r
 FORWARD2(caspar, core, struct write_frame);\r
+FORWARD2(caspar, core, struct data_frame);\r
 FORWARD2(caspar, core, struct frame_factory);\r
 \r
 namespace caspar { namespace ffmpeg {\r
@@ -53,6 +56,19 @@ static const int CASPAR_PIX_FMT_LUMA = 10; // Just hijack some unual pixel forma
 core::field_mode                                       get_mode(const AVFrame& frame);\r
 int                                                                    make_alpha_format(int format); // NOTE: Be careful about CASPAR_PIX_FMT_LUMA, change it to PIX_FMT_GRAY8 if you want to use the frame inside some ffmpeg function.\r
 spl::shared_ptr<core::write_frame>     make_write_frame(const void* tag, const spl::shared_ptr<AVFrame>& decoded_frame, const spl::shared_ptr<core::frame_factory>& frame_factory, int flags);\r
+spl::shared_ptr<AVFrame>                       make_av_frame(caspar::core::data_frame& frame);\r
+spl::shared_ptr<AVFrame>                       make_av_frame(std::array<void*, 4> data, const core::pixel_format_desc& pix_desc);\r
+\r
+template<typename C>\r
+std::shared_ptr<AVFrame> make_av_frame(const C& container, const core::pixel_format_desc& pix_desc)\r
+{      \r
+       std::array<void*, 4> data = {};\r
+       int n = 0;\r
+       for(auto it = std::begin(container); it != std::end(container); ++it)\r
+               data[n++] = (*it)->data();\r
+       return make_av_frame(data, pix_desc);\r
+}\r
+\r
 core::pixel_format_desc                                get_pixel_format_desc(PixelFormat pix_fmt, int width, int height);\r
 \r
 spl::shared_ptr<AVPacket> create_packet();\r