git.sesse.net Git - casparcg/blob - core/mixer/image/image_kernel.cpp

   1 /*\r
   2 * copyright (c) 2010 Sveriges Television AB <info@casparcg.com>\r
   3 *\r
   4 *  This file is part of CasparCG.\r
   5 *\r
   6 *    CasparCG is free software: you can redistribute it and/or modify\r
   7 *    it under the terms of the GNU General Public License as published by\r
   8 *    the Free Software Foundation, either version 3 of the License, or\r
   9 *    (at your option) any later version.\r
  10 *\r
  11 *    CasparCG is distributed in the hope that it will be useful,\r
  12 *    but WITHOUT ANY WARRANTY; without even the implied warranty of\r
  13 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\r
  14 *    GNU General Public License for more details.\r
  15 \r
  16 *    You should have received a copy of the GNU General Public License\r
  17 *    along with CasparCG.  If not, see <http://www.gnu.org/licenses/>.\r
  18 *\r
  19 */\r
  20 #include "../../stdafx.h"\r
  21 \r
  22 #include "image_kernel.h"\r
  23 #include "blending_glsl.h"\r
  24 #include "../gpu/shader.h"\r
  25 #include "../gpu/device_buffer.h"\r
  26 \r
  27 #include <common/exception/exceptions.h>\r
  28 #include <common/gl/gl_check.h>\r
  29 \r
  30 #include <core/video_format.h>\r
  31 #include <core/producer/frame/pixel_format.h>\r
  32 #include <core/producer/frame/image_transform.h>\r
  33 \r
  34 #include <boost/noncopyable.hpp>\r
  35 \r
  36 #include <unordered_map>\r
  37 \r
  38 namespace caspar { namespace core {\r
  39         \r
  40 struct image_kernel::implementation : boost::noncopyable\r
  41 {       \r
  42         std::unique_ptr<shader> shader_;\r
  43         \r
  44         implementation()\r
  45         {\r
  46                 std::string vertex = \r
  47                         "void main()                                                                                                                                            \n"\r
  48                         "{                                                                                                                                                                      \n"\r
  49                         "       gl_TexCoord[0] = gl_MultiTexCoord0;                                                                                             \n"\r
  50                         "       gl_TexCoord[1] = gl_MultiTexCoord1;                                                                                             \n"\r
  51                         "       gl_FrontColor  = gl_Color;                                                                                                              \n"\r
  52                         "       gl_Position    = ftransform();                                                                                                  \n"\r
  53                         "}                                                                                                                                                                      \n";\r
  54 \r
  55                 std::string fragment = std::string() +\r
  56                         "uniform sampler2D      background;                                                                                                             \n"\r
  57                         "uniform sampler2D      plane[4];                                                                                                               \n"\r
  58                         "uniform sampler2D      local_key;                                                                                                              \n"\r
  59                         "uniform sampler2D      layer_key;                                                                                                              \n"\r
  60                         "                                                                                                                                                                       \n"\r
  61                         "uniform bool           is_hd;                                                                                                                  \n"\r
  62                         "uniform bool           has_local_key;                                                                                                  \n"\r
  63                         "uniform bool           has_layer_key;                                                                                                  \n"\r
  64                         "uniform int            blend_mode;                                                                                                             \n"\r
  65                         "uniform int            alpha_mode;                                                                                                             \n"\r
  66                         "uniform int            interlace_mode;                                                                                                 \n"\r
  67                         "uniform int            pixel_format;                                                                                                   \n"\r
  68                         "                                                                                                                                                                       \n"\r
  69                         "uniform float          gain;                                                                                                                   \n"\r
  70                         "                                                                                                                                                                       \n"\r
  71                         "uniform bool           levels;                                                                                                                 \n"\r
  72                         "uniform float          min_input;                                                                                                              \n"\r
  73                         "uniform float          max_input;                                                                                                              \n"\r
  74                         "uniform float          gamma;                                                                                                                  \n"\r
  75                         "uniform float          min_output;                                                                                                             \n"\r
  76                         "uniform float          max_output;                                                                                                             \n"\r
  77                         "                                                                                                                                                                       \n"\r
  78                         "uniform bool           csb;                                                                                                                    \n"\r
  79                         "uniform float          brt;                                                                                                                    \n"\r
  80                         "uniform float          sat;                                                                                                                    \n"\r
  81                         "uniform float          con;                                                                                                                    \n"\r
  82                         "                                                                                                                                                                       \n"\r
  83 \r
  84                         +\r
  85 \r
  86                         get_blend_glsl()\r
  87                         \r
  88                         +\r
  89 \r
  90                         "vec3 get_blend_color(vec3 back, vec3 fore)                                                                                     \n"\r
  91                         "{                                                                                                                                                                      \n"\r
  92                         "       switch(blend_mode)                                                                                                                              \n"\r
  93                         "       {                                                                                                                                                               \n"\r
  94                         "       case  0: return BlendNormal(back, fore);                                                                                \n"\r
  95                         "       case  1: return BlendLighten(back, fore);                                                                               \n"\r
  96                         "       case  2: return BlendDarken(back, fore);                                                                                \n"\r
  97                         "       case  3: return BlendMultiply(back, fore);                                                                              \n"\r
  98                         "       case  4: return BlendAverage(back, fore);                                                                               \n"\r
  99                         "       case  5: return BlendAdd(back, fore);                                                                                   \n"\r
 100                         "       case  6: return BlendSubstract(back, fore);                                                                             \n"\r
 101                         "       case  7: return BlendDifference(back, fore);                                                                    \n"\r
 102                         "       case  8: return BlendNegation(back, fore);                                                                              \n"\r
 103                         "       case  9: return BlendExclusion(back, fore);                                                                             \n"\r
 104                         "       case 10: return BlendScreen(back, fore);                                                                                \n"\r
 105                         "       case 11: return BlendOverlay(back, fore);                                                                               \n"\r
 106                         //"     case 12: return BlendSoftLight(back, fore);                                                                             \n"\r
 107                         "       case 13: return BlendHardLight(back, fore);                                                                             \n"\r
 108                         "       case 14: return BlendColorDodge(back, fore);                                                                    \n"\r
 109                         "       case 15: return BlendColorBurn(back, fore);                                                                             \n"\r
 110                         "       case 16: return BlendLinearDodge(back, fore);                                                                   \n"\r
 111                         "       case 17: return BlendLinearBurn(back, fore);                                                                    \n"\r
 112                         "       case 18: return BlendLinearLight(back, fore);                                                                   \n"\r
 113                         "       case 19: return BlendVividLight(back, fore);                                                                    \n"\r
 114                         "       case 20: return BlendPinLight(back, fore);                                                                              \n"\r
 115                         "       case 21: return BlendHardMix(back, fore);                                                                               \n"\r
 116                         "       case 22: return BlendReflect(back, fore);                                                                               \n"\r
 117                         "       case 23: return BlendGlow(back, fore);                                                                                  \n"\r
 118                         "       case 24: return BlendPhoenix(back, fore);                                                                               \n"\r
 119                         "       case 25: return BlendHue(back, fore);                                                                                   \n"\r
 120                         "       case 26: return BlendSaturation(back, fore);                                                                    \n"\r
 121                         "       case 27: return BlendColor(back, fore);                                                                                 \n"\r
 122                         "       case 28: return BlendLuminosity(back, fore);                                                                    \n"\r
 123                         "       }                                                                                                                                                               \n"\r
 124                         "                                                                                                                                                                       \n"\r
 125                         "       return BlendNormal(back, fore);                                                                                                 \n"\r
 126                         "}                                                                                                                                                                      \n"\r
 127                         "                                                                                                                                                                       \n"                                                                                                                                                       \r
 128                         "vec4 blend_color(vec4 fore)                                                                                                            \n"\r
 129                         "{                                                                                                                                                                      \n"\r
 130                         "   fore.rgb = fore.bgr; // bgr to rgb                                                                                          \n"\r
 131                         "   vec4 back = texture2D(background, gl_TexCoord[1].st);                                                       \n"\r
 132                         "   if(levels)                                                                                                                                          \n"\r
 133                         "               fore.rgb = LevelsControl(fore.rgb, min_input, max_input, gamma, min_output, max_output); \n"\r
 134                         "       if(csb)                                                                                                                                                 \n"\r
 135                         "               fore.rgb = ContrastSaturationBrightness(fore.rgb, brt, sat, con);                       \n"\r
 136                         "   fore.rgb = get_blend_color(back.bgr, fore.rgb);                                                                     \n"\r
 137                         "   fore.rgb = fore.bgr; // rgb to bgr                                                                                          \n"\r
 138                         "                                                                                                                                                                       \n"\r
 139                         "       return vec4(fore.rgb * fore.a + back.rgb * (1.0-fore.a), back.a + fore.a);              \n"\r
 140                         "}                                                                                                                                                                      \n"\r
 141                         "                                                                                                                                                                       \n"\r
 142                         "// NOTE: YCbCr, ITU-R, http://www.intersil.com/data/an/an9717.pdf                                      \n"\r
 143                         "// TODO: Support for more yuv formats might be needed.                                                         \n"\r
 144                         "vec4 ycbcra_to_bgra_sd(float y, float cb, float cr, float a)                                           \n"\r
 145                         "{                                                                                                                                                                      \n"\r
 146                         "       cb -= 0.5;                                                                                                                                              \n"\r
 147                         "       cr -= 0.5;                                                                                                                                              \n"\r
 148                         "       y = 1.164*(y-0.0625);                                                                                                                   \n"\r
 149                         "                                                                                                                                                                       \n"\r
 150                         "       vec4 color;                                                                                                                                             \n"\r
 151                         "       color.r = y + 1.596 * cr;                                                                                                               \n"\r
 152                         "       color.g = y - 0.813 * cr - 0.391 * cb;                                                                                  \n"\r
 153                         "       color.b = y + 2.018 * cb;                                                                                                               \n"\r
 154                         "       color.a = a;                                                                                                                                    \n"\r
 155                         "                                                                                                                                                                       \n"\r
 156                         "       return color;                                                                                                                                   \n"\r
 157                         "}                                                                                                                                                                      \n"                     \r
 158                         "                                                                                                                                                                       \n"\r
 159                         "vec4 ycbcra_to_bgra_hd(float y, float cb, float cr, float a)                                           \n"\r
 160                         "{                                                                                                                                                                      \n"\r
 161                         "       cb -= 0.5;                                                                                                                                              \n"\r
 162                         "       cr -= 0.5;                                                                                                                                              \n"\r
 163                         "       y = 1.164*(y-0.0625);                                                                                                                   \n"\r
 164                         "                                                                                                                                                                       \n"\r
 165                         "       vec4 color;                                                                                                                                             \n"\r
 166                         "       color.r = y + 1.793 * cr;                                                                                                               \n"\r
 167                         "       color.g = y - 0.534 * cr - 0.213 * cb;                                                                                  \n"\r
 168                         "       color.b = y + 2.115 * cb;                                                                                                               \n"\r
 169                         "       color.a = a;                                                                                                                                    \n"\r
 170                         "                                                                                                                                                                       \n"\r
 171                         "       return color;                                                                                                                                   \n"\r
 172                         "}                                                                                                                                                                      \n"                     \r
 173                         "                                                                                                                                                                       \n"                     \r
 174                         "vec4 ycbcra_to_bgra(float y, float cb, float cr, float a)                                                      \n"\r
 175                         "{                                                                                                                                                                      \n"\r
 176                         "       if(is_hd)                                                                                                                                               \n"\r
 177                         "               return ycbcra_to_bgra_hd(y, cb, cr, a);                                                                         \n"\r
 178                         "       else                                                                                                                                                    \n"\r
 179                         "               return ycbcra_to_bgra_sd(y, cb, cr, a);                                                                         \n"\r
 180                         "}                                                                                                                                                                      \n"\r
 181                         "                                                                                                                                                                       \n"\r
 182                         "vec4 get_bgra_color()                                                                                                                          \n"\r
 183                         "{                                                                                                                                                                      \n"\r
 184                         "       switch(pixel_format)                                                                                                                    \n"\r
 185                         "       {                                                                                                                                                               \n"\r
 186                         "       case 0:         //gray                                                                                                                          \n"\r
 187                         "               return vec4(texture2D(plane[0], gl_TexCoord[0].st).rrr, 1.0);                           \n"\r
 188                         "       case 1:         //bgra,                                                                                                                         \n"\r
 189                         "               return texture2D(plane[0], gl_TexCoord[0].st).rgba;                                                     \n"\r
 190                         "       case 2:         //rgba,                                                                                                                         \n"\r
 191                         "               return texture2D(plane[0], gl_TexCoord[0].st).bgra;                                                     \n"\r
 192                         "       case 3:         //argb,                                                                                                                         \n"\r
 193                         "               return texture2D(plane[0], gl_TexCoord[0].st).abgr;                                                     \n"\r
 194                         "       case 4:         //abgr,                                                                                                                         \n"\r
 195                         "               return texture2D(plane[0], gl_TexCoord[0].st).grab;                                                     \n"\r
 196                         "       case 5:         //ycbcr,                                                                                                                        \n"\r
 197                         "               {                                                                                                                                                       \n"\r
 198                         "                       float y  = texture2D(plane[0], gl_TexCoord[0].st).r;                                    \n"\r
 199                         "                       float cb = texture2D(plane[1], gl_TexCoord[0].st).r;                                    \n"\r
 200                         "                       float cr = texture2D(plane[2], gl_TexCoord[0].st).r;                                    \n"\r
 201                         "                       return ycbcra_to_bgra(y, cb, cr, 1.0);                                                                  \n"\r
 202                         "               }                                                                                                                                                       \n"\r
 203                         "       case 6:         //ycbcra                                                                                                                        \n"\r
 204                         "               {                                                                                                                                                       \n"\r
 205                         "                       float y  = texture2D(plane[0], gl_TexCoord[0].st).r;                                    \n"\r
 206                         "                       float cb = texture2D(plane[1], gl_TexCoord[0].st).r;                                    \n"\r
 207                         "                       float cr = texture2D(plane[2], gl_TexCoord[0].st).r;                                    \n"\r
 208                         "                       float a  = texture2D(plane[3], gl_TexCoord[0].st).r;                                    \n"\r
 209                         "                       return ycbcra_to_bgra(y, cb, cr, a);                                                                    \n"\r
 210                         "               }                                                                                                                                                       \n"\r
 211                         "       }                                                                                                                                                               \n"\r
 212                         "       return vec4(0.0, 0.0, 0.0, 0.0);                                                                                                \n"\r
 213                         "}                                                                                                                                                                      \n"\r
 214                         "                                                                                                                                                                       \n"\r
 215                         "void main()                                                                                                                                            \n"\r
 216                         "{                                                                                                                                                                      \n"\r
 217                         "   switch(interlace_mode)                                                                                                                      \n"\r
 218                         "       {                                                                                                                                                               \n"\r
 219                         "       case 1: // lower                                                                                                                                \n"\r
 220                         "               {                                                                                                                                                       \n"\r
 221                         "                       bool odd = mod(floor(gl_FragCoord.y), 2.0) > 0.5;                                               \n"\r
 222                         "                       if(!odd)                                                                                                                                \n"\r
 223                         "                               discard;                                                                                                                        \n"\r
 224                         "                       break;                                                                                                                                  \n"\r
 225                         "               }                                                                                                                                                       \n"\r
 226                         "       case 2: //upper                                                                                                                                 \n"\r
 227                         "               {                                                                                                                                                       \n"\r
 228                         "                       bool odd = mod(floor(gl_FragCoord.y), 2.0) > 0.5;                                               \n"\r
 229                         "                       if(odd)                                                                                                                                 \n"\r
 230                         "                               discard;                                                                                                                        \n"\r
 231                         "                       break;                                                                                                                                  \n"\r
 232                         "               }                                                                                                                                                       \n"\r
 233                         "       }                                                                                                                                                               \n"\r
 234                         "                                                                                                                                                                       \n"\r
 235                         "       vec4 color = get_bgra_color();                                                                                                  \n"\r
 236                         "       if(has_local_key)                                                                                                                               \n"\r
 237                         "               color.a *= texture2D(local_key, gl_TexCoord[1].st).r;                                           \n"\r
 238                         "       if(has_layer_key)                                                                                                                               \n"\r
 239                         "               color.a *= texture2D(layer_key, gl_TexCoord[1].st).r;                                           \n"\r
 240                         "       gl_FragColor = blend_color(color * gl_Color * gain);                                                    \n"\r
 241                         "}                                                                                                                                                                      \n"\r
 242                         ;\r
 243 \r
 244                         shader_.reset(new shader(vertex, fragment));\r
 245         }\r
 246 \r
 247         \r
 248         void draw(size_t                                                                                width, \r
 249                           size_t                                                                                height, \r
 250                           const core::pixel_format_desc&                                pix_desc, \r
 251                           const core::image_transform&                                  transform, \r
 252                           const std::vector<safe_ptr<device_buffer>>&   planes, \r
 253                           const safe_ptr<device_buffer>&                                background,\r
 254                           const std::shared_ptr<device_buffer>&                 local_key,                        \r
 255                           const std::shared_ptr<device_buffer>&                 layer_key)\r
 256         {\r
 257                 GL(glEnable(GL_TEXTURE_2D));\r
 258                         \r
 259                 // Setup depth for interlacing and explicit z-culling\r
 260                 double z = 0.0;\r
 261                 if(transform.get_mode() == core::video_mode::upper)\r
 262                 {       \r
 263                         GL(glEnable(GL_DEPTH_TEST));    \r
 264                         z = 1.0;\r
 265                 }\r
 266                 else if(transform.get_mode() == core::video_mode::lower)\r
 267                         GL(glClear(GL_DEPTH_BUFFER_BIT));               \r
 268 \r
 269                 // Bind textures\r
 270 \r
 271                 for(size_t n = 0; n < planes.size(); ++n)\r
 272                         planes[n]->bind(n);\r
 273 \r
 274                 if(local_key)\r
 275                         local_key->bind(4);\r
 276                 \r
 277                 if(layer_key)\r
 278                         layer_key->bind(5);\r
 279 \r
 280                 background->bind(6);\r
 281 \r
 282                 // Setup shader\r
 283 \r
 284                 shader_->use(); \r
 285 \r
 286                 shader_->set("plane[0]",                0);\r
 287                 shader_->set("plane[1]",                1);\r
 288                 shader_->set("plane[2]",                2);\r
 289                 shader_->set("plane[3]",                3);\r
 290                 shader_->set("local_key",               4);\r
 291                 shader_->set("layer_key",               5);\r
 292                 shader_->set("background",              6);\r
 293                 shader_->set("gain",                    transform.get_gain());\r
 294                 shader_->set("is_hd",                   pix_desc.planes.at(0).height > 700 ? 1 : 0);\r
 295                 shader_->set("has_local_key",   local_key ? 1 : 0);\r
 296                 shader_->set("has_layer_key",   layer_key ? 1 : 0);\r
 297                 shader_->set("blend_mode",              transform.get_blend_mode());\r
 298                 shader_->set("alpha_mode",              transform.get_alpha_mode());\r
 299                 shader_->set("interlace_mode",  transform.get_mode());\r
 300                 shader_->set("pixel_format",    pix_desc.pix_fmt);      \r
 301 \r
 302                 auto levels = transform.get_levels();\r
 303 \r
 304                 if(levels.min_input  > 0.001 ||\r
 305                    levels.max_input  < 0.999 ||\r
 306                    levels.min_output > 0.001 ||\r
 307                    levels.max_output < 0.999 ||\r
 308                    std::abs(levels.gamma - 1.0) > 0.001)\r
 309                 {\r
 310                         shader_->set("levels", true);   \r
 311                         shader_->set("min_input", levels.min_input);    \r
 312                         shader_->set("max_input", levels.max_input);\r
 313                         shader_->set("min_output", levels.min_output);\r
 314                         shader_->set("max_output", levels.max_output);\r
 315                         shader_->set("gamma", levels.gamma);\r
 316                 }\r
 317                 else\r
 318                         shader_->set("levels", false);  \r
 319 \r
 320                 if(std::abs(transform.get_brightness() - 1.0) > 0.001 ||\r
 321                    std::abs(transform.get_saturation() - 1.0) > 0.001 ||\r
 322                    std::abs(transform.get_contrast() - 1.0) > 0.001)\r
 323                 {\r
 324                         shader_->set("csb",     true);  \r
 325                         \r
 326                         shader_->set("brt", transform.get_brightness());        \r
 327                         shader_->set("sat", transform.get_saturation());\r
 328                         shader_->set("con", transform.get_contrast());\r
 329                 }\r
 330                 else\r
 331                         shader_->set("csb",     false); \r
 332                 \r
 333                 // Setup drawing area\r
 334 \r
 335                 GL(glColor4d(1.0, 1.0, 1.0, transform.get_opacity()));\r
 336                 GL(glViewport(0, 0, width, height));\r
 337                                                 \r
 338                 auto m_p = transform.get_clip_translation();\r
 339                 auto m_s = transform.get_clip_scale();\r
 340                 double w = static_cast<double>(width);\r
 341                 double h = static_cast<double>(height);\r
 342 \r
 343                 GL(glEnable(GL_SCISSOR_TEST));\r
 344                 GL(glScissor(static_cast<size_t>(m_p[0]*w), static_cast<size_t>(m_p[1]*h), static_cast<size_t>(m_s[0]*w), static_cast<size_t>(m_s[1]*h)));\r
 345                         \r
 346                 auto f_p = transform.get_fill_translation();\r
 347                 auto f_s = transform.get_fill_scale();\r
 348                 \r
 349                 // Draw\r
 350 \r
 351                 glBegin(GL_QUADS);\r
 352                         glMultiTexCoord2d(GL_TEXTURE0, 0.0, 0.0); glMultiTexCoord2d(GL_TEXTURE1,  f_p[0]        ,  f_p[1]        );             glVertex3d( f_p[0]        *2.0-1.0,  f_p[1]        *2.0-1.0, z);\r
 353                         glMultiTexCoord2d(GL_TEXTURE0, 1.0, 0.0); glMultiTexCoord2d(GL_TEXTURE1, (f_p[0]+f_s[0]),  f_p[1]        );             glVertex3d((f_p[0]+f_s[0])*2.0-1.0,  f_p[1]        *2.0-1.0, z);\r
 354                         glMultiTexCoord2d(GL_TEXTURE0, 1.0, 1.0); glMultiTexCoord2d(GL_TEXTURE1, (f_p[0]+f_s[0]), (f_p[1]+f_s[1]));             glVertex3d((f_p[0]+f_s[0])*2.0-1.0, (f_p[1]+f_s[1])*2.0-1.0, z);\r
 355                         glMultiTexCoord2d(GL_TEXTURE0, 0.0, 1.0); glMultiTexCoord2d(GL_TEXTURE1,  f_p[0]        , (f_p[1]+f_s[1]));             glVertex3d( f_p[0]        *2.0-1.0, (f_p[1]+f_s[1])*2.0-1.0, z);\r
 356                 glEnd();\r
 357 \r
 358                 GL(glDisable(GL_SCISSOR_TEST)); \r
 359                 GL(glDisable(GL_DEPTH_TEST));   \r
 360         }\r
 361 };\r
 362 \r
 363 image_kernel::image_kernel() : impl_(new implementation()){}\r
 364 \r
 365 void image_kernel::draw(size_t width, size_t height, const core::pixel_format_desc& pix_desc, const core::image_transform& transform, const std::vector<safe_ptr<device_buffer>>& planes, \r
 366                                                           const safe_ptr<device_buffer>& background, const std::shared_ptr<device_buffer>& local_key, const std::shared_ptr<device_buffer>& layer_key)\r
 367 {\r
 368         impl_->draw(width, height, pix_desc, transform, planes, background, local_key, layer_key);\r
 369 }\r
 370 \r
 371 }}