git.sesse.net Git - casparcg/blob - accelerator/ogl/image/image_kernel.cpp

   1 /*
   2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
   3 *
   4 * This file is part of CasparCG (www.casparcg.com).
   5 *
   6 * CasparCG is free software: you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation, either version 3 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * CasparCG is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
  18 *
  19 * Author: Robert Nagy, ronag89@gmail.com
  20 */
  21
  22 #include "../../StdAfx.h"
  23
  24 #include "image_kernel.h"
  25
  26 #include "image_shader.h"
  27 #include "blending_glsl.h"
  28
  29 #include "../util/shader.h"
  30 #include "../util/texture.h"
  31 #include "../util/device.h"
  32
  33 #include <common/except.h>
  34 #include <common/gl/gl_check.h>
  35 #include <common/env.h>
  36
  37 #include <core/video_format.h>
  38 #include <core/frame/pixel_format.h>
  39 #include <core/frame/frame_transform.h>
  40
  41 #include <boost/algorithm/cxx11/all_of.hpp>
  42 #include <boost/lexical_cast.hpp>
  43 #include <boost/range/adaptor/transformed.hpp>
  44
  45 #include <cmath>
  46
  47 namespace caspar { namespace accelerator { namespace ogl {
  48
  49 // http://stackoverflow.com/questions/563198/how-do-you-detect-where-two-line-segments-intersect
  50 bool get_line_intersection(
  51                 double p0_x, double p0_y,
  52                 double p1_x, double p1_y,
  53                 double p2_x, double p2_y,
  54                 double p3_x, double p3_y,
  55                 double& result_x, double& result_y)
  56 {
  57         double s1_x = p1_x - p0_x;
  58         double s1_y = p1_y - p0_y;
  59         double s2_x = p3_x - p2_x;
  60         double s2_y = p3_y - p2_y;
  61
  62         double s = (-s1_y * (p0_x - p2_x) + s1_x * (p0_y - p2_y)) / (-s2_x * s1_y + s1_x * s2_y);
  63         double t = (s2_x * (p0_y - p2_y) - s2_y * (p0_x - p2_x)) / (-s2_x * s1_y + s1_x * s2_y);
  64
  65         if (s >= 0 && s <= 1 && t >= 0 && t <= 1)
  66         {
  67                 // Collision detected
  68                 result_x = p0_x + (t * s1_x);
  69                 result_y = p0_y + (t * s1_y);
  70
  71                 return true;
  72         }
  73
  74         return false; // No collision
  75 }
  76
  77 double hypotenuse(double x1, double y1, double x2, double y2)
  78 {
  79         auto x = x2 - x1;
  80         auto y = y2 - y1;
  81
  82         return std::sqrt(x * x + y * y);
  83 }
  84
  85 double calc_q(double close_diagonal, double distant_diagonal)
  86 {
  87         return (close_diagonal + distant_diagonal) / distant_diagonal;
  88 }
  89
  90 bool is_above_screen(double y)
  91 {
  92         return y < 0.0;
  93 }
  94
  95 bool is_below_screen(double y)
  96 {
  97         return y > 1.0;
  98 }
  99
 100 bool is_left_of_screen(double x)
 101 {
 102         return x < 0.0;
 103 }
 104
 105 bool is_right_of_screen(double x)
 106 {
 107         return x > 1.0;
 108 }
 109
 110 bool is_outside_screen(const std::vector<core::frame_geometry::coord>& coords)
 111 {
 112         auto x_coords = coords | boost::adaptors::transformed([](const core::frame_geometry::coord& c) { return c.vertex_x; });
 113         auto y_coords = coords | boost::adaptors::transformed([](const core::frame_geometry::coord& c) { return c.vertex_y; });
 114
 115         return boost::algorithm::all_of(x_coords, &is_left_of_screen)
 116                 || boost::algorithm::all_of(x_coords, &is_right_of_screen)
 117                 || boost::algorithm::all_of(y_coords, &is_above_screen)
 118                 || boost::algorithm::all_of(y_coords, &is_below_screen);
 119 }
 120
 121 GLubyte upper_pattern[] = {
 122         0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
 123         0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
 124         0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
 125         0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00};
 126
 127 GLubyte lower_pattern[] = {
 128         0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
 129         0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
 130         0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
 131         0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff};
 132
 133 struct image_kernel::impl
 134 {
 135         spl::shared_ptr<device> ogl_;
 136         spl::shared_ptr<shader> shader_;
 137         bool                                    blend_modes_;
 138         bool                                    post_processing_;
 139         bool                                    supports_texture_barrier_       = glTextureBarrierNV != 0;
 140
 141         impl(const spl::shared_ptr<device>& ogl, bool blend_modes_wanted, bool straight_alpha_wanted)
 142                 : ogl_(ogl)
 143                 , shader_(ogl_->invoke([&]{return get_image_shader(ogl, blend_modes_, blend_modes_wanted, post_processing_, straight_alpha_wanted); }))
 144         {
 145                 if (!supports_texture_barrier_)
 146                         CASPAR_LOG(warning) << L"[image_mixer] TextureBarrierNV not supported. Post processing will not be available";
 147         }
 148
 149         void draw(draw_params params)
 150         {
 151                 static const double epsilon = 0.001;
 152
 153                 CASPAR_ASSERT(params.pix_desc.planes.size() == params.textures.size());
 154
 155                 if(params.textures.empty() || !params.background)
 156                         return;
 157
 158                 if(params.transform.opacity < epsilon)
 159                         return;
 160
 161                 auto coords = params.geometry.data();
 162
 163                 if (coords.empty())
 164                         return;
 165
 166                 // Calculate transforms
 167                 auto f_p = params.transform.fill_translation;
 168                 auto f_s = params.transform.fill_scale;
 169
 170                 bool is_default_geometry = boost::equal(coords, core::frame_geometry::get_default().data());
 171                 auto aspect = params.aspect_ratio;
 172                 auto angle = params.transform.angle;
 173                 auto anchor = params.transform.anchor;
 174                 auto crop = params.transform.crop;
 175                 auto pers = params.transform.perspective;
 176                 pers.ur[0] -= 1.0;
 177                 pers.lr[0] -= 1.0;
 178                 pers.lr[1] -= 1.0;
 179                 pers.ll[1] -= 1.0;
 180                 std::vector<boost::array<double, 2>> pers_corners = { pers.ul, pers.ur, pers.lr, pers.ll };
 181
 182                 auto do_crop = [&](core::frame_geometry::coord& coord)
 183                 {
 184                         if (!is_default_geometry)
 185                                 // TODO implement support for non-default geometry.
 186                                 return;
 187
 188                         coord.vertex_x = std::max(coord.vertex_x, crop.ul[0]);
 189                         coord.vertex_x = std::min(coord.vertex_x, crop.lr[0]);
 190                         coord.vertex_y = std::max(coord.vertex_y, crop.ul[1]);
 191                         coord.vertex_y = std::min(coord.vertex_y, crop.lr[1]);
 192                         coord.texture_x = std::max(coord.texture_x, crop.ul[0]);
 193                         coord.texture_x = std::min(coord.texture_x, crop.lr[0]);
 194                         coord.texture_y = std::max(coord.texture_y, crop.ul[1]);
 195                         coord.texture_y = std::min(coord.texture_y, crop.lr[1]);
 196                 };
 197                 auto do_perspective = [=](core::frame_geometry::coord& coord, const boost::array<double, 2>& pers_corner)
 198                 {
 199                         if (!is_default_geometry)
 200                                 // TODO implement support for non-default geometry.
 201                                 return;
 202
 203                         coord.vertex_x += pers_corner[0];
 204                         coord.vertex_y += pers_corner[1];
 205                 };
 206                 auto rotate = [&](core::frame_geometry::coord& coord)
 207                 {
 208                         auto orig_x = (coord.vertex_x - anchor[0]) * f_s[0];
 209                         auto orig_y = (coord.vertex_y - anchor[1]) * f_s[1] / aspect;
 210                         coord.vertex_x = orig_x * std::cos(angle) - orig_y * std::sin(angle);
 211                         coord.vertex_y = orig_x * std::sin(angle) + orig_y * std::cos(angle);
 212                         coord.vertex_y *= aspect;
 213                 };
 214                 auto move = [&](core::frame_geometry::coord& coord)
 215                 {
 216                         coord.vertex_x += f_p[0];
 217                         coord.vertex_y += f_p[1];
 218                 };
 219
 220                 int corner = 0;
 221                 for (auto& coord : coords)
 222                 {
 223                         do_crop(coord);
 224                         do_perspective(coord, pers_corners.at(corner));
 225                         rotate(coord);
 226                         move(coord);
 227
 228                         if (++corner == 4)
 229                                 corner = 0;
 230                 }
 231
 232                 // Skip drawing if all the coordinates will be outside the screen.
 233                 if (is_outside_screen(coords))
 234                         return;
 235
 236                 // Bind textures
 237
 238                 for(int n = 0; n < params.textures.size(); ++n)
 239                         params.textures[n]->bind(n);
 240
 241                 if(params.local_key)
 242                         params.local_key->bind(static_cast<int>(texture_id::local_key));
 243
 244                 if(params.layer_key)
 245                         params.layer_key->bind(static_cast<int>(texture_id::layer_key));
 246
 247                 // Setup shader
 248
 249                 shader_->use();
 250
 251                 shader_->set("post_processing", false);
 252                 shader_->set("plane[0]",                texture_id::plane0);
 253                 shader_->set("plane[1]",                texture_id::plane1);
 254                 shader_->set("plane[2]",                texture_id::plane2);
 255                 shader_->set("plane[3]",                texture_id::plane3);
 256                 for (int n = 0; n < params.textures.size(); ++n)
 257                         shader_->set("plane_size[" + boost::lexical_cast<std::string>(n) + "]",
 258                                                  static_cast<float>(params.textures[n]->width()),
 259                                                  static_cast<float>(params.textures[n]->height()));
 260
 261                 shader_->set("local_key",               texture_id::local_key);
 262                 shader_->set("layer_key",               texture_id::layer_key);
 263                 shader_->set("is_hd",                   params.pix_desc.planes.at(0).height > 700 ? 1 : 0);
 264                 shader_->set("has_local_key",   static_cast<bool>(params.local_key));
 265                 shader_->set("has_layer_key",   static_cast<bool>(params.layer_key));
 266                 shader_->set("pixel_format",    params.pix_desc.format);
 267                 shader_->set("opacity",                 params.transform.is_key ? 1.0 : params.transform.opacity);
 268
 269                 if (params.transform.chroma.key != core::chroma::type::none)
 270                 {
 271                         shader_->set("chroma", true);
 272                         shader_->set("chroma_mode", static_cast<int>(params.transform.chroma.key));
 273                         shader_->set("chroma_blend", params.transform.chroma.threshold, params.transform.chroma.softness);
 274                         shader_->set("chroma_spill", params.transform.chroma.spill);
 275                 }
 276                 else
 277                         shader_->set("chroma", false);
 278
 279
 280                 // Setup blend_func
 281
 282                 if(params.transform.is_key)
 283                         params.blend_mode = core::blend_mode::normal;
 284
 285                 if(blend_modes_)
 286                 {
 287                         params.background->bind(static_cast<int>(texture_id::background));
 288
 289                         shader_->set("background",      texture_id::background);
 290                         shader_->set("blend_mode",      params.blend_mode);
 291                         shader_->set("keyer",           params.keyer);
 292                 }
 293                 else
 294                 {
 295                         GL(glEnable(GL_BLEND));
 296
 297                         switch(params.keyer)
 298                         {
 299                         case keyer::additive:
 300                                 GL(glBlendFunc(GL_ONE, GL_ONE));
 301                                 break;
 302                         case keyer::linear:
 303                         default:
 304                                 GL(glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA));
 305                         }
 306                 }
 307
 308                 // Setup image-adjustements
 309
 310                 if (params.transform.levels.min_input  > epsilon                ||
 311                         params.transform.levels.max_input  < 1.0-epsilon        ||
 312                         params.transform.levels.min_output > epsilon            ||
 313                         params.transform.levels.max_output < 1.0-epsilon        ||
 314                         std::abs(params.transform.levels.gamma - 1.0) > epsilon)
 315                 {
 316                         shader_->set("levels", true);
 317                         shader_->set("min_input",       params.transform.levels.min_input);
 318                         shader_->set("max_input",       params.transform.levels.max_input);
 319                         shader_->set("min_output",      params.transform.levels.min_output);
 320                         shader_->set("max_output",      params.transform.levels.max_output);
 321                         shader_->set("gamma",           params.transform.levels.gamma);
 322                 }
 323                 else
 324                         shader_->set("levels", false);
 325
 326                 if (std::abs(params.transform.brightness - 1.0) > epsilon ||
 327                         std::abs(params.transform.saturation - 1.0) > epsilon ||
 328                         std::abs(params.transform.contrast - 1.0)   > epsilon)
 329                 {
 330                         shader_->set("csb",     true);
 331
 332                         shader_->set("brt", params.transform.brightness);
 333                         shader_->set("sat", params.transform.saturation);
 334                         shader_->set("con", params.transform.contrast);
 335                 }
 336                 else
 337                         shader_->set("csb",     false);
 338
 339                 // Setup interlacing
 340
 341                 if (params.transform.field_mode != core::field_mode::progressive)
 342                 {
 343                         GL(glEnable(GL_POLYGON_STIPPLE));
 344
 345                         if(params.transform.field_mode == core::field_mode::upper)
 346                                 glPolygonStipple(upper_pattern);
 347                         else if(params.transform.field_mode == core::field_mode::lower)
 348                                 glPolygonStipple(lower_pattern);
 349                 }
 350
 351                 // Setup drawing area
 352
 353                 GL(glViewport(0, 0, params.background->width(), params.background->height()));
 354                 glDisable(GL_DEPTH_TEST);
 355
 356                 auto m_p = params.transform.clip_translation;
 357                 auto m_s = params.transform.clip_scale;
 358
 359                 bool scissor = m_p[0] > std::numeric_limits<double>::epsilon()                  || m_p[1] > std::numeric_limits<double>::epsilon() ||
 360                                            m_s[0] < (1.0 - std::numeric_limits<double>::epsilon())      || m_s[1] < (1.0 - std::numeric_limits<double>::epsilon());
 361
 362                 if (scissor)
 363                 {
 364                         double w = static_cast<double>(params.background->width());
 365                         double h = static_cast<double>(params.background->height());
 366
 367                         GL(glEnable(GL_SCISSOR_TEST));
 368                         glScissor(static_cast<int>(m_p[0] * w), static_cast<int>(m_p[1] * h), std::max(0, static_cast<int>(m_s[0] * w)), std::max(0, static_cast<int>(m_s[1] * h)));
 369                 }
 370
 371                 // Synchronize and set render target
 372
 373                 if (blend_modes_)
 374                 {
 375                         // http://www.opengl.org/registry/specs/NV/texture_barrier.txt
 376                         // This allows us to use framebuffer (background) both as source and target while blending.
 377                         glTextureBarrierNV();
 378                 }
 379
 380                 params.background->attach();
 381
 382                 // Perspective correction
 383                 double diagonal_intersection_x;
 384                 double diagonal_intersection_y;
 385
 386                 if (get_line_intersection(
 387                                 pers.ul[0] + crop.ul[0], pers.ul[1] + crop.ul[1],
 388                                 pers.lr[0] + crop.lr[0], pers.lr[1] + crop.lr[1],
 389                                 pers.ur[0] + crop.lr[0], pers.ur[1] + crop.ul[1],
 390                                 pers.ll[0] + crop.ul[0], pers.ll[1] + crop.lr[1],
 391                                 diagonal_intersection_x,
 392                                 diagonal_intersection_y) &&
 393                         is_default_geometry)
 394                 {
 395                         // http://www.reedbeta.com/blog/2012/05/26/quadrilateral-interpolation-part-1/
 396                         auto d0 = hypotenuse(pers.ll[0] + crop.ul[0], pers.ll[1] + crop.lr[1], diagonal_intersection_x, diagonal_intersection_y);
 397                         auto d1 = hypotenuse(pers.lr[0] + crop.lr[0], pers.lr[1] + crop.lr[1], diagonal_intersection_x, diagonal_intersection_y);
 398                         auto d2 = hypotenuse(pers.ur[0] + crop.lr[0], pers.ur[1] + crop.ul[1], diagonal_intersection_x, diagonal_intersection_y);
 399                         auto d3 = hypotenuse(pers.ul[0] + crop.ul[0], pers.ul[1] + crop.ul[1], diagonal_intersection_x, diagonal_intersection_y);
 400
 401                         auto ulq = calc_q(d3, d1);
 402                         auto urq = calc_q(d2, d0);
 403                         auto lrq = calc_q(d1, d3);
 404                         auto llq = calc_q(d0, d2);
 405
 406                         std::vector<double> q_values = { ulq, urq, lrq, llq };
 407
 408                         corner = 0;
 409                         for (auto& coord : coords)
 410                         {
 411                                 coord.texture_q = q_values[corner];
 412                                 coord.texture_x *= q_values[corner];
 413                                 coord.texture_y *= q_values[corner];
 414
 415                                 if (++corner == 4)
 416                                         corner = 0;
 417                         }
 418                 }
 419
 420                 // Draw
 421                 switch(params.geometry.type())
 422                 {
 423                 case core::frame_geometry::geometry_type::quad:
 424                 case core::frame_geometry::geometry_type::quad_list:
 425                         {
 426                                 glClientActiveTexture(GL_TEXTURE0);
 427
 428                                 glDisableClientState(GL_EDGE_FLAG_ARRAY);
 429                                 glDisableClientState(GL_COLOR_ARRAY);
 430                                 glDisableClientState(GL_INDEX_ARRAY);
 431                                 glDisableClientState(GL_NORMAL_ARRAY);
 432
 433                                 glEnableClientState(GL_TEXTURE_COORD_ARRAY);
 434                                 glEnableClientState(GL_VERTEX_ARRAY);
 435
 436                                 auto stride = static_cast<GLsizei>(sizeof(core::frame_geometry::coord));
 437                                 auto vertex_coord_member = &core::frame_geometry::coord::vertex_x;
 438                                 auto texture_coord_member = &core::frame_geometry::coord::texture_x;
 439                                 auto data_ptr = coords.data();
 440                                 auto vertex_coord_ptr = &(data_ptr->*vertex_coord_member);
 441                                 auto texture_coord_ptr = &(data_ptr->*texture_coord_member);
 442
 443                                 glVertexPointer(2, GL_DOUBLE, stride, vertex_coord_ptr);
 444                                 glTexCoordPointer(4, GL_DOUBLE, stride, texture_coord_ptr);
 445                                 glDrawArrays(GL_QUADS, 0, static_cast<GLsizei>(coords.size())); //each vertex is four doubles.
 446
 447                                 glDisableClientState(GL_TEXTURE_COORD_ARRAY);
 448                                 glDisableClientState(GL_VERTEX_ARRAY);
 449                         }
 450                         break;
 451                 default:
 452                         break;
 453                 }
 454
 455                 // Cleanup
 456                 GL(glDisable(GL_SCISSOR_TEST));
 457                 GL(glDisable(GL_POLYGON_STIPPLE));
 458                 GL(glDisable(GL_BLEND));
 459         }
 460
 461         void post_process(const std::shared_ptr<texture>& background, bool straighten_alpha)
 462         {
 463                 bool should_post_process =
 464                                 supports_texture_barrier_
 465                                 && straighten_alpha
 466                                 && post_processing_;
 467
 468                 if (!should_post_process)
 469                         return;
 470
 471                 background->attach();
 472
 473                 background->bind(static_cast<int>(texture_id::background));
 474
 475                 shader_->use();
 476                 shader_->set("background", texture_id::background);
 477                 shader_->set("post_processing", true);
 478                 shader_->set("straighten_alpha", straighten_alpha);
 479
 480                 GL(glViewport(0, 0, background->width(), background->height()));
 481
 482                 glBegin(GL_QUADS);
 483                         glMultiTexCoord2d(GL_TEXTURE0, 0.0, 0.0); glVertex2d(0.0, 0.0);
 484                         glMultiTexCoord2d(GL_TEXTURE0, 1.0, 0.0); glVertex2d(1.0, 0.0);
 485                         glMultiTexCoord2d(GL_TEXTURE0, 1.0, 1.0); glVertex2d(1.0, 1.0);
 486                         glMultiTexCoord2d(GL_TEXTURE0, 0.0, 1.0); glVertex2d(0.0, 1.0);
 487                 glEnd();
 488
 489                 glTextureBarrierNV();
 490         }
 491 };
 492
 493 image_kernel::image_kernel(const spl::shared_ptr<device>& ogl, bool blend_modes_wanted, bool straight_alpha_wanted) : impl_(new impl(ogl, blend_modes_wanted, straight_alpha_wanted)){}
 494 image_kernel::~image_kernel(){}
 495 void image_kernel::draw(const draw_params& params){impl_->draw(params);}
 496 void image_kernel::post_process(const std::shared_ptr<texture>& background, bool straighten_alpha) { impl_->post_process(background, straighten_alpha);}
 497 bool image_kernel::has_blend_modes() const{return impl_->blend_modes_;}
 498
 499 }}}