2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
4 * This file is part of CasparCG (www.casparcg.com).
6 * CasparCG is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
11 * CasparCG is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
19 * Author: Robert Nagy, ronag89@gmail.com
22 #include "../../StdAfx.h"
24 #include "image_kernel.h"
26 #include "image_shader.h"
27 #include "blending_glsl.h"
29 #include "../util/shader.h"
30 #include "../util/texture.h"
31 #include "../util/device.h"
33 #include <common/except.h>
34 #include <common/gl/gl_check.h>
35 #include <common/env.h>
37 #include <core/video_format.h>
38 #include <core/frame/pixel_format.h>
39 #include <core/frame/frame_transform.h>
41 #include <boost/algorithm/cxx11/all_of.hpp>
42 #include <boost/lexical_cast.hpp>
43 #include <boost/range/adaptor/transformed.hpp>
47 namespace caspar { namespace accelerator { namespace ogl {
49 // http://stackoverflow.com/questions/563198/how-do-you-detect-where-two-line-segments-intersect
50 bool get_line_intersection(
51 double p0_x, double p0_y,
52 double p1_x, double p1_y,
53 double p2_x, double p2_y,
54 double p3_x, double p3_y,
55 double& result_x, double& result_y)
57 double s1_x = p1_x - p0_x;
58 double s1_y = p1_y - p0_y;
59 double s2_x = p3_x - p2_x;
60 double s2_y = p3_y - p2_y;
62 double s = (-s1_y * (p0_x - p2_x) + s1_x * (p0_y - p2_y)) / (-s2_x * s1_y + s1_x * s2_y);
63 double t = (s2_x * (p0_y - p2_y) - s2_y * (p0_x - p2_x)) / (-s2_x * s1_y + s1_x * s2_y);
65 if (s >= 0 && s <= 1 && t >= 0 && t <= 1)
68 result_x = p0_x + (t * s1_x);
69 result_y = p0_y + (t * s1_y);
74 return false; // No collision
77 double hypotenuse(double x1, double y1, double x2, double y2)
82 return std::sqrt(x * x + y * y);
85 double calc_q(double close_diagonal, double distant_diagonal)
87 return (close_diagonal + distant_diagonal) / distant_diagonal;
90 bool is_above_screen(double y)
95 bool is_below_screen(double y)
100 bool is_left_of_screen(double x)
105 bool is_right_of_screen(double x)
110 bool is_outside_screen(const std::vector<core::frame_geometry::coord>& coords)
112 auto x_coords = coords | boost::adaptors::transformed([](const core::frame_geometry::coord& c) { return c.vertex_x; });
113 auto y_coords = coords | boost::adaptors::transformed([](const core::frame_geometry::coord& c) { return c.vertex_y; });
115 return boost::algorithm::all_of(x_coords, &is_left_of_screen)
116 || boost::algorithm::all_of(x_coords, &is_right_of_screen)
117 || boost::algorithm::all_of(y_coords, &is_above_screen)
118 || boost::algorithm::all_of(y_coords, &is_below_screen);
121 GLubyte upper_pattern[] = {
122 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
123 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
124 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
125 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00};
127 GLubyte lower_pattern[] = {
128 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
129 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
130 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
131 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff};
133 struct image_kernel::impl
135 spl::shared_ptr<device> ogl_;
136 spl::shared_ptr<shader> shader_;
138 bool post_processing_;
139 bool supports_texture_barrier_ = glTextureBarrierNV != 0;
141 impl(const spl::shared_ptr<device>& ogl, bool blend_modes_wanted, bool straight_alpha_wanted)
143 , shader_(ogl_->invoke([&]{return get_image_shader(ogl, blend_modes_, blend_modes_wanted, post_processing_, straight_alpha_wanted); }))
145 if (!supports_texture_barrier_)
146 CASPAR_LOG(warning) << L"[image_mixer] TextureBarrierNV not supported. Post processing will not be available";
149 void draw(draw_params params)
151 static const double epsilon = 0.001;
153 CASPAR_ASSERT(params.pix_desc.planes.size() == params.textures.size());
155 if(params.textures.empty() || !params.background)
158 if(params.transform.opacity < epsilon)
161 auto coords = params.geometry.data();
166 // Calculate transforms
167 auto f_p = params.transform.fill_translation;
168 auto f_s = params.transform.fill_scale;
170 bool is_default_geometry = boost::equal(coords, core::frame_geometry::get_default().data());
171 auto aspect = params.aspect_ratio;
172 auto angle = params.transform.angle;
173 auto anchor = params.transform.anchor;
174 auto crop = params.transform.crop;
175 auto pers = params.transform.perspective;
180 std::vector<boost::array<double, 2>> pers_corners = { pers.ul, pers.ur, pers.lr, pers.ll };
182 auto do_crop = [&](core::frame_geometry::coord& coord)
184 if (!is_default_geometry)
185 // TODO implement support for non-default geometry.
188 coord.vertex_x = std::max(coord.vertex_x, crop.ul[0]);
189 coord.vertex_x = std::min(coord.vertex_x, crop.lr[0]);
190 coord.vertex_y = std::max(coord.vertex_y, crop.ul[1]);
191 coord.vertex_y = std::min(coord.vertex_y, crop.lr[1]);
192 coord.texture_x = std::max(coord.texture_x, crop.ul[0]);
193 coord.texture_x = std::min(coord.texture_x, crop.lr[0]);
194 coord.texture_y = std::max(coord.texture_y, crop.ul[1]);
195 coord.texture_y = std::min(coord.texture_y, crop.lr[1]);
197 auto do_perspective = [=](core::frame_geometry::coord& coord, const boost::array<double, 2>& pers_corner)
199 if (!is_default_geometry)
200 // TODO implement support for non-default geometry.
203 coord.vertex_x += pers_corner[0];
204 coord.vertex_y += pers_corner[1];
206 auto rotate = [&](core::frame_geometry::coord& coord)
208 auto orig_x = (coord.vertex_x - anchor[0]) * f_s[0];
209 auto orig_y = (coord.vertex_y - anchor[1]) * f_s[1] / aspect;
210 coord.vertex_x = orig_x * std::cos(angle) - orig_y * std::sin(angle);
211 coord.vertex_y = orig_x * std::sin(angle) + orig_y * std::cos(angle);
212 coord.vertex_y *= aspect;
214 auto move = [&](core::frame_geometry::coord& coord)
216 coord.vertex_x += f_p[0];
217 coord.vertex_y += f_p[1];
221 for (auto& coord : coords)
224 do_perspective(coord, pers_corners.at(corner));
232 // Skip drawing if all the coordinates will be outside the screen.
233 if (is_outside_screen(coords))
238 for(int n = 0; n < params.textures.size(); ++n)
239 params.textures[n]->bind(n);
242 params.local_key->bind(static_cast<int>(texture_id::local_key));
245 params.layer_key->bind(static_cast<int>(texture_id::layer_key));
251 shader_->set("post_processing", false);
252 shader_->set("plane[0]", texture_id::plane0);
253 shader_->set("plane[1]", texture_id::plane1);
254 shader_->set("plane[2]", texture_id::plane2);
255 shader_->set("plane[3]", texture_id::plane3);
256 for (int n = 0; n < params.textures.size(); ++n)
257 shader_->set("plane_size[" + boost::lexical_cast<std::string>(n) + "]",
258 static_cast<float>(params.textures[n]->width()),
259 static_cast<float>(params.textures[n]->height()));
261 shader_->set("local_key", texture_id::local_key);
262 shader_->set("layer_key", texture_id::layer_key);
263 shader_->set("is_hd", params.pix_desc.planes.at(0).height > 700 ? 1 : 0);
264 shader_->set("has_local_key", static_cast<bool>(params.local_key));
265 shader_->set("has_layer_key", static_cast<bool>(params.layer_key));
266 shader_->set("pixel_format", params.pix_desc.format);
267 shader_->set("opacity", params.transform.is_key ? 1.0 : params.transform.opacity);
269 if (params.transform.chroma.key != core::chroma::type::none)
271 shader_->set("chroma", true);
272 shader_->set("chroma_mode", static_cast<int>(params.transform.chroma.key));
273 shader_->set("chroma_blend", params.transform.chroma.threshold, params.transform.chroma.softness);
274 shader_->set("chroma_spill", params.transform.chroma.spill);
277 shader_->set("chroma", false);
282 if(params.transform.is_key)
283 params.blend_mode = core::blend_mode::normal;
287 params.background->bind(static_cast<int>(texture_id::background));
289 shader_->set("background", texture_id::background);
290 shader_->set("blend_mode", params.blend_mode);
291 shader_->set("keyer", params.keyer);
295 GL(glEnable(GL_BLEND));
299 case keyer::additive:
300 GL(glBlendFunc(GL_ONE, GL_ONE));
304 GL(glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA));
308 // Setup image-adjustements
310 if (params.transform.levels.min_input > epsilon ||
311 params.transform.levels.max_input < 1.0-epsilon ||
312 params.transform.levels.min_output > epsilon ||
313 params.transform.levels.max_output < 1.0-epsilon ||
314 std::abs(params.transform.levels.gamma - 1.0) > epsilon)
316 shader_->set("levels", true);
317 shader_->set("min_input", params.transform.levels.min_input);
318 shader_->set("max_input", params.transform.levels.max_input);
319 shader_->set("min_output", params.transform.levels.min_output);
320 shader_->set("max_output", params.transform.levels.max_output);
321 shader_->set("gamma", params.transform.levels.gamma);
324 shader_->set("levels", false);
326 if (std::abs(params.transform.brightness - 1.0) > epsilon ||
327 std::abs(params.transform.saturation - 1.0) > epsilon ||
328 std::abs(params.transform.contrast - 1.0) > epsilon)
330 shader_->set("csb", true);
332 shader_->set("brt", params.transform.brightness);
333 shader_->set("sat", params.transform.saturation);
334 shader_->set("con", params.transform.contrast);
337 shader_->set("csb", false);
341 if (params.transform.field_mode != core::field_mode::progressive)
343 GL(glEnable(GL_POLYGON_STIPPLE));
345 if(params.transform.field_mode == core::field_mode::upper)
346 glPolygonStipple(upper_pattern);
347 else if(params.transform.field_mode == core::field_mode::lower)
348 glPolygonStipple(lower_pattern);
351 // Setup drawing area
353 GL(glViewport(0, 0, params.background->width(), params.background->height()));
354 glDisable(GL_DEPTH_TEST);
356 auto m_p = params.transform.clip_translation;
357 auto m_s = params.transform.clip_scale;
359 bool scissor = m_p[0] > std::numeric_limits<double>::epsilon() || m_p[1] > std::numeric_limits<double>::epsilon() ||
360 m_s[0] < (1.0 - std::numeric_limits<double>::epsilon()) || m_s[1] < (1.0 - std::numeric_limits<double>::epsilon());
364 double w = static_cast<double>(params.background->width());
365 double h = static_cast<double>(params.background->height());
367 GL(glEnable(GL_SCISSOR_TEST));
368 glScissor(static_cast<int>(m_p[0] * w), static_cast<int>(m_p[1] * h), std::max(0, static_cast<int>(m_s[0] * w)), std::max(0, static_cast<int>(m_s[1] * h)));
371 // Synchronize and set render target
375 // http://www.opengl.org/registry/specs/NV/texture_barrier.txt
376 // This allows us to use framebuffer (background) both as source and target while blending.
377 glTextureBarrierNV();
380 params.background->attach();
382 // Perspective correction
383 double diagonal_intersection_x;
384 double diagonal_intersection_y;
386 if (get_line_intersection(
387 pers.ul[0] + crop.ul[0], pers.ul[1] + crop.ul[1],
388 pers.lr[0] + crop.lr[0], pers.lr[1] + crop.lr[1],
389 pers.ur[0] + crop.lr[0], pers.ur[1] + crop.ul[1],
390 pers.ll[0] + crop.ul[0], pers.ll[1] + crop.lr[1],
391 diagonal_intersection_x,
392 diagonal_intersection_y) &&
395 // http://www.reedbeta.com/blog/2012/05/26/quadrilateral-interpolation-part-1/
396 auto d0 = hypotenuse(pers.ll[0] + crop.ul[0], pers.ll[1] + crop.lr[1], diagonal_intersection_x, diagonal_intersection_y);
397 auto d1 = hypotenuse(pers.lr[0] + crop.lr[0], pers.lr[1] + crop.lr[1], diagonal_intersection_x, diagonal_intersection_y);
398 auto d2 = hypotenuse(pers.ur[0] + crop.lr[0], pers.ur[1] + crop.ul[1], diagonal_intersection_x, diagonal_intersection_y);
399 auto d3 = hypotenuse(pers.ul[0] + crop.ul[0], pers.ul[1] + crop.ul[1], diagonal_intersection_x, diagonal_intersection_y);
401 auto ulq = calc_q(d3, d1);
402 auto urq = calc_q(d2, d0);
403 auto lrq = calc_q(d1, d3);
404 auto llq = calc_q(d0, d2);
406 std::vector<double> q_values = { ulq, urq, lrq, llq };
409 for (auto& coord : coords)
411 coord.texture_q = q_values[corner];
412 coord.texture_x *= q_values[corner];
413 coord.texture_y *= q_values[corner];
421 switch(params.geometry.type())
423 case core::frame_geometry::geometry_type::quad:
424 case core::frame_geometry::geometry_type::quad_list:
426 glClientActiveTexture(GL_TEXTURE0);
428 glDisableClientState(GL_EDGE_FLAG_ARRAY);
429 glDisableClientState(GL_COLOR_ARRAY);
430 glDisableClientState(GL_INDEX_ARRAY);
431 glDisableClientState(GL_NORMAL_ARRAY);
433 glEnableClientState(GL_TEXTURE_COORD_ARRAY);
434 glEnableClientState(GL_VERTEX_ARRAY);
436 auto stride = static_cast<GLsizei>(sizeof(core::frame_geometry::coord));
437 auto vertex_coord_member = &core::frame_geometry::coord::vertex_x;
438 auto texture_coord_member = &core::frame_geometry::coord::texture_x;
439 auto data_ptr = coords.data();
440 auto vertex_coord_ptr = &(data_ptr->*vertex_coord_member);
441 auto texture_coord_ptr = &(data_ptr->*texture_coord_member);
443 glVertexPointer(2, GL_DOUBLE, stride, vertex_coord_ptr);
444 glTexCoordPointer(4, GL_DOUBLE, stride, texture_coord_ptr);
445 glDrawArrays(GL_QUADS, 0, static_cast<GLsizei>(coords.size())); //each vertex is four doubles.
447 glDisableClientState(GL_TEXTURE_COORD_ARRAY);
448 glDisableClientState(GL_VERTEX_ARRAY);
456 GL(glDisable(GL_SCISSOR_TEST));
457 GL(glDisable(GL_POLYGON_STIPPLE));
458 GL(glDisable(GL_BLEND));
461 void post_process(const std::shared_ptr<texture>& background, bool straighten_alpha)
463 bool should_post_process =
464 supports_texture_barrier_
468 if (!should_post_process)
471 background->attach();
473 background->bind(static_cast<int>(texture_id::background));
476 shader_->set("background", texture_id::background);
477 shader_->set("post_processing", true);
478 shader_->set("straighten_alpha", straighten_alpha);
480 GL(glViewport(0, 0, background->width(), background->height()));
483 glMultiTexCoord2d(GL_TEXTURE0, 0.0, 0.0); glVertex2d(0.0, 0.0);
484 glMultiTexCoord2d(GL_TEXTURE0, 1.0, 0.0); glVertex2d(1.0, 0.0);
485 glMultiTexCoord2d(GL_TEXTURE0, 1.0, 1.0); glVertex2d(1.0, 1.0);
486 glMultiTexCoord2d(GL_TEXTURE0, 0.0, 1.0); glVertex2d(0.0, 1.0);
489 glTextureBarrierNV();
493 image_kernel::image_kernel(const spl::shared_ptr<device>& ogl, bool blend_modes_wanted, bool straight_alpha_wanted) : impl_(new impl(ogl, blend_modes_wanted, straight_alpha_wanted)){}
494 image_kernel::~image_kernel(){}
495 void image_kernel::draw(const draw_params& params){impl_->draw(params);}
496 void image_kernel::post_process(const std::shared_ptr<texture>& background, bool straighten_alpha) { impl_->post_process(background, straighten_alpha);}
497 bool image_kernel::has_blend_modes() const{return impl_->blend_modes_;}