2 * Copyright 2013 Sveriges Television AB http://casparcg.com/
\r
4 * This file is part of CasparCG (www.casparcg.com).
\r
6 * CasparCG is free software: you can redistribute it and/or modify
\r
7 * it under the terms of the GNU General Public License as published by
\r
8 * the Free Software Foundation, either version 3 of the License, or
\r
9 * (at your option) any later version.
\r
11 * CasparCG is distributed in the hope that it will be useful,
\r
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
\r
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
\r
14 * GNU General Public License for more details.
\r
16 * You should have received a copy of the GNU General Public License
\r
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
\r
19 * Author: Robert Nagy, ronag89@gmail.com
\r
22 #include "../../stdafx.h"
\r
24 #include "image_kernel.h"
\r
26 #include "shader/image_shader.h"
\r
27 #include "shader/blending_glsl.h"
\r
29 #include "../gpu/shader.h"
\r
30 #include "../gpu/device_buffer.h"
\r
31 #include "../gpu/ogl_device.h"
\r
33 #include <common/exception/exceptions.h>
\r
34 #include <common/gl/gl_check.h>
\r
35 #include <common/env.h>
\r
37 #include <core/video_format.h>
\r
38 #include <core/producer/frame/pixel_format.h>
\r
39 #include <core/producer/frame/frame_transform.h>
\r
41 #include <boost/noncopyable.hpp>
\r
43 namespace caspar { namespace core {
\r
45 // http://stackoverflow.com/questions/563198/how-do-you-detect-where-two-line-segments-intersect
\r
46 bool get_line_intersection(
\r
47 double p0_x, double p0_y,
\r
48 double p1_x, double p1_y,
\r
49 double p2_x, double p2_y,
\r
50 double p3_x, double p3_y,
\r
51 double& result_x, double& result_y)
\r
53 double s1_x = p1_x - p0_x;
\r
54 double s1_y = p1_y - p0_y;
\r
55 double s2_x = p3_x - p2_x;
\r
56 double s2_y = p3_y - p2_y;
\r
58 double s = (-s1_y * (p0_x - p2_x) + s1_x * (p0_y - p2_y)) / (-s2_x * s1_y + s1_x * s2_y);
\r
59 double t = ( s2_x * (p0_y - p2_y) - s2_y * (p0_x - p2_x)) / (-s2_x * s1_y + s1_x * s2_y);
\r
61 if (s >= 0 && s <= 1 && t >= 0 && t <= 1)
\r
63 // Collision detected
\r
64 result_x = p0_x + (t * s1_x);
\r
65 result_y = p0_y + (t * s1_y);
\r
70 return false; // No collision
\r
73 double hypotenuse(double x1, double y1, double x2, double y2)
\r
78 return std::sqrt(x * x + y * y);
\r
81 double calc_q(double close_diagonal, double distant_diagonal)
\r
83 return (close_diagonal + distant_diagonal) / distant_diagonal;
\r
86 bool is_above_screen(double y)
\r
91 bool is_below_screen(double y)
\r
96 bool is_left_of_screen(double x)
\r
101 bool is_right_of_screen(double x)
\r
106 bool is_outside_screen(
\r
107 double x1, double y1,
\r
108 double x2, double y2,
\r
109 double x3, double y3,
\r
110 double x4, double y4)
\r
112 // Every point needs to be outside the screen on the *same* side in order to be considered outside the screen.
\r
113 return (is_above_screen(y1) && is_above_screen(y2) && is_above_screen(y3) && is_above_screen(y4))
\r
114 || (is_below_screen(y1) && is_below_screen(y2) && is_below_screen(y3) && is_below_screen(y4))
\r
115 || (is_left_of_screen(x1) && is_left_of_screen(x2) && is_left_of_screen(x3) && is_left_of_screen(x4))
\r
116 || (is_right_of_screen(x1) && is_right_of_screen(x2) && is_right_of_screen(x3) && is_right_of_screen(x4));
\r
119 GLubyte upper_pattern[] = {
\r
120 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
\r
121 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
\r
122 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
\r
123 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00};
\r
125 GLubyte lower_pattern[] = {
\r
126 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
\r
127 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
\r
128 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
\r
129 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff};
\r
131 struct image_kernel::implementation : boost::noncopyable
\r
133 safe_ptr<ogl_device> ogl_;
\r
134 safe_ptr<shader> shader_;
\r
136 bool post_processing_;
\r
137 bool supports_texture_barrier_;
\r
139 implementation(const safe_ptr<ogl_device>& ogl)
\r
141 , shader_(ogl_->invoke([&]{return get_image_shader(*ogl, blend_modes_, post_processing_);}))
\r
142 , supports_texture_barrier_(glTextureBarrierNV != 0)
\r
144 if (!supports_texture_barrier_)
\r
145 CASPAR_LOG(warning) << L"[image_mixer] TextureBarrierNV not supported. Post processing will not be available";
\r
148 void draw(draw_params&& params)
\r
150 static const double epsilon = 0.001;
\r
152 CASPAR_ASSERT(params.pix_desc.planes.size() == params.textures.size());
\r
154 if(params.textures.empty() || !params.background)
\r
157 if(params.transform.opacity < epsilon)
\r
160 auto f_p = params.transform.fill_translation;
\r
161 auto f_s = params.transform.fill_scale;
\r
163 // Calculate rotation
\r
164 auto aspect = params.aspect_ratio;
\r
165 auto angle = params.transform.angle;
\r
167 auto rotate = [angle, aspect](double orig_x, double orig_y) -> boost::array<double, 2>
\r
169 boost::array<double, 2> result;
\r
170 result[0] = orig_x * std::cos(angle) - orig_y * std::sin(angle);
\r
171 result[1] = orig_x * std::sin(angle) + orig_y * std::cos(angle);
\r
172 result[1] *= aspect;
\r
177 auto anchor = params.transform.anchor;
\r
178 auto crop = params.transform.crop;
\r
179 auto pers = params.transform.perspective;
\r
181 auto ul = rotate((-anchor[0] + pers.ul[0]) * f_s[0], (-anchor[1] + pers.ul[1]) * f_s[1] / aspect);
\r
182 auto ur = rotate((-anchor[0] + pers.ur[0]) * f_s[0], (-anchor[1] + pers.ur[1]) * f_s[1] / aspect);
\r
183 auto lr = rotate((-anchor[0] + pers.lr[0]) * f_s[0], (-anchor[1] + pers.lr[1]) * f_s[1] / aspect);
\r
184 auto ll = rotate((-anchor[0] + pers.ll[0]) * f_s[0], (-anchor[1] + pers.ll[1]) * f_s[1] / aspect);
\r
186 auto upper_left_x = f_p[0] + ul[0];
\r
187 auto upper_left_y = f_p[1] + ul[1];
\r
188 auto upper_right_x = f_p[0] + ur[0];
\r
189 auto upper_right_y = f_p[1] + ur[1];
\r
190 auto lower_right_x = f_p[0] + lr[0];
\r
191 auto lower_right_y = f_p[1] + lr[1];
\r
192 auto lower_left_x = f_p[0] + ll[0];
\r
193 auto lower_left_y = f_p[1] + ll[1];
\r
195 // Skip drawing if the QUAD will be outside the screen.
\r
196 if (is_outside_screen(
\r
197 upper_left_x, upper_left_y,
\r
198 upper_right_x, upper_right_y,
\r
199 lower_right_x, lower_right_y,
\r
200 lower_left_x, lower_left_y))
\r
205 if(!std::all_of(params.textures.begin(), params.textures.end(), std::mem_fn(&device_buffer::ready)))
\r
207 CASPAR_LOG(trace) << L"[image_mixer] Performance warning. Host to device transfer not complete, GPU will be stalled";
\r
208 ogl_->yield(); // Try to give it some more time.
\r
213 for(size_t n = 0; n < params.textures.size(); ++n)
\r
214 params.textures[n]->bind(n);
\r
216 if(params.local_key)
\r
217 params.local_key->bind(texture_id::local_key);
\r
219 if(params.layer_key)
\r
220 params.layer_key->bind(texture_id::layer_key);
\r
224 ogl_->use(*shader_);
\r
226 shader_->set("plane[0]", texture_id::plane0);
\r
227 shader_->set("plane[1]", texture_id::plane1);
\r
228 shader_->set("plane[2]", texture_id::plane2);
\r
229 shader_->set("plane[3]", texture_id::plane3);
\r
230 shader_->set("local_key", texture_id::local_key);
\r
231 shader_->set("layer_key", texture_id::layer_key);
\r
232 shader_->set("is_hd", params.pix_desc.planes.at(0).height > 700 ? 1 : 0);
\r
233 shader_->set("has_local_key", bool(params.local_key));
\r
234 shader_->set("has_layer_key", bool(params.layer_key));
\r
235 shader_->set("pixel_format", params.pix_desc.pix_fmt);
\r
236 shader_->set("opacity", params.transform.is_key ? 1.0 : params.transform.opacity);
\r
237 shader_->set("post_processing", false);
\r
239 shader_->set("chroma_mode", params.blend_mode.chroma.key == chroma::green ? 1 : (params.blend_mode.chroma.key == chroma::blue ? 2 : 0));
\r
240 shader_->set("chroma_blend", params.blend_mode.chroma.threshold, params.blend_mode.chroma.softness);
\r
241 shader_->set("chroma_spill", params.blend_mode.chroma.spill);
\r
242 // shader_->set("chroma.key", ((params.blend_mode.chroma.key >> 24) && 0xff)/255.0f,
\r
243 // ((params.blend_mode.chroma.key >> 16) && 0xff)/255.0f,
\r
244 // (params.blend_mode.chroma.key & 0xff)/255.0f);
\r
245 // if (params.blend_mode.chroma.key != chroma::none)
\r
247 // shader_->set("chroma.threshold", params.blend_mode.chroma.threshold);
\r
248 // shader_->set("chroma.softness", params.blend_mode.chroma.softness);
\r
249 // shader_->set("chroma.blur", params.blend_mode.chroma.blur);
\r
250 // shader_->set("chroma.spill", params.blend_mode.chroma.spill);
\r
251 // shader_->set("chroma.show_mask", params.blend_mode.chroma.show_mask);
\r
254 // Setup blend_func
\r
255 if(params.transform.is_key)
\r
256 params.blend_mode = blend_mode::normal;
\r
260 params.background->bind(texture_id::background);
\r
262 shader_->set("background", texture_id::background);
\r
263 shader_->set("blend_mode", params.blend_mode.mode);
\r
264 shader_->set("keyer", params.keyer);
\r
268 switch(params.keyer)
\r
270 case keyer::additive:
\r
271 ogl_->blend_func(GL_ONE, GL_ONE);
\r
273 case keyer::linear:
\r
275 ogl_->blend_func(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
\r
279 // Setup image-adjustements
\r
281 if(params.transform.levels.min_input > epsilon ||
\r
282 params.transform.levels.max_input < 1.0-epsilon ||
\r
283 params.transform.levels.min_output > epsilon ||
\r
284 params.transform.levels.max_output < 1.0-epsilon ||
\r
285 std::abs(params.transform.levels.gamma - 1.0) > epsilon)
\r
287 shader_->set("levels", true);
\r
288 shader_->set("min_input", params.transform.levels.min_input);
\r
289 shader_->set("max_input", params.transform.levels.max_input);
\r
290 shader_->set("min_output", params.transform.levels.min_output);
\r
291 shader_->set("max_output", params.transform.levels.max_output);
\r
292 shader_->set("gamma", params.transform.levels.gamma);
\r
295 shader_->set("levels", false);
\r
297 if(std::abs(params.transform.brightness - 1.0) > epsilon ||
\r
298 std::abs(params.transform.saturation - 1.0) > epsilon ||
\r
299 std::abs(params.transform.contrast - 1.0) > epsilon)
\r
301 shader_->set("csb", true);
\r
303 shader_->set("brt", params.transform.brightness);
\r
304 shader_->set("sat", params.transform.saturation);
\r
305 shader_->set("con", params.transform.contrast);
\r
308 shader_->set("csb", false);
\r
310 // Setup interlacing
\r
312 if(params.transform.field_mode == core::field_mode::progressive)
\r
313 ogl_->disable(GL_POLYGON_STIPPLE);
\r
316 ogl_->enable(GL_POLYGON_STIPPLE);
\r
318 if(params.transform.field_mode == core::field_mode::upper)
\r
319 ogl_->stipple_pattern(upper_pattern);
\r
320 else if(params.transform.field_mode == core::field_mode::lower)
\r
321 ogl_->stipple_pattern(lower_pattern);
\r
324 // Setup drawing area
\r
326 ogl_->viewport(0, 0, params.background->width(), params.background->height());
\r
328 auto m_p = params.transform.clip_translation;
\r
329 auto m_s = params.transform.clip_scale;
\r
331 bool scissor = m_p[0] > std::numeric_limits<double>::epsilon() || m_p[1] > std::numeric_limits<double>::epsilon() ||
\r
332 m_s[0] < (1.0 - std::numeric_limits<double>::epsilon()) || m_s[1] < (1.0 - std::numeric_limits<double>::epsilon());
\r
336 double w = static_cast<double>(params.background->width());
\r
337 double h = static_cast<double>(params.background->height());
\r
339 ogl_->enable(GL_SCISSOR_TEST);
\r
340 ogl_->scissor(static_cast<size_t>(m_p[0]*w), static_cast<size_t>(m_p[1]*h), static_cast<size_t>(m_s[0]*w), static_cast<size_t>(m_s[1]*h));
\r
343 // Set render target
\r
345 ogl_->attach(*params.background);
\r
347 // Perspective correction
\r
352 double diagonal_intersection_x;
\r
353 double diagonal_intersection_y;
\r
355 if (get_line_intersection(
\r
356 pers.ul[0], pers.ul[1],
\r
357 pers.lr[0], pers.lr[1],
\r
358 pers.ur[0], pers.ur[1],
\r
359 pers.ll[0], pers.ll[1],
\r
360 diagonal_intersection_x,
\r
361 diagonal_intersection_y))
\r
363 // http://www.reedbeta.com/blog/2012/05/26/quadrilateral-interpolation-part-1/
\r
364 auto d0 = hypotenuse(pers.ll[0], pers.ll[1], diagonal_intersection_x, diagonal_intersection_y);
\r
365 auto d1 = hypotenuse(pers.lr[0], pers.lr[1], diagonal_intersection_x, diagonal_intersection_y);
\r
366 auto d2 = hypotenuse(pers.ur[0], pers.ur[1], diagonal_intersection_x, diagonal_intersection_y);
\r
367 auto d3 = hypotenuse(pers.ul[0], pers.ul[1], diagonal_intersection_x, diagonal_intersection_y);
\r
369 ulq = calc_q(d3, d1);
\r
370 urq = calc_q(d2, d0);
\r
371 lrq = calc_q(d1, d3);
\r
372 llq = calc_q(d0, d2);
\r
377 GL_TEXTURE0 are texture coordinates to the source material, what will be rendered with this call. These are always set to the whole thing.
\r
378 GL_TEXTURE1 are texture coordinates to background- / key-material, that which will have to be taken in consideration when blending. These are set to the rectangle over which the source will be rendered
\r
381 glMultiTexCoord4d(GL_TEXTURE0, crop.ul[0] * ulq, crop.ul[1] * ulq, 0, ulq); glMultiTexCoord2d(GL_TEXTURE1, upper_left_x, upper_left_y); glVertex2d(upper_left_x * 2.0 - 1.0, upper_left_y * 2.0 - 1.0);
\r
382 glMultiTexCoord4d(GL_TEXTURE0, crop.lr[0] * urq, crop.ul[1] * urq, 0, urq); glMultiTexCoord2d(GL_TEXTURE1, upper_right_x, upper_right_y); glVertex2d(upper_right_x * 2.0 - 1.0, upper_right_y * 2.0 - 1.0);
\r
383 glMultiTexCoord4d(GL_TEXTURE0, crop.lr[0] * lrq, crop.lr[1] * lrq, 0, lrq); glMultiTexCoord2d(GL_TEXTURE1, lower_right_x, lower_right_y); glVertex2d(lower_right_x * 2.0 - 1.0, lower_right_y * 2.0 - 1.0);
\r
384 glMultiTexCoord4d(GL_TEXTURE0, crop.ul[0] * llq, crop.lr[1] * llq, 0, llq); glMultiTexCoord2d(GL_TEXTURE1, lower_left_x, lower_left_y); glVertex2d(lower_left_x * 2.0 - 1.0, lower_left_y * 2.0 - 1.0);
\r
389 ogl_->disable(GL_SCISSOR_TEST);
\r
391 params.textures.clear();
\r
392 ogl_->yield(); // Return resources to pool as early as possible.
\r
396 // http://www.opengl.org/registry/specs/NV/texture_barrier.txt
\r
397 // This allows us to use framebuffer (background) both as source and target while blending.
\r
398 glTextureBarrierNV();
\r
403 const safe_ptr<device_buffer>& background, bool straighten_alpha)
\r
405 bool should_post_process =
\r
406 supports_texture_barrier_
\r
407 && straighten_alpha
\r
408 && post_processing_;
\r
410 if (!should_post_process)
\r
414 ogl_->disable(GL_BLEND);
\r
416 ogl_->disable(GL_POLYGON_STIPPLE);
\r
418 ogl_->attach(*background);
\r
420 background->bind(texture_id::background);
\r
422 ogl_->use(*shader_);
\r
423 shader_->set("background", texture_id::background);
\r
424 shader_->set("post_processing", should_post_process);
\r
425 shader_->set("straighten_alpha", straighten_alpha);
\r
427 ogl_->viewport(0, 0, background->width(), background->height());
\r
430 glMultiTexCoord2d(GL_TEXTURE0, 0.0, 0.0); glVertex2d(-1.0, -1.0);
\r
431 glMultiTexCoord2d(GL_TEXTURE0, 1.0, 0.0); glVertex2d( 1.0, -1.0);
\r
432 glMultiTexCoord2d(GL_TEXTURE0, 1.0, 1.0); glVertex2d( 1.0, 1.0);
\r
433 glMultiTexCoord2d(GL_TEXTURE0, 0.0, 1.0); glVertex2d(-1.0, 1.0);
\r
436 glTextureBarrierNV();
\r
439 ogl_->enable(GL_BLEND);
\r
443 image_kernel::image_kernel(const safe_ptr<ogl_device>& ogl) : impl_(new implementation(ogl)){}
\r
444 void image_kernel::draw(draw_params&& params)
\r
446 impl_->draw(std::move(params));
\r
449 void image_kernel::post_process(
\r
450 const safe_ptr<device_buffer>& background, bool straighten_alpha)
\r
452 impl_->post_process(background, straighten_alpha);
\r