]> git.sesse.net Git - casparcg/blob - accelerator/ogl/image/image_kernel.cpp
[mixer] #503 Fixed bug where already drawn GL_QUADS were not composited against,...
[casparcg] / accelerator / ogl / image / image_kernel.cpp
1 /*
2 * Copyright (c) 2011 Sveriges Television AB <info@casparcg.com>
3 *
4 * This file is part of CasparCG (www.casparcg.com).
5 *
6 * CasparCG is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
10 *
11 * CasparCG is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with CasparCG. If not, see <http://www.gnu.org/licenses/>.
18 *
19 * Author: Robert Nagy, ronag89@gmail.com
20 */
21
22 #include "../../StdAfx.h"
23
24 #include "image_kernel.h"
25
26 #include "image_shader.h"
27 #include "blending_glsl.h"
28
29 #include "../util/shader.h"
30 #include "../util/texture.h"
31 #include "../util/device.h"
32
33 #include <common/except.h>
34 #include <common/gl/gl_check.h>
35 #include <common/env.h>
36
37 #include <core/video_format.h>
38 #include <core/frame/pixel_format.h>
39 #include <core/frame/frame_transform.h>
40
41 #include <boost/algorithm/cxx11/all_of.hpp>
42 #include <boost/lexical_cast.hpp>
43 #include <boost/range/adaptor/transformed.hpp>
44
45 #include <cmath>
46
47 namespace caspar { namespace accelerator { namespace ogl {
48
49 // http://stackoverflow.com/questions/563198/how-do-you-detect-where-two-line-segments-intersect
50 bool get_line_intersection(
51                 double p0_x, double p0_y,
52                 double p1_x, double p1_y,
53                 double p2_x, double p2_y,
54                 double p3_x, double p3_y,
55                 double& result_x, double& result_y)
56 {
57         double s1_x = p1_x - p0_x;
58         double s1_y = p1_y - p0_y;
59         double s2_x = p3_x - p2_x;
60         double s2_y = p3_y - p2_y;
61
62         double s = (-s1_y * (p0_x - p2_x) + s1_x * (p0_y - p2_y)) / (-s2_x * s1_y + s1_x * s2_y);
63         double t = (s2_x * (p0_y - p2_y) - s2_y * (p0_x - p2_x)) / (-s2_x * s1_y + s1_x * s2_y);
64
65         if (s >= 0 && s <= 1 && t >= 0 && t <= 1)
66         {
67                 // Collision detected
68                 result_x = p0_x + (t * s1_x);
69                 result_y = p0_y + (t * s1_y);
70
71                 return true;
72         }
73
74         return false; // No collision
75 }
76
77 double hypotenuse(double x1, double y1, double x2, double y2)
78 {
79         auto x = x2 - x1;
80         auto y = y2 - y1;
81
82         return std::sqrt(x * x + y * y);
83 }
84
85 double calc_q(double close_diagonal, double distant_diagonal)
86 {
87         return (close_diagonal + distant_diagonal) / distant_diagonal;
88 }
89
90 bool is_above_screen(double y)
91 {
92         return y < 0.0;
93 }
94
95 bool is_below_screen(double y)
96 {
97         return y > 1.0;
98 }
99
100 bool is_left_of_screen(double x)
101 {
102         return x < 0.0;
103 }
104
105 bool is_right_of_screen(double x)
106 {
107         return x > 1.0;
108 }
109
110 bool is_outside_screen(const std::vector<core::frame_geometry::coord>& coords)
111 {
112         auto x_coords = coords | boost::adaptors::transformed([](const core::frame_geometry::coord& c) { return c.vertex_x; });
113         auto y_coords = coords | boost::adaptors::transformed([](const core::frame_geometry::coord& c) { return c.vertex_y; });
114
115         return boost::algorithm::all_of(x_coords, &is_left_of_screen)
116                 || boost::algorithm::all_of(x_coords, &is_right_of_screen)
117                 || boost::algorithm::all_of(y_coords, &is_above_screen)
118                 || boost::algorithm::all_of(y_coords, &is_below_screen);
119 }
120
121 GLubyte upper_pattern[] = {
122         0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
123         0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
124         0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
125         0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00};
126
127 GLubyte lower_pattern[] = {
128         0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
129         0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
130         0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
131         0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff};
132
133 struct image_kernel::impl
134 {
135         spl::shared_ptr<device> ogl_;
136         spl::shared_ptr<shader> shader_;
137         bool                                    blend_modes_;
138         bool                                    post_processing_;
139         bool                                    supports_texture_barrier_       = glTextureBarrierNV != 0;
140
141         impl(const spl::shared_ptr<device>& ogl, bool blend_modes_wanted, bool straight_alpha_wanted)
142                 : ogl_(ogl)
143                 , shader_(ogl_->invoke([&]{return get_image_shader(ogl, blend_modes_, blend_modes_wanted, post_processing_, straight_alpha_wanted); }))
144         {
145                 if (!supports_texture_barrier_)
146                         CASPAR_LOG(warning) << L"[image_mixer] TextureBarrierNV not supported. Post processing will not be available";
147         }
148
149         void draw(draw_params params)
150         {
151                 static const double epsilon = 0.001;
152
153                 CASPAR_ASSERT(params.pix_desc.planes.size() == params.textures.size());
154
155                 if(params.textures.empty() || !params.background)
156                         return;
157
158                 if(params.transform.opacity < epsilon)
159                         return;
160
161                 auto coords = params.geometry.data();
162
163                 if (coords.empty())
164                         return;
165
166                 // Calculate transforms
167                 auto f_p = params.transform.fill_translation;
168                 auto f_s = params.transform.fill_scale;
169
170                 bool is_default_geometry = boost::equal(coords, core::frame_geometry::get_default().data());
171                 auto aspect = params.aspect_ratio;
172                 auto angle = params.transform.angle;
173                 auto anchor = params.transform.anchor;
174                 auto crop = params.transform.crop;
175                 auto pers = params.transform.perspective;
176                 pers.ur[0] -= 1.0;
177                 pers.lr[0] -= 1.0;
178                 pers.lr[1] -= 1.0;
179                 pers.ll[1] -= 1.0;
180                 std::vector<boost::array<double, 2>> pers_corners = { pers.ul, pers.ur, pers.lr, pers.ll };
181
182                 auto do_crop = [&](core::frame_geometry::coord& coord)
183                 {
184                         if (!is_default_geometry)
185                                 // TODO implement support for non-default geometry.
186                                 return;
187
188                         coord.vertex_x = std::max(coord.vertex_x, crop.ul[0]);
189                         coord.vertex_x = std::min(coord.vertex_x, crop.lr[0]);
190                         coord.vertex_y = std::max(coord.vertex_y, crop.ul[1]);
191                         coord.vertex_y = std::min(coord.vertex_y, crop.lr[1]);
192                         coord.texture_x = std::max(coord.texture_x, crop.ul[0]);
193                         coord.texture_x = std::min(coord.texture_x, crop.lr[0]);
194                         coord.texture_y = std::max(coord.texture_y, crop.ul[1]);
195                         coord.texture_y = std::min(coord.texture_y, crop.lr[1]);
196                 };
197                 auto do_perspective = [=](core::frame_geometry::coord& coord, const boost::array<double, 2>& pers_corner)
198                 {
199                         if (!is_default_geometry)
200                                 // TODO implement support for non-default geometry.
201                                 return;
202
203                         coord.vertex_x += pers_corner[0];
204                         coord.vertex_y += pers_corner[1];
205                 };
206                 auto rotate = [&](core::frame_geometry::coord& coord)
207                 {
208                         auto orig_x = (coord.vertex_x - anchor[0]) * f_s[0];
209                         auto orig_y = (coord.vertex_y - anchor[1]) * f_s[1] / aspect;
210                         coord.vertex_x = orig_x * std::cos(angle) - orig_y * std::sin(angle);
211                         coord.vertex_y = orig_x * std::sin(angle) + orig_y * std::cos(angle);
212                         coord.vertex_y *= aspect;
213                 };
214                 auto move = [&](core::frame_geometry::coord& coord)
215                 {
216                         coord.vertex_x += f_p[0];
217                         coord.vertex_y += f_p[1];
218                 };
219
220                 int corner = 0;
221                 for (auto& coord : coords)
222                 {
223                         do_crop(coord);
224                         do_perspective(coord, pers_corners.at(corner));
225                         rotate(coord);
226                         move(coord);
227
228                         if (++corner == 4)
229                                 corner = 0;
230                 }
231
232                 // Skip drawing if all the coordinates will be outside the screen.
233                 if (is_outside_screen(coords))
234                         return;
235
236                 // Bind textures
237
238                 for(int n = 0; n < params.textures.size(); ++n)
239                         params.textures[n]->bind(n);
240
241                 if(params.local_key)
242                         params.local_key->bind(static_cast<int>(texture_id::local_key));
243
244                 if(params.layer_key)
245                         params.layer_key->bind(static_cast<int>(texture_id::layer_key));
246
247                 // Setup shader
248
249                 shader_->use();
250
251                 shader_->set("post_processing", false);
252                 shader_->set("plane[0]",                texture_id::plane0);
253                 shader_->set("plane[1]",                texture_id::plane1);
254                 shader_->set("plane[2]",                texture_id::plane2);
255                 shader_->set("plane[3]",                texture_id::plane3);
256                 for (int n = 0; n < params.textures.size(); ++n)
257                         shader_->set("plane_size[" + boost::lexical_cast<std::string>(n) + "]",
258                                                  static_cast<float>(params.textures[n]->width()),
259                                                  static_cast<float>(params.textures[n]->height()));
260
261                 shader_->set("local_key",               texture_id::local_key);
262                 shader_->set("layer_key",               texture_id::layer_key);
263                 shader_->set("is_hd",                   params.pix_desc.planes.at(0).height > 700 ? 1 : 0);
264                 shader_->set("has_local_key",   static_cast<bool>(params.local_key));
265                 shader_->set("has_layer_key",   static_cast<bool>(params.layer_key));
266                 shader_->set("pixel_format",            params.pix_desc.format);
267                 shader_->set("opacity",                 params.transform.is_key ? 1.0 : params.transform.opacity);
268
269                 if (params.transform.chroma.enable)
270                 {
271                         shader_->set("chroma",                                          true);
272
273                         shader_->set("chroma_show_mask",                                                params.transform.chroma.show_mask);
274                         shader_->set("chroma_target_hue",                                       params.transform.chroma.target_hue / 360.0);
275                         shader_->set("chroma_hue_width",                                                params.transform.chroma.hue_width);
276                         shader_->set("chroma_min_saturation",                           params.transform.chroma.min_saturation);
277                         shader_->set("chroma_min_brightness",                           params.transform.chroma.min_brightness);
278                         shader_->set("chroma_softness",                                         1.0 + params.transform.chroma.softness);
279                         shader_->set("chroma_spill_suppress",                           params.transform.chroma.spill_suppress / 360.0);
280                         shader_->set("chroma_spill_suppress_saturation",                params.transform.chroma.spill_suppress_saturation);
281                 }
282                 else
283                         shader_->set("chroma", false);
284
285
286                 // Setup blend_func
287
288                 if(params.transform.is_key)
289                         params.blend_mode = core::blend_mode::normal;
290
291                 if(blend_modes_)
292                 {
293                         params.background->bind(static_cast<int>(texture_id::background));
294
295                         shader_->set("background",      texture_id::background);
296                         shader_->set("blend_mode",      params.blend_mode);
297                         shader_->set("keyer",           params.keyer);
298                 }
299                 else
300                 {
301                         GL(glEnable(GL_BLEND));
302
303                         switch(params.keyer)
304                         {
305                         case keyer::additive:
306                                 GL(glBlendFunc(GL_ONE, GL_ONE));
307                                 break;
308                         case keyer::linear:
309                         default:
310                                 GL(glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA));
311                         }
312                 }
313
314                 // Setup image-adjustements
315
316                 if (params.transform.levels.min_input  > epsilon                ||
317                         params.transform.levels.max_input  < 1.0-epsilon        ||
318                         params.transform.levels.min_output > epsilon            ||
319                         params.transform.levels.max_output < 1.0-epsilon        ||
320                         std::abs(params.transform.levels.gamma - 1.0) > epsilon)
321                 {
322                         shader_->set("levels", true);
323                         shader_->set("min_input",       params.transform.levels.min_input);
324                         shader_->set("max_input",       params.transform.levels.max_input);
325                         shader_->set("min_output",      params.transform.levels.min_output);
326                         shader_->set("max_output",      params.transform.levels.max_output);
327                         shader_->set("gamma",           params.transform.levels.gamma);
328                 }
329                 else
330                         shader_->set("levels", false);
331
332                 if (std::abs(params.transform.brightness - 1.0) > epsilon ||
333                         std::abs(params.transform.saturation - 1.0) > epsilon ||
334                         std::abs(params.transform.contrast - 1.0)   > epsilon)
335                 {
336                         shader_->set("csb",     true);
337
338                         shader_->set("brt", params.transform.brightness);
339                         shader_->set("sat", params.transform.saturation);
340                         shader_->set("con", params.transform.contrast);
341                 }
342                 else
343                         shader_->set("csb",     false);
344
345                 // Setup interlacing
346
347                 if (params.transform.field_mode != core::field_mode::progressive)
348                 {
349                         GL(glEnable(GL_POLYGON_STIPPLE));
350
351                         if(params.transform.field_mode == core::field_mode::upper)
352                                 glPolygonStipple(upper_pattern);
353                         else if(params.transform.field_mode == core::field_mode::lower)
354                                 glPolygonStipple(lower_pattern);
355                 }
356
357                 // Setup drawing area
358
359                 GL(glViewport(0, 0, params.background->width(), params.background->height()));
360                 glDisable(GL_DEPTH_TEST);
361
362                 auto m_p = params.transform.clip_translation;
363                 auto m_s = params.transform.clip_scale;
364
365                 bool scissor = m_p[0] > std::numeric_limits<double>::epsilon()                  || m_p[1] > std::numeric_limits<double>::epsilon() ||
366                                            m_s[0] < (1.0 - std::numeric_limits<double>::epsilon())      || m_s[1] < (1.0 - std::numeric_limits<double>::epsilon());
367
368                 if (scissor)
369                 {
370                         double w = static_cast<double>(params.background->width());
371                         double h = static_cast<double>(params.background->height());
372
373                         GL(glEnable(GL_SCISSOR_TEST));
374                         glScissor(static_cast<int>(m_p[0] * w), static_cast<int>(m_p[1] * h), std::max(0, static_cast<int>(m_s[0] * w)), std::max(0, static_cast<int>(m_s[1] * h)));
375                 }
376
377                 // Set render target
378                 params.background->attach();
379
380                 // Perspective correction
381                 double diagonal_intersection_x;
382                 double diagonal_intersection_y;
383
384                 if (get_line_intersection(
385                                 pers.ul[0] + crop.ul[0], pers.ul[1] + crop.ul[1],
386                                 pers.lr[0] + crop.lr[0], pers.lr[1] + crop.lr[1],
387                                 pers.ur[0] + crop.lr[0], pers.ur[1] + crop.ul[1],
388                                 pers.ll[0] + crop.ul[0], pers.ll[1] + crop.lr[1],
389                                 diagonal_intersection_x,
390                                 diagonal_intersection_y) &&
391                         is_default_geometry)
392                 {
393                         // http://www.reedbeta.com/blog/2012/05/26/quadrilateral-interpolation-part-1/
394                         auto d0 = hypotenuse(pers.ll[0] + crop.ul[0], pers.ll[1] + crop.lr[1], diagonal_intersection_x, diagonal_intersection_y);
395                         auto d1 = hypotenuse(pers.lr[0] + crop.lr[0], pers.lr[1] + crop.lr[1], diagonal_intersection_x, diagonal_intersection_y);
396                         auto d2 = hypotenuse(pers.ur[0] + crop.lr[0], pers.ur[1] + crop.ul[1], diagonal_intersection_x, diagonal_intersection_y);
397                         auto d3 = hypotenuse(pers.ul[0] + crop.ul[0], pers.ul[1] + crop.ul[1], diagonal_intersection_x, diagonal_intersection_y);
398
399                         auto ulq = calc_q(d3, d1);
400                         auto urq = calc_q(d2, d0);
401                         auto lrq = calc_q(d1, d3);
402                         auto llq = calc_q(d0, d2);
403
404                         std::vector<double> q_values = { ulq, urq, lrq, llq };
405
406                         corner = 0;
407                         for (auto& coord : coords)
408                         {
409                                 coord.texture_q = q_values[corner];
410                                 coord.texture_x *= q_values[corner];
411                                 coord.texture_y *= q_values[corner];
412
413                                 if (++corner == 4)
414                                         corner = 0;
415                         }
416                 }
417
418                 // Draw
419                 switch(params.geometry.type())
420                 {
421                 case core::frame_geometry::geometry_type::quad:
422                 case core::frame_geometry::geometry_type::quad_list:
423                         {
424                                 glClientActiveTexture(GL_TEXTURE0);
425
426                                 glDisableClientState(GL_EDGE_FLAG_ARRAY);
427                                 glDisableClientState(GL_COLOR_ARRAY);
428                                 glDisableClientState(GL_INDEX_ARRAY);
429                                 glDisableClientState(GL_NORMAL_ARRAY);
430
431                                 glEnableClientState(GL_TEXTURE_COORD_ARRAY);
432                                 glEnableClientState(GL_VERTEX_ARRAY);
433
434                                 auto stride = static_cast<GLsizei>(sizeof(core::frame_geometry::coord));
435                                 auto vertex_coord_member = &core::frame_geometry::coord::vertex_x;
436                                 auto texture_coord_member = &core::frame_geometry::coord::texture_x;
437                                 auto data_ptr = coords.data();
438                                 auto vertex_coord_ptr = &(data_ptr->*vertex_coord_member);
439                                 auto texture_coord_ptr = &(data_ptr->*texture_coord_member);
440
441                                 glVertexPointer(2, GL_DOUBLE, stride, vertex_coord_ptr);
442                                 glTexCoordPointer(4, GL_DOUBLE, stride, texture_coord_ptr);
443
444                                 if (blend_modes_)
445                                 {
446                                         for (int i = 0; i < coords.size(); i += 4)
447                                         {
448                                                 // http://www.opengl.org/registry/specs/NV/texture_barrier.txt
449                                                 // This allows us to use framebuffer (background) both as source and target while blending.
450                                                 glTextureBarrierNV();
451                                                 glDrawArrays(GL_QUADS, i, 4);
452                                         }
453                                 }
454                                 else
455                                         glDrawArrays(GL_QUADS, 0, static_cast<GLsizei>(coords.size()));
456
457
458                                 glDisableClientState(GL_TEXTURE_COORD_ARRAY);
459                                 glDisableClientState(GL_VERTEX_ARRAY);
460                         }
461                         break;
462                 default:
463                         break;
464                 }
465
466                 // Cleanup
467                 GL(glDisable(GL_SCISSOR_TEST));
468                 GL(glDisable(GL_POLYGON_STIPPLE));
469                 GL(glDisable(GL_BLEND));
470         }
471
472         void post_process(const std::shared_ptr<texture>& background, bool straighten_alpha)
473         {
474                 bool should_post_process =
475                                 supports_texture_barrier_
476                                 && straighten_alpha
477                                 && post_processing_;
478
479                 if (!should_post_process)
480                         return;
481
482                 background->attach();
483
484                 background->bind(static_cast<int>(texture_id::background));
485
486                 shader_->use();
487                 shader_->set("background", texture_id::background);
488                 shader_->set("post_processing", true);
489                 shader_->set("straighten_alpha", straighten_alpha);
490
491                 GL(glViewport(0, 0, background->width(), background->height()));
492
493                 glBegin(GL_QUADS);
494                         glMultiTexCoord2d(GL_TEXTURE0, 0.0, 0.0); glVertex2d(0.0, 0.0);
495                         glMultiTexCoord2d(GL_TEXTURE0, 1.0, 0.0); glVertex2d(1.0, 0.0);
496                         glMultiTexCoord2d(GL_TEXTURE0, 1.0, 1.0); glVertex2d(1.0, 1.0);
497                         glMultiTexCoord2d(GL_TEXTURE0, 0.0, 1.0); glVertex2d(0.0, 1.0);
498                 glEnd();
499
500                 glTextureBarrierNV();
501         }
502 };
503
504 image_kernel::image_kernel(const spl::shared_ptr<device>& ogl, bool blend_modes_wanted, bool straight_alpha_wanted) : impl_(new impl(ogl, blend_modes_wanted, straight_alpha_wanted)){}
505 image_kernel::~image_kernel(){}
506 void image_kernel::draw(const draw_params& params){impl_->draw(params);}
507 void image_kernel::post_process(const std::shared_ptr<texture>& background, bool straighten_alpha) { impl_->post_process(background, straighten_alpha);}
508 bool image_kernel::has_blend_modes() const{return impl_->blend_modes_;}
509
510 }}}