X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=core%2Fmixer%2Fimage%2Fimage_kernel.cpp;h=1eeb50bd334ecb71c55460b96d4408a0e99a0485;hb=0324a1c7714f2f71063c274a4e03953a6145ff9e;hp=c5bade3910401f25f79f12d12083ed436145cf27;hpb=3b791188a22bf082d1327d1e8f4489bc2f796073;p=casparcg diff --git a/core/mixer/image/image_kernel.cpp b/core/mixer/image/image_kernel.cpp index c5bade391..1eeb50bd3 100644 --- a/core/mixer/image/image_kernel.cpp +++ b/core/mixer/image/image_kernel.cpp @@ -20,16 +20,23 @@ #include "../../stdafx.h" #include "image_kernel.h" + +#include "image_shader.h" + #include "blending_glsl.h" #include "../gpu/shader.h" #include "../gpu/device_buffer.h" +#include "../gpu/ogl_device.h" #include #include +#include #include #include -#include +#include + +#include #include @@ -37,332 +44,200 @@ namespace caspar { namespace core { +GLubyte upper_pattern[] = { + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00}; + +GLubyte lower_pattern[] = { + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff}; + struct image_kernel::implementation : boost::noncopyable { - std::unique_ptr shader_; - - implementation() + std::shared_ptr shader_; + bool blend_modes_; + + void draw(ogl_device& ogl, draw_params&& params) { - std::string vertex = - "void main() \n" - "{ \n" - " gl_TexCoord[0] = gl_MultiTexCoord0; \n" - " gl_TexCoord[1] = gl_MultiTexCoord1; \n" - " gl_FrontColor = gl_Color; \n" - " gl_Position = ftransform(); \n" - "} \n"; - - std::string fragment = std::string() + - "uniform sampler2D background; \n" - "uniform sampler2D plane[4]; \n" - "uniform sampler2D local_key; \n" - "uniform sampler2D layer_key; \n" - " \n" - "uniform bool is_hd; \n" - "uniform bool has_local_key; \n" - "uniform bool has_layer_key; \n" - "uniform int blend_mode; \n" - "uniform int interlace_mode; \n" - "uniform int pixel_format; \n" - " \n" - "uniform float gain; \n" - " \n" - "uniform bool levels; \n" - "uniform float min_input; \n" - "uniform float max_input; \n" - "uniform float gamma; \n" - "uniform float min_output; \n" - "uniform float max_output; \n" - " \n" - "uniform bool csb; \n" - "uniform float brt; \n" - "uniform float sat; \n" - "uniform float con; \n" - " \n" - - + - - get_blend_glsl() - - + - - "vec3 get_blend_color(vec3 back, vec3 fore) \n" - "{ \n" - " switch(blend_mode) \n" - " { \n" - " case 0: return BlendNormal(back, fore); \n" - " case 1: return BlendLighten(back, fore); \n" - " case 2: return BlendDarken(back, fore); \n" - " case 3: return BlendMultiply(back, fore); \n" - " case 4: return BlendAverage(back, fore); \n" - " case 5: return BlendAdd(back, fore); \n" - " case 6: return BlendSubstract(back, fore); \n" - " case 7: return BlendDifference(back, fore); \n" - " case 8: return BlendNegation(back, fore); \n" - " case 9: return BlendExclusion(back, fore); \n" - " case 10: return BlendScreen(back, fore); \n" - " case 11: return BlendOverlay(back, fore); \n" - //" case 12: return BlendSoftLight(back, fore); \n" - " case 13: return BlendHardLight(back, fore); \n" - " case 14: return BlendColorDodge(back, fore); \n" - " case 15: return BlendColorBurn(back, fore); \n" - " case 16: return BlendLinearDodge(back, fore); \n" - " case 17: return BlendLinearBurn(back, fore); \n" - " case 18: return BlendLinearLight(back, fore); \n" - " case 19: return BlendVividLight(back, fore); \n" - " case 20: return BlendPinLight(back, fore); \n" - " case 21: return BlendHardMix(back, fore); \n" - " case 22: return BlendReflect(back, fore); \n" - " case 23: return BlendGlow(back, fore); \n" - " case 24: return BlendPhoenix(back, fore); \n" - " case 25: return BlendHue(back, fore); \n" - " case 26: return BlendSaturation(back, fore); \n" - " case 27: return BlendColor(back, fore); \n" - " case 28: return BlendLuminosity(back, fore); \n" - " } \n" - " \n" - " return BlendNormal(back, fore); \n" - "} \n" - " \n" - "vec4 blend_color(vec4 fore) \n" - "{ \n" - " fore.rgb = fore.bgr; // bgr to rgb \n" - " vec4 back = texture2D(background, gl_TexCoord[1].st); \n" - " if(levels) \n" - " fore.rgb = LevelsControl(fore.rgb, min_input, max_input, gamma, min_output, max_output); \n" - " if(csb) \n" - " fore.rgb = ContrastSaturationBrightness(fore.rgb, brt, sat, con); \n" - " fore.rgb = get_blend_color(back.bgr, fore.rgb); \n" - " fore.rgb = fore.bgr; // rgb to bgr \n" - " return vec4(fore.rgb * fore.a + back.rgb * (1.0-fore.a), back.a + fore.a); \n" - "} \n" - " \n" - "// NOTE: YCbCr, ITU-R, http://www.intersil.com/data/an/an9717.pdf \n" - "// TODO: Support for more yuv formats might be needed. \n" - "vec4 ycbcra_to_bgra_sd(float y, float cb, float cr, float a) \n" - "{ \n" - " cb -= 0.5; \n" - " cr -= 0.5; \n" - " y = 1.164*(y-0.0625); \n" - " \n" - " vec4 color; \n" - " color.r = y + 1.596 * cr; \n" - " color.g = y - 0.813 * cr - 0.391 * cb; \n" - " color.b = y + 2.018 * cb; \n" - " color.a = a; \n" - " \n" - " return color; \n" - "} \n" - " \n" - "vec4 ycbcra_to_bgra_hd(float y, float cb, float cr, float a) \n" - "{ \n" - " cb -= 0.5; \n" - " cr -= 0.5; \n" - " y = 1.164*(y-0.0625); \n" - " \n" - " vec4 color; \n" - " color.r = y + 1.793 * cr; \n" - " color.g = y - 0.534 * cr - 0.213 * cb; \n" - " color.b = y + 2.115 * cb; \n" - " color.a = a; \n" - " \n" - " return color; \n" - "} \n" - " \n" - "vec4 ycbcra_to_bgra(float y, float cb, float cr, float a) \n" - "{ \n" - " if(is_hd) \n" - " return ycbcra_to_bgra_hd(y, cb, cr, a); \n" - " else \n" - " return ycbcra_to_bgra_sd(y, cb, cr, a); \n" - "} \n" - " \n" - "vec4 get_bgra_color() \n" - "{ \n" - " switch(pixel_format) \n" - " { \n" - " case 0: //gray \n" - " return vec4(texture2D(plane[0], gl_TexCoord[0].st).rrr, 1.0); \n" - " case 1: //bgra, \n" - " return texture2D(plane[0], gl_TexCoord[0].st).rgba; \n" - " case 2: //rgba, \n" - " return texture2D(plane[0], gl_TexCoord[0].st).bgra; \n" - " case 3: //argb, \n" - " return texture2D(plane[0], gl_TexCoord[0].st).abgr; \n" - " case 4: //abgr, \n" - " return texture2D(plane[0], gl_TexCoord[0].st).grab; \n" - " case 5: //ycbcr, \n" - " { \n" - " float y = texture2D(plane[0], gl_TexCoord[0].st).r; \n" - " float cb = texture2D(plane[1], gl_TexCoord[0].st).r; \n" - " float cr = texture2D(plane[2], gl_TexCoord[0].st).r; \n" - " return ycbcra_to_bgra(y, cb, cr, 1.0); \n" - " } \n" - " case 6: //ycbcra \n" - " { \n" - " float y = texture2D(plane[0], gl_TexCoord[0].st).r; \n" - " float cb = texture2D(plane[1], gl_TexCoord[0].st).r; \n" - " float cr = texture2D(plane[2], gl_TexCoord[0].st).r; \n" - " float a = texture2D(plane[3], gl_TexCoord[0].st).r; \n" - " return ycbcra_to_bgra(y, cb, cr, a); \n" - " } \n" - " } \n" - " return vec4(0.0, 0.0, 0.0, 0.0); \n" - "} \n" - " \n" - "void main() \n" - "{ \n" - " switch(interlace_mode) \n" - " { \n" - " case 1: // lower \n" - " { \n" - " bool odd = mod(floor(gl_FragCoord.y), 2.0) > 0.5; \n" - " if(!odd) \n" - " discard; \n" - " break; \n" - " } \n" - " case 2: //upper \n" - " { \n" - " bool odd = mod(floor(gl_FragCoord.y), 2.0) > 0.5; \n" - " if(odd) \n" - " discard; \n" - " break; \n" - " } \n" - " } \n" - " \n" - " vec4 color = get_bgra_color(); \n" - " if(has_local_key) \n" - " color.a *= texture2D(local_key, gl_TexCoord[1].st).r; \n" - " if(has_layer_key) \n" - " color.a *= texture2D(layer_key, gl_TexCoord[1].st).r; \n" - " gl_FragColor = blend_color(color * gl_Color * gain); \n" - "} \n" - ; - - shader_.reset(new shader(vertex, fragment)); - } + static const double epsilon = 0.001; - - void draw(size_t width, - size_t height, - const core::pixel_format_desc& pix_desc, - const core::image_transform& transform, - const std::vector>& planes, - const safe_ptr& background, - const std::shared_ptr& local_key, - const std::shared_ptr& layer_key) - { - GL(glEnable(GL_TEXTURE_2D)); - - // Setup depth for interlacing and explicit z-culling - double z = 0.0; - if(transform.get_mode() == core::video_mode::upper) - { - GL(glEnable(GL_DEPTH_TEST)); - z = 1.0; - } - else if(transform.get_mode() == core::video_mode::lower) - GL(glClear(GL_DEPTH_BUFFER_BIT)); + ogl.yield(); - // Bind textures + CASPAR_ASSERT(params.pix_desc.planes.size() == params.textures.size()); - for(size_t n = 0; n < planes.size(); ++n) - planes[n]->bind(n); + if(params.textures.empty() || !params.background) + return; - if(local_key) - local_key->bind(4); + if(params.transform.opacity < epsilon) + return; - if(layer_key) - layer_key->bind(5); + if(!std::all_of(params.textures.begin(), params.textures.end(), std::mem_fn(&device_buffer::ready))) + { + CASPAR_LOG(warning) << L"[image_mixer] Performance warning. Host to device transfer not complete, GPU will be stalled"; + ogl.yield(); // Try to give it some more time. + } + + // Bind textures - background->bind(6); + for(size_t n = 0; n < params.textures.size(); ++n) + params.textures[n]->bind(n); + if(params.local_key) + params.local_key->bind(texture_id::local_key); + + if(params.layer_key) + params.layer_key->bind(texture_id::layer_key); + // Setup shader - shader_->use(); - - shader_->set("plane[0]", 0); - shader_->set("plane[1]", 1); - shader_->set("plane[2]", 2); - shader_->set("plane[3]", 3); - shader_->set("local_key", 4); - shader_->set("layer_key", 5); - shader_->set("background", 6); - shader_->set("gain", transform.get_gain()); - shader_->set("is_hd", pix_desc.planes.at(0).height > 700 ? 1 : 0); - shader_->set("has_local_key", local_key ? 1 : 0); - shader_->set("has_layer_key", layer_key ? 1 : 0); - shader_->set("blend_mode", transform.get_blend_mode()); - shader_->set("interlace_mode", transform.get_mode()); - shader_->set("pixel_format", pix_desc.pix_fmt); - - auto levels = transform.get_levels(); - - if(levels.min_input > 0.001 || - levels.max_input < 0.999 || - levels.min_output > 0.001 || - levels.max_output < 0.999 || - std::abs(levels.gamma - 1.0) > 0.001) + if(!shader_) + shader_ = get_image_shader(ogl, blend_modes_); + + ogl.use(*shader_); + + shader_->set("plane[0]", texture_id::plane0); + shader_->set("plane[1]", texture_id::plane1); + shader_->set("plane[2]", texture_id::plane2); + shader_->set("plane[3]", texture_id::plane3); + shader_->set("local_key", texture_id::local_key); + shader_->set("layer_key", texture_id::layer_key); + shader_->set("is_hd", params.pix_desc.planes.at(0).height > 700 ? 1 : 0); + shader_->set("has_local_key", params.local_key); + shader_->set("has_layer_key", params.layer_key); + shader_->set("pixel_format", params.pix_desc.pix_fmt); + shader_->set("opacity", params.transform.is_key ? 1.0 : params.transform.opacity); + + // Setup blend_func + + if(params.transform.is_key) + params.blend_mode = blend_mode::normal; + + if(blend_modes_) + { + params.background->bind(6); + + shader_->set("background", texture_id::background); + shader_->set("blend_mode", params.blend_mode); + shader_->set("keyer", params.keyer); + } + else + { + switch(params.keyer) + { + case keyer::additive: + ogl.blend_func(GL_ONE, GL_ONE); + break; + case keyer::linear: + default: + ogl.blend_func(GL_ONE, GL_ONE_MINUS_SRC_ALPHA); + } + } + + // Setup image-adjustements + + if(params.transform.levels.min_input > epsilon || + params.transform.levels.max_input < 1.0-epsilon || + params.transform.levels.min_output > epsilon || + params.transform.levels.max_output < 1.0-epsilon || + std::abs(params.transform.levels.gamma - 1.0) > epsilon) { shader_->set("levels", true); - shader_->set("min_input", levels.min_input); - shader_->set("max_input", levels.max_input); - shader_->set("min_output", levels.min_output); - shader_->set("max_output", levels.max_output); - shader_->set("gamma", levels.gamma); + shader_->set("min_input", params.transform.levels.min_input); + shader_->set("max_input", params.transform.levels.max_input); + shader_->set("min_output", params.transform.levels.min_output); + shader_->set("max_output", params.transform.levels.max_output); + shader_->set("gamma", params.transform.levels.gamma); } else shader_->set("levels", false); - if(std::abs(transform.get_brightness() - 1.0) > 0.001 || - std::abs(transform.get_saturation() - 1.0) > 0.001 || - std::abs(transform.get_contrast() - 1.0) > 0.001) + if(std::abs(params.transform.brightness - 1.0) > epsilon || + std::abs(params.transform.saturation - 1.0) > epsilon || + std::abs(params.transform.contrast - 1.0) > epsilon) { shader_->set("csb", true); - shader_->set("brt", transform.get_brightness()); - shader_->set("sat", transform.get_saturation()); - shader_->set("con", transform.get_contrast()); + shader_->set("brt", params.transform.brightness); + shader_->set("sat", params.transform.saturation); + shader_->set("con", params.transform.contrast); } else shader_->set("csb", false); + // Setup interlacing + + if(params.transform.field_mode == core::field_mode::progressive) + ogl.disable(GL_POLYGON_STIPPLE); + else + { + ogl.enable(GL_POLYGON_STIPPLE); + + if(params.transform.field_mode == core::field_mode::upper) + ogl.stipple_pattern(upper_pattern); + else if(params.transform.field_mode == core::field_mode::lower) + ogl.stipple_pattern(lower_pattern); + } + // Setup drawing area + + ogl.viewport(0, 0, params.background->width(), params.background->height()); + + auto m_p = params.transform.clip_translation; + auto m_s = params.transform.clip_scale; - GL(glColor4d(1.0, 1.0, 1.0, transform.get_opacity())); - GL(glViewport(0, 0, width, height)); - - auto m_p = transform.get_clip_translation(); - auto m_s = transform.get_clip_scale(); - double w = static_cast(width); - double h = static_cast(height); + bool scissor = m_p[0] > std::numeric_limits::epsilon() || m_p[1] > std::numeric_limits::epsilon() || + m_s[0] < (1.0 - std::numeric_limits::epsilon()) || m_s[1] < (1.0 - std::numeric_limits::epsilon()); - GL(glEnable(GL_SCISSOR_TEST)); - GL(glScissor(static_cast(m_p[0]*w), static_cast(m_p[1]*h), static_cast(m_s[0]*w), static_cast(m_s[1]*h))); - - auto f_p = transform.get_fill_translation(); - auto f_s = transform.get_fill_scale(); + if(scissor) + { + double w = static_cast(params.background->width()); + double h = static_cast(params.background->height()); + + ogl.enable(GL_SCISSOR_TEST); + ogl.scissor(static_cast(m_p[0]*w), static_cast(m_p[1]*h), static_cast(m_s[0]*w), static_cast(m_s[1]*h)); + } + + auto f_p = params.transform.fill_translation; + auto f_s = params.transform.fill_scale; + + // Set render target + + ogl.yield(); + ogl.attach(*params.background); // Draw glBegin(GL_QUADS); - glMultiTexCoord2d(GL_TEXTURE0, 0.0, 0.0); glMultiTexCoord2d(GL_TEXTURE1, f_p[0] , f_p[1] ); glVertex3d( f_p[0] *2.0-1.0, f_p[1] *2.0-1.0, z); - glMultiTexCoord2d(GL_TEXTURE0, 1.0, 0.0); glMultiTexCoord2d(GL_TEXTURE1, (f_p[0]+f_s[0]), f_p[1] ); glVertex3d((f_p[0]+f_s[0])*2.0-1.0, f_p[1] *2.0-1.0, z); - glMultiTexCoord2d(GL_TEXTURE0, 1.0, 1.0); glMultiTexCoord2d(GL_TEXTURE1, (f_p[0]+f_s[0]), (f_p[1]+f_s[1])); glVertex3d((f_p[0]+f_s[0])*2.0-1.0, (f_p[1]+f_s[1])*2.0-1.0, z); - glMultiTexCoord2d(GL_TEXTURE0, 0.0, 1.0); glMultiTexCoord2d(GL_TEXTURE1, f_p[0] , (f_p[1]+f_s[1])); glVertex3d( f_p[0] *2.0-1.0, (f_p[1]+f_s[1])*2.0-1.0, z); + glMultiTexCoord2d(GL_TEXTURE0, 0.0, 0.0); glMultiTexCoord2d(GL_TEXTURE1, f_p[0] , f_p[1] ); glVertex2d( f_p[0] *2.0-1.0, f_p[1] *2.0-1.0); + glMultiTexCoord2d(GL_TEXTURE0, 1.0, 0.0); glMultiTexCoord2d(GL_TEXTURE1, (f_p[0]+f_s[0]), f_p[1] ); glVertex2d((f_p[0]+f_s[0])*2.0-1.0, f_p[1] *2.0-1.0); + glMultiTexCoord2d(GL_TEXTURE0, 1.0, 1.0); glMultiTexCoord2d(GL_TEXTURE1, (f_p[0]+f_s[0]), (f_p[1]+f_s[1])); glVertex2d((f_p[0]+f_s[0])*2.0-1.0, (f_p[1]+f_s[1])*2.0-1.0); + glMultiTexCoord2d(GL_TEXTURE0, 0.0, 1.0); glMultiTexCoord2d(GL_TEXTURE1, f_p[0] , (f_p[1]+f_s[1])); glVertex2d( f_p[0] *2.0-1.0, (f_p[1]+f_s[1])*2.0-1.0); glEnd(); - GL(glDisable(GL_SCISSOR_TEST)); - GL(glDisable(GL_DEPTH_TEST)); + // Cleanup + + ogl.disable(GL_SCISSOR_TEST); + + params.textures.clear(); + ogl.yield(); // Return resources to pool as early as possible. + + if(blend_modes_) + { + // http://www.opengl.org/registry/specs/NV/texture_barrier.txt + // This allows us to use framebuffer (background) both as source and target while blending. + glTextureBarrierNV(); + } } }; image_kernel::image_kernel() : impl_(new implementation()){} - -void image_kernel::draw(size_t width, size_t height, const core::pixel_format_desc& pix_desc, const core::image_transform& transform, const std::vector>& planes, - const safe_ptr& background, const std::shared_ptr& local_key, const std::shared_ptr& layer_key) +void image_kernel::draw(ogl_device& ogl, draw_params&& params) { - impl_->draw(width, height, pix_desc, transform, planes, background, local_key, layer_key); + impl_->draw(ogl, std::move(params)); } }} \ No newline at end of file