From: Steinar H. Gunderson Date: Mon, 10 Mar 2014 23:40:00 +0000 (+0100) Subject: Merge branch 'master' into epoxy X-Git-Tag: 1.1~12^2~37 X-Git-Url: https://git.sesse.net/?p=movit;a=commitdiff_plain;h=15daf781e6ce00ec37ca806a16e490d59dcb4e24;hp=56ff92c5828a143595eeb7a1906418cc30668f9d Merge branch 'master' into epoxy --- diff --git a/Makefile.in b/Makefile.in index 660bf7a..ea76465 100644 --- a/Makefile.in +++ b/Makefile.in @@ -60,6 +60,7 @@ TESTED_EFFECTS += deconvolution_sharpen_effect TESTED_EFFECTS += fft_pass_effect TESTED_EFFECTS += vignette_effect TESTED_EFFECTS += slice_effect +TESTED_EFFECTS += complex_modulate_effect UNTESTED_EFFECTS = sandbox_effect UNTESTED_EFFECTS += mirror_effect diff --git a/complex_modulate_effect.cpp b/complex_modulate_effect.cpp new file mode 100644 index 0000000..656de3a --- /dev/null +++ b/complex_modulate_effect.cpp @@ -0,0 +1,60 @@ +#include + +#include "complex_modulate_effect.h" +#include "effect_chain.h" +#include "effect_util.h" +#include "util.h" + +using namespace std; + +namespace movit { + +ComplexModulateEffect::ComplexModulateEffect() + : num_repeats_x(1), num_repeats_y(1) +{ + register_int("num_repeats_x", &num_repeats_x); + register_int("num_repeats_y", &num_repeats_y); +} + +string ComplexModulateEffect::output_fragment_shader() +{ + return read_file("complex_modulate_effect.frag"); +} + +void ComplexModulateEffect::set_gl_state(GLuint glsl_program_num, const string &prefix, unsigned *sampler_num) +{ + Effect::set_gl_state(glsl_program_num, prefix, sampler_num); + + float num_repeats[] = { num_repeats_x, num_repeats_y }; + set_uniform_vec2(glsl_program_num, prefix, "num_repeats", num_repeats); + + // Set the secondary input to repeat (and nearest while we're at it). + Node *self = chain->find_node_for_effect(this); + glActiveTexture(chain->get_input_sampler(self, 1)); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT); + check_error(); +} + +void ComplexModulateEffect::inform_input_size(unsigned input_num, unsigned width, unsigned height) +{ + if (input_num == 0) { + primary_input_width = width; + primary_input_height = height; + } +} + +void ComplexModulateEffect::get_output_size(unsigned *width, unsigned *height, + unsigned *virtual_width, unsigned *virtual_height) const +{ + *width = *virtual_width = primary_input_width; + *height = *virtual_height = primary_input_height; +} + +} // namespace movit diff --git a/complex_modulate_effect.frag b/complex_modulate_effect.frag new file mode 100644 index 0000000..46de5da --- /dev/null +++ b/complex_modulate_effect.frag @@ -0,0 +1,9 @@ +uniform vec2 PREFIX(num_repeats); + +vec4 FUNCNAME(vec2 tc) { + vec4 pixel = INPUT1(tc); + vec2 pattern = INPUT2(tc * PREFIX(num_repeats)).xy; + + // Complex multiplication between each of (pixel.xy, pixel.zw) and pattern.xy. + return pattern.x * pixel + pattern.y * vec4(-pixel.y, pixel.x, -pixel.w, pixel.z); +} diff --git a/complex_modulate_effect.h b/complex_modulate_effect.h new file mode 100644 index 0000000..fd4eb8b --- /dev/null +++ b/complex_modulate_effect.h @@ -0,0 +1,61 @@ +#ifndef _MOVIT_COMPLEX_MODULATE_EFFECT_H +#define _MOVIT_COMPLEX_MODULATE_EFFECT_H 1 + +// An effect that treats each pixel as two complex numbers (xy and zw), +// and multiplies it with some other complex number (xy and xy, so the +// same in both cases). The latter can be repeated both horizontally and +// vertically if desired. +// +// The typical use is to implement convolution by way of FFT; since +// FFT(A ⊙ B) = FFT(A) * FFT(B), you can FFT both inputs (where B +// would often even be a constant, so you'd only need to do FFT once), +// multiply them together and then IFFT the result to get a convolution. +// +// It is in a sense “wrong” to do this directly on pixels, since the color +// channels are independent and real-valued (ie., not complex numbers), but +// since convolution is a linear operation, it's unproblematic to treat R + Gi +// as a single complex number and B + Ai and another one; barring numerical +// errors, there should be no leakage between the channels as long as you're +// convolving with a real quantity. (There are more sophisticated ways of doing +// two real FFTs with a single complex one, but we won't need them, as we +// don't care about the actual FFT result, just that the convolution property +// holds.) + +#include +#include + +#include "effect.h" + +namespace movit { + +class EffectChain; + +class ComplexModulateEffect : public Effect { +public: + ComplexModulateEffect(); + virtual std::string effect_type_id() const { return "ComplexModulateEffect"; } + std::string output_fragment_shader(); + + // Technically we only need texture bounce for the second input + // (to be allowed to mess with its sampler state), but there's + // no way of expressing that currently. + virtual bool needs_texture_bounce() const { return true; } + virtual bool changes_output_size() const { return true; } + + virtual void inform_input_size(unsigned input_num, unsigned width, unsigned height); + virtual void get_output_size(unsigned *width, unsigned *height, + unsigned *virtual_width, unsigned *virtual_height) const; + virtual unsigned num_inputs() const { return 2; } + virtual void inform_added(EffectChain *chain) { this->chain = chain; } + + void set_gl_state(GLuint glsl_program_num, const std::string &prefix, unsigned *sampler_num); + +private: + EffectChain *chain; + int primary_input_width, primary_input_height; + int num_repeats_x, num_repeats_y; +}; + +} // namespace movit + +#endif // !defined(_MOVIT_COMPLEX_MODULATE_EFFECT_H) diff --git a/complex_modulate_effect_test.cpp b/complex_modulate_effect_test.cpp new file mode 100644 index 0000000..b505d86 --- /dev/null +++ b/complex_modulate_effect_test.cpp @@ -0,0 +1,99 @@ +// Unit tests for ComplexModulateEffect. + +#include + +#include "effect_chain.h" +#include "gtest/gtest.h" +#include "complex_modulate_effect.h" +#include "image_format.h" +#include "input.h" +#include "test_util.h" + +namespace movit { + +TEST(ComplexModulateEffectTest, Identity) { + const int size = 3; + float data_a[size * 4] = { + 0.0f, 0.1f, 0.2f, 0.1f, + 0.4f, 0.3f, 0.8f, 2.0f, + 0.5f, 0.2f, 0.1f, 0.0f, + }; + float data_b[size * 2] = { + 1.0f, 0.0f, + 1.0f, 0.0f, + 1.0f, 0.0f, + }; + float out_data[size * 4]; + + EffectChainTester tester(data_a, 1, size, FORMAT_RGBA_PREMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); + Effect *input1 = tester.get_chain()->last_added_effect(); + Effect *input2 = tester.add_input(data_b, FORMAT_RG, COLORSPACE_sRGB, GAMMA_LINEAR); + + tester.get_chain()->add_effect(new ComplexModulateEffect(), input1, input2); + tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR, OUTPUT_ALPHA_FORMAT_PREMULTIPLIED); + + expect_equal(data_a, out_data, 4, size); +} + +TEST(ComplexModulateEffectTest, ComplexMultiplication) { + const int size = 2; + float data_a[size * 4] = { + 0.0f, 0.1f, 0.2f, 0.1f, + 0.4f, 0.3f, 0.8f, 2.0f, + }; + float data_b[size * 2] = { + 0.0f, 1.0f, + 0.5f, -0.8f, + }; + float expected_data[size * 4] = { + -0.1f, 0.0f, -0.1f, 0.2f, + 0.44f, -0.17f, 2.0f, 0.36f, + }; + float out_data[size * 4]; + + EffectChainTester tester(data_a, 1, size, FORMAT_RGBA_PREMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); + Effect *input1 = tester.get_chain()->last_added_effect(); + Effect *input2 = tester.add_input(data_b, FORMAT_RG, COLORSPACE_sRGB, GAMMA_LINEAR); + + tester.get_chain()->add_effect(new ComplexModulateEffect(), input1, input2); + tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR, OUTPUT_ALPHA_FORMAT_PREMULTIPLIED); + + expect_equal(expected_data, out_data, 4, size); +} + +TEST(ComplexModulateEffectTest, Repeat) { + const int size = 2, repeats = 3; + float data_a[size * repeats * 4] = { + 0.0f, 0.1f, 0.2f, 0.3f, + 1.0f, 1.1f, 1.2f, 1.3f, + 2.0f, 2.1f, 2.2f, 2.3f, + 3.0f, 3.1f, 3.2f, 3.3f, + 4.0f, 4.1f, 4.2f, 4.3f, + 5.0f, 5.1f, 5.2f, 5.3f, + }; + float data_b[size * 2] = { + 1.0f, 0.0f, + 0.0f, -1.0f, + }; + float expected_data[size * repeats * 4] = { + 0.0f, 0.1f, 0.2f, 0.3f, + 1.1f, -1.0f, 1.3f, -1.2f, + 2.0f, 2.1f, 2.2f, 2.3f, + 3.1f, -3.0f, 3.3f, -3.2f, + 4.0f, 4.1f, 4.2f, 4.3f, + 5.1f, -5.0f, 5.3f, -5.2f, + }; + float out_data[size * repeats * 4]; + + EffectChainTester tester(data_a, 1, repeats * size, FORMAT_RGBA_PREMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); + Effect *input1 = tester.get_chain()->last_added_effect(); + Effect *input2 = tester.add_input(data_b, FORMAT_RG, COLORSPACE_sRGB, GAMMA_LINEAR, 1, size); + + Effect *effect = tester.get_chain()->add_effect(new ComplexModulateEffect(), input1, input2); + ASSERT_TRUE(effect->set_int("num_repeats_y", repeats)); + tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR, OUTPUT_ALPHA_FORMAT_PREMULTIPLIED); + + expect_equal(expected_data, out_data, 4, size * repeats); +} + +} // namespace movit diff --git a/effect_chain.cpp b/effect_chain.cpp index 0cd4802..ef44024 100644 --- a/effect_chain.cpp +++ b/effect_chain.cpp @@ -149,6 +149,15 @@ void EffectChain::insert_node_between(Node *sender, Node *middle, Node *receiver assert(middle->incoming_links.size() == middle->effect->num_inputs()); } +GLenum EffectChain::get_input_sampler(Node *node, unsigned input_num) const +{ + assert(node->effect->needs_texture_bounce()); + assert(input_num < node->incoming_links.size()); + assert(node->incoming_links[input_num]->bound_sampler_num >= 0); + assert(node->incoming_links[input_num]->bound_sampler_num < 8); + return GL_TEXTURE0 + node->incoming_links[input_num]->bound_sampler_num; +} + void EffectChain::find_all_nonlinear_inputs(Node *node, vector *nonlinear_inputs) { if (node->output_gamma_curve == GAMMA_LINEAR && @@ -1480,6 +1489,7 @@ void EffectChain::render_to_fbo(GLuint dest_fbo, unsigned width, unsigned height for (unsigned sampler = 0; sampler < phases[phase]->inputs.size(); ++sampler) { glActiveTexture(GL_TEXTURE0 + sampler); Node *input = phases[phase]->inputs[sampler]; + input->bound_sampler_num = sampler; glBindTexture(GL_TEXTURE_2D, output_textures[input->phase]); check_error(); if (phases[phase]->input_needs_mipmaps) { @@ -1533,8 +1543,16 @@ void EffectChain::render_to_fbo(GLuint dest_fbo, unsigned width, unsigned height unsigned sampler_num = phases[phase]->inputs.size(); for (unsigned i = 0; i < phases[phase]->effects.size(); ++i) { Node *node = phases[phase]->effects[i]; + unsigned old_sampler_num = sampler_num; node->effect->set_gl_state(glsl_program_num, phases[phase]->effect_ids[node], &sampler_num); check_error(); + + if (node->effect->is_single_texture()) { + assert(sampler_num - old_sampler_num == 1); + node->bound_sampler_num = old_sampler_num; + } else { + node->bound_sampler_num = -1; + } } // Now draw! diff --git a/effect_chain.h b/effect_chain.h index 4c3232b..549fa60 100644 --- a/effect_chain.h +++ b/effect_chain.h @@ -71,6 +71,15 @@ private: // phases as inputs, instead of Node. Phase *phase; + // If the effect has is_single_texture(), or if the output went to RTT + // and that texture has been bound to a sampler, the sampler number + // will be stored here. + // + // TODO: Can an RTT texture be used as inputs to multiple effects + // within the same phase? If so, we have a problem with modifying + // sampler state here. + int bound_sampler_num; + // Used during the building of the effect chain. Colorspace output_color_space; GammaCurve output_gamma_curve; @@ -173,6 +182,16 @@ public: void replace_receiver(Node *old_receiver, Node *new_receiver); void replace_sender(Node *new_sender, Node *receiver); void insert_node_between(Node *sender, Node *middle, Node *receiver); + Node *find_node_for_effect(Effect *effect) { return node_map[effect]; } + + // Get the OpenGL sampler (GL_TEXTURE0, GL_TEXTURE1, etc.) for the + // input of the given node, so that one can modify the sampler state + // directly. Only valid to call during set_gl_state(). + // + // Also, for this to be allowed, 's effect must have + // needs_texture_bounce() set, so that it samples directly from a + // single-sampler input, or from an RTT texture. + GLenum get_input_sampler(Node *node, unsigned input_num) const; // Get the current resource pool assigned to this EffectChain. // Primarily to let effects allocate textures as needed. diff --git a/effect_chain_test.cpp b/effect_chain_test.cpp index 67b3dfc..aabcc33 100644 --- a/effect_chain_test.cpp +++ b/effect_chain_test.cpp @@ -522,17 +522,27 @@ class MipmapNeedingEffect : public Effect { public: MipmapNeedingEffect() {} virtual bool needs_mipmaps() const { return true; } + + // To be allowed to mess with the sampler state. + virtual bool needs_texture_bounce() const { return true; } + virtual string effect_type_id() const { return "MipmapNeedingEffect"; } string output_fragment_shader() { return read_file("mipmap_needing_effect.frag"); } + virtual void inform_added(EffectChain *chain) { this->chain = chain; } + void set_gl_state(GLuint glsl_program_num, const string& prefix, unsigned *sampler_num) { - glActiveTexture(GL_TEXTURE0); + Node *self = chain->find_node_for_effect(this); + glActiveTexture(chain->get_input_sampler(self, 0)); check_error(); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT); check_error(); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT); check_error(); } + +private: + EffectChain *chain; }; TEST(EffectChainTest, MipmapGenerationWorks) { diff --git a/fft_pass_effect.cpp b/fft_pass_effect.cpp index ec550b2..0ffaa9e 100644 --- a/fft_pass_effect.cpp +++ b/fft_pass_effect.cpp @@ -1,6 +1,7 @@ #include #include +#include "effect_chain.h" #include "effect_util.h" #include "fp16.h" #include "fft_pass_effect.h" @@ -40,15 +41,13 @@ void FFTPassEffect::set_gl_state(GLuint glsl_program_num, const string &prefix, int input_size = (direction == VERTICAL) ? input_height : input_width; - // See the comments on changes_output_size() in the .h file to see - // why this is legal. It is _needed_ because it counteracts the - // precision issues we get because we sample the input texture with - // normalized coordinates (especially when the repeat count along - // the axis is not a power of two); we very rapidly end up in narrowly - // missing a texel center, which causes precision loss to propagate - // throughout the FFT. - assert(*sampler_num == 1); - glActiveTexture(GL_TEXTURE0); + // This is needed because it counteracts the precision issues we get + // because we sample the input texture with normalized coordinates + // (especially when the repeat count along the axis is not a power of + // two); we very rapidly end up in narrowly missing a texel center, + // which causes precision loss to propagate throughout the FFT. + Node *self = chain->find_node_for_effect(this); + glActiveTexture(chain->get_input_sampler(self, 0)); check_error(); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); check_error(); diff --git a/fft_pass_effect.h b/fft_pass_effect.h index 501c562..90e88bc 100644 --- a/fft_pass_effect.h +++ b/fft_pass_effect.h @@ -98,10 +98,13 @@ public: *width = *virtual_width = input_width; *height = *virtual_height = input_height; } + + virtual void inform_added(EffectChain *chain) { this->chain = chain; } enum Direction { HORIZONTAL = 0, VERTICAL = 1 }; private: + EffectChain *chain; int input_width, input_height; GLuint tex; int fft_size; diff --git a/resample_effect.cpp b/resample_effect.cpp index b988e5a..50b7c6b 100644 --- a/resample_effect.cpp +++ b/resample_effect.cpp @@ -413,7 +413,8 @@ void SingleResamplePassEffect::set_gl_state(GLuint glsl_program_num, const strin // We specifically do not want mipmaps on the input texture; // they break minification. - glActiveTexture(GL_TEXTURE0); + Node *self = chain->find_node_for_effect(this); + glActiveTexture(chain->get_input_sampler(self, 0)); check_error(); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); check_error(); diff --git a/resample_effect.h b/resample_effect.h index 24e6372..f0112b3 100644 --- a/resample_effect.h +++ b/resample_effect.h @@ -73,6 +73,7 @@ public: virtual bool needs_srgb_primaries() const { return false; } virtual AlphaHandling alpha_handling() const { return INPUT_PREMULTIPLIED_ALPHA_KEEP_BLANK; } + virtual void inform_added(EffectChain *chain) { this->chain = chain; } virtual void inform_input_size(unsigned input_num, unsigned width, unsigned height) { if (parent != NULL) { parent->inform_input_size(input_num, width, height); @@ -93,6 +94,7 @@ private: void update_texture(GLuint glsl_program_num, const std::string &prefix, unsigned *sampler_num); ResampleEffect *parent; + EffectChain *chain; Direction direction; GLuint texnum; int input_width, input_height, output_width, output_height; diff --git a/slice_effect.cpp b/slice_effect.cpp index 3cdd415..3a88465 100644 --- a/slice_effect.cpp +++ b/slice_effect.cpp @@ -1,5 +1,6 @@ #include +#include "effect_chain.h" #include "slice_effect.h" #include "effect_util.h" #include "util.h" @@ -60,12 +61,10 @@ void SliceEffect::set_gl_state(GLuint glsl_program_num, const string &prefix, un set_uniform_float(glsl_program_num, prefix, "slice_offset_to_input_coord", float(output_slice_size) / float(input_height)); } - // Normalized coordinates could potentially cause blurring of the - // image; it's not critical, but we have set changes_output_size() - // and needs_texture_bounce(), so simply turning off the interpolation - // is allowed. - assert(*sampler_num == 1); - glActiveTexture(GL_TEXTURE0); + // Normalized coordinates could potentially cause blurring of the image. + // It isn't critical, but still good practice. + Node *self = chain->find_node_for_effect(this); + glActiveTexture(chain->get_input_sampler(self, 0)); check_error(); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); check_error(); diff --git a/slice_effect.h b/slice_effect.h index 3448941..1c363ea 100644 --- a/slice_effect.h +++ b/slice_effect.h @@ -29,10 +29,12 @@ public: unsigned *virtual_width, unsigned *virtual_height) const; void set_gl_state(GLuint glsl_program_num, const std::string &prefix, unsigned *sampler_num); + virtual void inform_added(EffectChain *chain) { this->chain = chain; } enum Direction { HORIZONTAL = 0, VERTICAL = 1 }; private: + EffectChain *chain; int input_width, input_height; int input_slice_size, output_slice_size; Direction direction;