From b10c546f579c7ccb5939161e61a71cd18a3f9bbd Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Mon, 14 Jan 2013 02:27:35 +0100 Subject: [PATCH] Add the rest of the files for the premultiplied alpha commit. --- .gitignore | 3 + Makefile | 4 + README | 4 +- blur_effect.h | 1 + colorspace_conversion_effect.h | 1 + colorspace_conversion_effect_test.cpp | 12 +- demo.cpp | 67 +++++- dither_effect.h | 5 + effect.h | 43 ++++ effect_chain.cpp | 303 ++++++++++++++++++++++++-- effect_chain.h | 24 +- effect_chain_test.cpp | 116 ++++++++++ flat_input.cpp | 6 +- flat_input.h | 18 ++ flat_input_test.cpp | 6 +- gamma_compression_effect.h | 4 + gamma_expansion_effect.h | 4 + gamma_expansion_effect_test.cpp | 4 +- glow_effect.h | 3 + image_format.h | 10 +- lift_gamma_gain_effect.frag | 2 + lift_gamma_gain_effect.h | 3 + lift_gamma_gain_effect_test.cpp | 8 +- mirror_effect.h | 1 + mix_effect.h | 3 +- mix_effect_test.cpp | 24 +- overlay_effect.frag | 18 +- overlay_effect.h | 5 + overlay_effect_test.cpp | 12 +- saturation_effect.h | 1 + saturation_effect_test.cpp | 6 +- test_util.cpp | 12 +- test_util.h | 6 +- vignette_effect.h | 1 + white_balance_effect.h | 1 + white_balance_effect_test.cpp | 6 +- ycbcr_input.h | 1 + 37 files changed, 668 insertions(+), 80 deletions(-) diff --git a/.gitignore b/.gitignore index dff5e7b..2303e5a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,8 +11,11 @@ demo effect_chain_test gamma_compression_effect_test gamma_expansion_effect_test +alpha_multiplication_effect_test +alpha_division_effect_test colorspace_conversion_effect_test mix_effect_test +overlay_effect_test saturation_effect_test deconvolution_sharpen_effect_test blur_effect_test diff --git a/Makefile b/Makefile index ad886c0..b9b66f4 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,8 @@ TESTS += overlay_effect_test TESTS += gamma_expansion_effect_test TESTS += gamma_compression_effect_test TESTS += colorspace_conversion_effect_test +TESTS += alpha_multiplication_effect_test +TESTS += alpha_division_effect_test TESTS += saturation_effect_test TESTS += deconvolution_sharpen_effect_test TESTS += blur_effect_test @@ -60,6 +62,8 @@ LIB_OBJS += white_balance_effect.o LIB_OBJS += gamma_expansion_effect.o LIB_OBJS += gamma_compression_effect.o LIB_OBJS += colorspace_conversion_effect.o +LIB_OBJS += alpha_multiplication_effect.o +LIB_OBJS += alpha_division_effect.o LIB_OBJS += saturation_effect.o LIB_OBJS += vignette_effect.o LIB_OBJS += mirror_effect.o diff --git a/README b/README index 485fd34..c4d14c7 100644 --- a/README +++ b/README @@ -62,7 +62,7 @@ Assuming you have an OpenGL context already set up: ImageFormat inout_format; inout_format.color_space = COLORSPACE_sRGB; inout_format.gamma_curve = GAMMA_sRGB; - FlatInput *input = knew FlatInput(inout_format, FORMAT_BGRA, GL_UNSIGNED_BYTE, 1280, 720)); + FlatInput *input = knew FlatInput(inout_format, FORMAT_BGRA_POSTMULTIPLIED_ALPHA, GL_UNSIGNED_BYTE, 1280, 720)); chain.add_input(input); Effect *saturation_effect = chain.add_effect(new SaturationEffect()); @@ -72,7 +72,7 @@ Assuming you have an OpenGL context already set up: const float gain[] = { 0.8f, 1.0f, 1.0f }; lift_gamma_gain_effect->set_vec3("gain", &gain); - chain.add_output(inout_format); + chain.add_output(inout_format, OUTPUT_POSTMULTIPLIED_ALPHA); chain.finalize(); for ( ;; ) { diff --git a/blur_effect.h b/blur_effect.h index 7f92074..b680947 100644 --- a/blur_effect.h +++ b/blur_effect.h @@ -24,6 +24,7 @@ public: virtual bool needs_texture_bounce() const { return true; } virtual bool needs_mipmaps() const { return true; } virtual bool needs_srgb_primaries() const { return false; } + virtual AlphaHandling alpha_handling() const { return INPUT_AND_OUTPUT_ALPHA_PREMULTIPLIED; } virtual void inform_input_size(unsigned input_num, unsigned width, unsigned height); diff --git a/colorspace_conversion_effect.h b/colorspace_conversion_effect.h index 33a04ed..93db321 100644 --- a/colorspace_conversion_effect.h +++ b/colorspace_conversion_effect.h @@ -18,6 +18,7 @@ public: std::string output_fragment_shader(); virtual bool needs_srgb_primaries() const { return false; } + virtual AlphaHandling alpha_handling() const { return DONT_CARE_ALPHA_TYPE; } private: Colorspace source_space, destination_space; diff --git a/colorspace_conversion_effect_test.cpp b/colorspace_conversion_effect_test.cpp index 803773d..3361489 100644 --- a/colorspace_conversion_effect_test.cpp +++ b/colorspace_conversion_effect_test.cpp @@ -16,11 +16,11 @@ TEST(ColorspaceConversionEffectTest, Reversible) { float temp_data[4 * 6], out_data[4 * 6]; { - EffectChainTester tester(data, 1, 6, FORMAT_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); + EffectChainTester tester(data, 1, 6, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); tester.run(temp_data, GL_RGBA, COLORSPACE_REC_601_525, GAMMA_LINEAR); } { - EffectChainTester tester(temp_data, 1, 6, FORMAT_RGBA, COLORSPACE_REC_601_525, GAMMA_LINEAR); + EffectChainTester tester(temp_data, 1, 6, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_REC_601_525, GAMMA_LINEAR); tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); } @@ -37,7 +37,7 @@ TEST(ColorspaceConversionEffectTest, sRGB_Primaries) { }; float out_data[4 * 5]; - EffectChainTester tester(data, 1, 5, FORMAT_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); + EffectChainTester tester(data, 1, 5, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); tester.run(out_data, GL_RGBA, COLORSPACE_XYZ, GAMMA_LINEAR); // Black should stay black. @@ -95,7 +95,7 @@ TEST(ColorspaceConversionEffectTest, Rec601_525_Primaries) { }; float out_data[4 * 5]; - EffectChainTester tester(data, 1, 5, FORMAT_RGBA, COLORSPACE_REC_601_525, GAMMA_LINEAR); + EffectChainTester tester(data, 1, 5, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_REC_601_525, GAMMA_LINEAR); tester.run(out_data, GL_RGBA, COLORSPACE_XYZ, GAMMA_LINEAR); // Black should stay black. @@ -145,7 +145,7 @@ TEST(ColorspaceConversionEffectTest, Rec601_625_Primaries) { }; float out_data[4 * 5]; - EffectChainTester tester(data, 1, 5, FORMAT_RGBA, COLORSPACE_REC_601_625, GAMMA_LINEAR); + EffectChainTester tester(data, 1, 5, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_REC_601_625, GAMMA_LINEAR); tester.run(out_data, GL_RGBA, COLORSPACE_XYZ, GAMMA_LINEAR); // Black should stay black. @@ -221,7 +221,7 @@ TEST(ColorspaceConversionEffectTest, sRGBToRec601_525) { }; float out_data[4 * 6]; - EffectChainTester tester(data, 1, 6, FORMAT_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); + EffectChainTester tester(data, 1, 6, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); tester.run(out_data, GL_RGBA, COLORSPACE_REC_601_525, GAMMA_LINEAR); expect_equal(expected_data, out_data, 4, 6); diff --git a/demo.cpp b/demo.cpp index 02c923c..1e32e08 100644 --- a/demo.cpp +++ b/demo.cpp @@ -30,6 +30,9 @@ #include "lift_gamma_gain_effect.h" #include "saturation_effect.h" #include "diffusion_effect.h" +#include "overlay_effect.h" +#include "resample_effect.h" +#include "resize_effect.h" unsigned char result[WIDTH * HEIGHT * 4]; @@ -126,6 +129,15 @@ unsigned char *load_image(const char *filename, unsigned *w, unsigned *h) SDL_FreeSurface(img); + unsigned char *x = (unsigned char *)converted->pixels; + for (int i = 0; i < img->w * img->h; ++i) { + if (x[i * 4 + 3] == 0) { + x[i * 4 + 0] = 255; + x[i * 4 + 1] = 255; + x[i * 4 + 2] = 255; + } + } + return (unsigned char *)converted->pixels; } @@ -167,18 +179,52 @@ int main(int argc, char **argv) ImageFormat inout_format; inout_format.color_space = COLORSPACE_sRGB; - inout_format.gamma_curve = GAMMA_sRGB; + inout_format.gamma_curve = GAMMA_LINEAR; - FlatInput *input = new FlatInput(inout_format, FORMAT_BGRA, GL_UNSIGNED_BYTE, img_w, img_h); + FlatInput *input = new FlatInput(inout_format, FORMAT_BGRA_POSTMULTIPLIED_ALPHA, GL_UNSIGNED_BYTE, img_w, img_h); chain.add_input(input); - Effect *lift_gamma_gain_effect = chain.add_effect(new LiftGammaGainEffect()); - Effect *saturation_effect = chain.add_effect(new SaturationEffect()); - Effect *diffusion_effect = chain.add_effect(new DiffusionEffect()); + + unsigned char *src_overlay1 = load_image("overlay1.png", &img_w, &img_h); +#if 0 + float *src_bleh = new float[img_w * img_h * 4]; + for (int i = 0; i < img_w * img_h; ++i) { + float r = src_overlay1[i * 4 + 0] / 255.0f; + float g = src_overlay1[i * 4 + 1] / 255.0f; + float b = src_overlay1[i * 4 + 2] / 255.0f; + float a = src_overlay1[i * 4 + 3] / 255.0f; + // src_bleh[i * 4 + 0] = r * a; + // src_bleh[i * 4 + 1] = g * a; + // src_bleh[i * 4 + 2] = b * a; + src_bleh[i * 4 + 0] = r; + src_bleh[i * 4 + 1] = g; + src_bleh[i * 4 + 2] = b; + src_bleh[i * 4 + 3] = a; + } + FlatInput *overlay1 = new FlatInput(inout_format, FORMAT_BGRA_POSTMULTIPLIED_ALPHA, GL_FLOAT, img_w, img_h); +#endif + FlatInput *overlay1 = new FlatInput(inout_format, FORMAT_BGRA_POSTMULTIPLIED_ALPHA, GL_UNSIGNED_BYTE, img_w, img_h); + chain.add_input(overlay1); + + unsigned char *src_overlay2 = load_image("overlay2.png", &img_w, &img_h); + FlatInput *overlay2 = new FlatInput(inout_format, FORMAT_BGRA_POSTMULTIPLIED_ALPHA, GL_UNSIGNED_BYTE, img_w, img_h); + chain.add_input(overlay2); + + Effect *mix1 = chain.add_effect(new OverlayEffect(), overlay2, overlay1); + //Effect *mix1_resized = chain.add_effect(new ResampleEffect(), mix1); + Effect *mix1_resized = chain.add_effect(new ResizeEffect(), mix1); + CHECK(mix1_resized->set_int("width", 1280)); + CHECK(mix1_resized->set_int("height", 720)); + Effect *mix2 = chain.add_effect(new OverlayEffect(), input, mix1_resized); +// Effect *mix2 = chain.add_effect(new OverlayEffect(), input, overlay1); + + //Effect *lift_gamma_gain_effect = chain.add_effect(new LiftGammaGainEffect()); + //Effect *saturation_effect = chain.add_effect(new SaturationEffect()); + //Effect *diffusion_effect = chain.add_effect(new DiffusionEffect()); //Effect *vignette_effect = chain.add_effect(new VignetteEffect()); //Effect *sandbox_effect = chain.add_effect(new SandboxEffect()); //sandbox_effect->set_float("parm", 42.0f); //chain.add_effect(new MirrorEffect()); - chain.add_output(inout_format); + chain.add_output(inout_format, OUTPUT_ALPHA_POSTMULTIPLIED); chain.set_dither_bits(8); chain.finalize(); @@ -219,15 +265,18 @@ int main(int argc, char **argv) ++frame; - update_hsv(lift_gamma_gain_effect, saturation_effect); + //update_hsv(lift_gamma_gain_effect, saturation_effect); //vignette_effect->set_float("radius", radius); //vignette_effect->set_float("inner_radius", inner_radius); //vignette_effect->set_vec2("center", (float[]){ 0.7f, 0.5f }); - CHECK(diffusion_effect->set_float("radius", blur_radius)); - CHECK(diffusion_effect->set_float("blurred_mix_amount", blurred_mix_amount)); + //CHECK(diffusion_effect->set_float("radius", blur_radius)); + //CHECK(diffusion_effect->set_float("blurred_mix_amount", blurred_mix_amount)); input->set_pixel_data(src_img); + overlay1->set_pixel_data(src_overlay1); + //overlay1->set_pixel_data(src_bleh); + overlay2->set_pixel_data(src_overlay2); chain.render_to_screen(); glBindBuffer(GL_PIXEL_PACK_BUFFER_ARB, pbo); diff --git a/dither_effect.h b/dither_effect.h index 943220a..535b783 100644 --- a/dither_effect.h +++ b/dither_effect.h @@ -52,6 +52,11 @@ public: virtual std::string effect_type_id() const { return "DitherEffect"; } std::string output_fragment_shader(); + // Note that if we did error diffusion, we'd actually want to diffuse the + // premultiplied error. However, we need to do dithering in the same + // space as quantization, whether that be pre- or postmultiply. + virtual AlphaHandling alpha_handling() const { return DONT_CARE_ALPHA_TYPE; } + void set_gl_state(GLuint glsl_program_num, const std::string &prefix, unsigned *sampler_num); private: diff --git a/effect.h b/effect.h index 235558f..39c6126 100644 --- a/effect.h +++ b/effect.h @@ -78,6 +78,49 @@ public: // in a linear fashion. virtual bool needs_srgb_primaries() const { return true; } + // How this effect handles alpha, ie. what it outputs in its + // alpha channel. The choices are basically blank (alpha is always 1.0), + // premultiplied and postmultiplied. + // + // Premultiplied alpha is when the alpha value has been be multiplied + // into the three color components, so e.g. 100% red at 50% alpha + // would be (0.5, 0.0, 0.0, 0.5) instead of (1.0, 0.0, 0.0, 0.5) + // as it is stored in most image formats (postmultiplied alpha). + // The multiplication is taken to have happened in linear light. + // This is the most natural format for processing, and the default in + // most of Movit (just like linear light is). + // + // If you set INPUT_AND_OUTPUT_ALPHA_PREMULTIPLIED, all of your inputs + // (if any) are guaranteed to also be in premultiplied alpha. + // Otherwise, you can get postmultiplied or premultiplied alpha; + // you won't know. If you have multiple inputs, you will get the same + // (pre- or postmultiplied) for all inputs, although most likely, + // you will want to combine them in a premultiplied fashion anyway + // in that case. + enum AlphaHandling { + // Always outputs blank alpha (ie. alpha=1.0). Only appropriate + // for inputs that do not output an alpha channel. + // Blank alpha is special in that it can be treated as both + // pre- and postmultiplied. + OUTPUT_BLANK_ALPHA, + + // Always outputs premultiplied alpha. As noted above, + // you will then also get all inputs in premultiplied alpha. + // If you set this, you should also set needs_linear_light(). + INPUT_AND_OUTPUT_ALPHA_PREMULTIPLIED, + + // Always outputs postmultiplied alpha. Only appropriate for inputs. + OUTPUT_ALPHA_POSTMULTIPLIED, + + // Keeps the type of alpha unchanged from input to output. + // Usually appropriate if you process all color channels + // in a linear fashion, and do not change alpha. + // + // Does not make sense for inputs. + DONT_CARE_ALPHA_TYPE, + }; + virtual AlphaHandling alpha_handling() const { return INPUT_AND_OUTPUT_ALPHA_PREMULTIPLIED; } + // Whether this effect expects its input to come directly from // a texture. If this is true, the framework will not chain the // input from other effects, but will store the results of the diff --git a/effect_chain.cpp b/effect_chain.cpp index 1832b9e..8db5376 100644 --- a/effect_chain.cpp +++ b/effect_chain.cpp @@ -16,6 +16,8 @@ #include "gamma_expansion_effect.h" #include "gamma_compression_effect.h" #include "colorspace_conversion_effect.h" +#include "alpha_multiplication_effect.h" +#include "alpha_division_effect.h" #include "dither_effect.h" #include "input.h" #include "init.h" @@ -55,9 +57,10 @@ Input *EffectChain::add_input(Input *input) return input; } -void EffectChain::add_output(const ImageFormat &format) +void EffectChain::add_output(const ImageFormat &format, OutputAlphaFormat alpha_format) { output_format = format; + output_alpha_format = alpha_format; } Node *EffectChain::add_node(Effect *effect) @@ -71,6 +74,7 @@ Node *EffectChain::add_node(Effect *effect) node->effect_id = effect_id; node->output_color_space = COLORSPACE_INVALID; node->output_gamma_curve = GAMMA_INVALID; + node->output_alpha_type = ALPHA_INVALID; node->output_texture = 0; nodes.push_back(node); @@ -494,6 +498,20 @@ void EffectChain::output_dot(const char *filename) break; } + switch (nodes[i]->output_alpha_type) { + case ALPHA_INVALID: + labels.push_back("alpha[invalid]"); + break; + case ALPHA_BLANK: + labels.push_back("alpha[blank]"); + break; + case ALPHA_POSTMULTIPLIED: + labels.push_back("alpha[postmult]"); + break; + default: + break; + } + if (labels.empty()) { fprintf(fp, " n%ld -> n%ld;\n", (long)nodes[i], (long)nodes[i]->outgoing_links[j]); } else { @@ -656,6 +674,22 @@ void EffectChain::find_color_spaces_for_inputs() Input *input = static_cast(node->effect); node->output_color_space = input->get_color_space(); node->output_gamma_curve = input->get_gamma_curve(); + + Effect::AlphaHandling alpha_handling = input->alpha_handling(); + switch (alpha_handling) { + case Effect::OUTPUT_BLANK_ALPHA: + node->output_alpha_type = ALPHA_BLANK; + break; + case Effect::INPUT_AND_OUTPUT_ALPHA_PREMULTIPLIED: + node->output_alpha_type = ALPHA_PREMULTIPLIED; + break; + case Effect::OUTPUT_ALPHA_POSTMULTIPLIED: + node->output_alpha_type = ALPHA_POSTMULTIPLIED; + break; + case Effect::DONT_CARE_ALPHA_TYPE: + default: + assert(false); + } } } } @@ -703,6 +737,129 @@ void EffectChain::propagate_gamma_and_color_space() } } +// Propagate alpha information as far as we can in the graph. +// Similar to propagate_gamma_and_color_space(). +void EffectChain::propagate_alpha() +{ + // We depend on going through the nodes in order. + sort_nodes_topologically(); + + for (unsigned i = 0; i < nodes.size(); ++i) { + Node *node = nodes[i]; + if (node->disabled) { + continue; + } + assert(node->incoming_links.size() == node->effect->num_inputs()); + if (node->incoming_links.size() == 0) { + assert(node->output_alpha_type != ALPHA_INVALID); + continue; + } + + // The alpha multiplication/division effects are special cases. + if (node->effect->effect_type_id() == "AlphaMultiplicationEffect") { + assert(node->incoming_links.size() == 1); + assert(node->incoming_links[0]->output_alpha_type == ALPHA_POSTMULTIPLIED); + node->output_alpha_type = ALPHA_PREMULTIPLIED; + continue; + } + if (node->effect->effect_type_id() == "AlphaDivisionEffect") { + assert(node->incoming_links.size() == 1); + assert(node->incoming_links[0]->output_alpha_type == ALPHA_PREMULTIPLIED); + node->output_alpha_type = ALPHA_POSTMULTIPLIED; + continue; + } + + // GammaCompressionEffect and GammaExpansionEffect are also a special case, + // because they are the only one that _need_ postmultiplied alpha. + if (node->effect->effect_type_id() == "GammaCompressionEffect" || + node->effect->effect_type_id() == "GammaExpansionEffect") { + assert(node->incoming_links.size() == 1); + if (node->incoming_links[0]->output_alpha_type == ALPHA_BLANK) { + node->output_alpha_type = ALPHA_BLANK; + } else if (node->incoming_links[0]->output_alpha_type == ALPHA_POSTMULTIPLIED) { + node->output_alpha_type = ALPHA_POSTMULTIPLIED; + } else { + node->output_alpha_type = ALPHA_INVALID; + } + continue; + } + + // Only inputs can have unconditional alpha output (OUTPUT_BLANK_ALPHA + // or OUTPUT_ALPHA_POSTMULTIPLIED), and they have already been + // taken care of above. Rationale: Even if you could imagine + // e.g. an effect that took in an image and set alpha=1.0 + // unconditionally, it wouldn't make any sense to have it as + // e.g. OUTPUT_BLANK_ALPHA, since it wouldn't know whether it + // got its input pre- or postmultiplied, so it wouldn't know + // whether to divide away the old alpha or not. + Effect::AlphaHandling alpha_handling = node->effect->alpha_handling(); + assert(alpha_handling == Effect::INPUT_AND_OUTPUT_ALPHA_PREMULTIPLIED || + alpha_handling == Effect::DONT_CARE_ALPHA_TYPE); + + // If the node has multiple inputs, check that they are all valid and + // the same. + bool any_invalid = false; + bool any_premultiplied = false; + bool any_postmultiplied = false; + + for (unsigned j = 0; j < node->incoming_links.size(); ++j) { + switch (node->incoming_links[j]->output_alpha_type) { + case ALPHA_INVALID: + any_invalid = true; + break; + case ALPHA_BLANK: + // Blank is good as both pre- and postmultiplied alpha, + // so just ignore it. + break; + case ALPHA_PREMULTIPLIED: + any_premultiplied = true; + break; + case ALPHA_POSTMULTIPLIED: + any_postmultiplied = true; + break; + default: + assert(false); + } + } + + if (any_invalid) { + node->output_alpha_type = ALPHA_INVALID; + continue; + } + + // Inputs must be of the same type. + if (any_premultiplied && any_postmultiplied) { + node->output_alpha_type = ALPHA_INVALID; + continue; + } + + if (alpha_handling == Effect::INPUT_AND_OUTPUT_ALPHA_PREMULTIPLIED) { + // If the effect has asked for premultiplied alpha, check that it has got it. + if (any_postmultiplied) { + node->output_alpha_type = ALPHA_INVALID; + } else { + // In some rare cases, it might be advantageous to say + // that blank input alpha yields blank output alpha. + // However, this would cause a more complex Effect interface + // an effect would need to guarantee that it doesn't mess with + // blank alpha), so this is the simplest. + node->output_alpha_type = ALPHA_PREMULTIPLIED; + } + } else { + // OK, all inputs are the same, and this effect is not going + // to change it. + assert(alpha_handling == Effect::DONT_CARE_ALPHA_TYPE); + if (any_premultiplied) { + node->output_alpha_type = ALPHA_PREMULTIPLIED; + } else if (any_postmultiplied) { + node->output_alpha_type = ALPHA_POSTMULTIPLIED; + } else { + node->output_alpha_type = ALPHA_BLANK; + } + } + } +} + bool EffectChain::node_needs_colorspace_fix(Node *node) { if (node->disabled) { @@ -762,7 +919,7 @@ void EffectChain::fix_internal_color_spaces() } char filename[256]; - sprintf(filename, "step3-colorspacefix-iter%u.dot", ++colorspace_propagation_pass); + sprintf(filename, "step5-colorspacefix-iter%u.dot", ++colorspace_propagation_pass); output_dot(filename); assert(colorspace_propagation_pass < 100); } while (found_any); @@ -776,6 +933,87 @@ void EffectChain::fix_internal_color_spaces() } } +bool EffectChain::node_needs_alpha_fix(Node *node) +{ + if (node->disabled) { + return false; + } + + // propagate_alpha() has already set our output to ALPHA_INVALID if the + // inputs differ or we are otherwise in mismatch, so we can rely on that. + return (node->output_alpha_type == ALPHA_INVALID); +} + +// Fix up alpha so that there are no ALPHA_INVALID nodes left in +// the graph. Similar to fix_internal_color_spaces(). +void EffectChain::fix_internal_alpha(unsigned step) +{ + unsigned alpha_propagation_pass = 0; + bool found_any; + do { + found_any = false; + for (unsigned i = 0; i < nodes.size(); ++i) { + Node *node = nodes[i]; + if (!node_needs_alpha_fix(node)) { + continue; + } + + // If we need to fix up GammaExpansionEffect, then clearly something + // is wrong, since the combination of premultiplied alpha and nonlinear inputs + // is meaningless. + assert(node->effect->effect_type_id() != "GammaExpansionEffect"); + + AlphaType desired_type = ALPHA_PREMULTIPLIED; + + // GammaCompressionEffect is special; it needs postmultiplied alpha. + if (node->effect->effect_type_id() == "GammaCompressionEffect") { + assert(node->incoming_links.size() == 1); + assert(node->incoming_links[0]->output_alpha_type == ALPHA_PREMULTIPLIED); + desired_type = ALPHA_POSTMULTIPLIED; + } + + // Go through each input that is not premultiplied alpha, and insert + // a conversion before it. + for (unsigned j = 0; j < node->incoming_links.size(); ++j) { + Node *input = node->incoming_links[j]; + assert(input->output_alpha_type != ALPHA_INVALID); + if (input->output_alpha_type == desired_type || + input->output_alpha_type == ALPHA_BLANK) { + continue; + } + Node *conversion; + if (desired_type == ALPHA_PREMULTIPLIED) { + conversion = add_node(new AlphaMultiplicationEffect()); + } else { + conversion = add_node(new AlphaDivisionEffect()); + } + conversion->output_alpha_type = desired_type; + insert_node_between(input, conversion, node); + } + + // Re-sort topologically, and propagate the new information. + propagate_gamma_and_color_space(); + propagate_alpha(); + + found_any = true; + break; + } + + char filename[256]; + sprintf(filename, "step%u-alphafix-iter%u.dot", step, ++alpha_propagation_pass); + output_dot(filename); + assert(alpha_propagation_pass < 100); + } while (found_any); + + for (unsigned i = 0; i < nodes.size(); ++i) { + Node *node = nodes[i]; + if (node->disabled) { + continue; + } + assert(node->output_alpha_type != ALPHA_INVALID); + } +} + // Make so that the output is in the desired color space. void EffectChain::fix_output_color_space() { @@ -786,6 +1024,32 @@ void EffectChain::fix_output_color_space() CHECK(conversion->effect->set_int("destination_space", output_format.color_space)); conversion->output_color_space = output_format.color_space; connect_nodes(output, conversion); + propagate_alpha(); + propagate_gamma_and_color_space(); + } +} + +// Make so that the output is in the desired pre-/postmultiplication alpha state. +void EffectChain::fix_output_alpha() +{ + Node *output = find_output_node(); + assert(output->output_alpha_type != ALPHA_INVALID); + if (output->output_alpha_type == ALPHA_BLANK) { + // No alpha output, so we don't care. + return; + } + if (output->output_alpha_type == ALPHA_PREMULTIPLIED && + output_alpha_format == OUTPUT_ALPHA_POSTMULTIPLIED) { + Node *conversion = add_node(new AlphaDivisionEffect()); + connect_nodes(output, conversion); + propagate_alpha(); + propagate_gamma_and_color_space(); + } + if (output->output_alpha_type == ALPHA_POSTMULTIPLIED && + output_alpha_format == OUTPUT_ALPHA_PREMULTIPLIED) { + Node *conversion = add_node(new AlphaMultiplicationEffect()); + connect_nodes(output, conversion); + propagate_alpha(); propagate_gamma_and_color_space(); } } @@ -921,6 +1185,7 @@ void EffectChain::fix_internal_gamma_by_inserting_nodes(unsigned step) } // Re-sort topologically, and propagate the new information. + propagate_alpha(); propagate_gamma_and_color_space(); found_any = true; @@ -1007,34 +1272,46 @@ void EffectChain::finalize() find_color_spaces_for_inputs(); output_dot("step2-input-colorspace.dot"); + propagate_alpha(); + output_dot("step3-propagated-alpha.dot"); + propagate_gamma_and_color_space(); - output_dot("step3-propagated.dot"); + output_dot("step4-propagated-all.dot"); fix_internal_color_spaces(); + fix_internal_alpha(6); fix_output_color_space(); - output_dot("step4-output-colorspacefix.dot"); + output_dot("step7-output-colorspacefix.dot"); + fix_output_alpha(); + output_dot("step8-output-alphafix.dot"); // Note that we need to fix gamma after colorspace conversion, // because colorspace conversions might create needs for gamma conversions. // Also, we need to run an extra pass of fix_internal_gamma() after - // fixing the output gamma, as we only have conversions to/from linear. - fix_internal_gamma_by_asking_inputs(5); - fix_internal_gamma_by_inserting_nodes(6); + // fixing the output gamma, as we only have conversions to/from linear, + // and fix_internal_alpha() since GammaCompressionEffect needs + // postmultiplied input. + fix_internal_gamma_by_asking_inputs(9); + fix_internal_gamma_by_inserting_nodes(10); fix_output_gamma(); - output_dot("step7-output-gammafix.dot"); - fix_internal_gamma_by_asking_inputs(8); - fix_internal_gamma_by_inserting_nodes(9); + output_dot("step11-output-gammafix.dot"); + propagate_alpha(); + output_dot("step12-output-alpha-propagated.dot"); + fix_internal_alpha(13); + output_dot("step14-output-alpha-fixed.dot"); + fix_internal_gamma_by_asking_inputs(15); + fix_internal_gamma_by_inserting_nodes(16); - output_dot("step10-before-dither.dot"); + output_dot("step17-before-dither.dot"); add_dither_if_needed(); - output_dot("step11-final.dot"); + output_dot("step18-final.dot"); // Construct all needed GLSL programs, starting at the output. construct_glsl_programs(find_output_node()); - output_dot("step12-split-to-phases.dot"); + output_dot("step19-split-to-phases.dot"); // If we have more than one phase, we need intermediate render-to-texture. // Construct an FBO, and then as many textures as we need. diff --git a/effect_chain.h b/effect_chain.h index 6ceb0c3..666df9e 100644 --- a/effect_chain.h +++ b/effect_chain.h @@ -11,6 +11,21 @@ class EffectChain; class Phase; +// For internal use within Node. +enum AlphaType { + ALPHA_INVALID = -1, + ALPHA_BLANK, + ALPHA_PREMULTIPLIED, + ALPHA_POSTMULTIPLIED, +}; + +// Whether you want pre- or postmultiplied alpha in the output +// (see effect.h for a discussion of pre- versus postmultiplied alpha). +enum OutputAlphaFormat { + OUTPUT_ALPHA_PREMULTIPLIED, + OUTPUT_ALPHA_POSTMULTIPLIED, +}; + // A node in the graph; basically an effect and some associated information. class Node { public: @@ -42,6 +57,7 @@ private: // Used during the building of the effect chain. Colorspace output_color_space; GammaCurve output_gamma_curve; + AlphaType output_alpha_type; friend class EffectChain; }; @@ -89,7 +105,7 @@ public: } Effect *add_effect(Effect *effect, const std::vector &inputs); - void add_output(const ImageFormat &format); + void add_output(const ImageFormat &format, OutputAlphaFormat alpha_format); // Set number of output bits, to scale the dither. // 8 is the right value for most outputs. @@ -168,6 +184,7 @@ private: // Used during finalize(). void find_color_spaces_for_inputs(); + void propagate_alpha(); void propagate_gamma_and_color_space(); Node *find_output_node(); @@ -175,6 +192,10 @@ private: void fix_internal_color_spaces(); void fix_output_color_space(); + bool node_needs_alpha_fix(Node *node); + void fix_internal_alpha(unsigned step); + void fix_output_alpha(); + bool node_needs_gamma_fix(Node *node); void fix_internal_gamma_by_asking_inputs(unsigned step); void fix_internal_gamma_by_inserting_nodes(unsigned step); @@ -183,6 +204,7 @@ private: float aspect_nom, aspect_denom; ImageFormat output_format; + OutputAlphaFormat output_alpha_format; std::vector nodes; std::map node_map; diff --git a/effect_chain_test.cpp b/effect_chain_test.cpp index ace160f..356d628 100644 --- a/effect_chain_test.cpp +++ b/effect_chain_test.cpp @@ -89,6 +89,10 @@ public: InvertEffect() {} virtual std::string effect_type_id() const { return "InvertEffect"; } std::string output_fragment_shader() { return read_file("invert_effect.frag"); } + + // A real invert would actually care about its alpha, + // but in this unit test, it only complicates things. + virtual AlphaHandling alpha_handling() const { return DONT_CARE_ALPHA_TYPE; } }; // Like IdentityEffect, but rewrites itself out of the loop, @@ -351,6 +355,118 @@ TEST(EffectChainTest, IdentityThroughRec709) { expect_equal(data, out_data, 256, 1); } +// The identity effect needs premultiplied alpha, and thus will get conversions on both sides. +TEST(EffectChainTest, IdentityThroughAlphaConversions) { + const int size = 3; + float data[4 * size] = { + 0.8f, 0.0f, 0.0f, 0.5f, + 0.0f, 0.2f, 0.2f, 0.3f, + 0.1f, 0.0f, 1.0f, 1.0f, + }; + float out_data[6]; + EffectChainTester tester(data, size, 1, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); + tester.get_chain()->add_effect(new IdentityEffect()); + tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); + + expect_equal(data, out_data, 4, size); +} + +TEST(EffectChainTest, NoAlphaConversionsWhenPremultipliedAlphaNotNeeded) { + const int size = 3; + float data[4 * size] = { + 0.8f, 0.0f, 0.0f, 0.5f, + 0.0f, 0.2f, 0.2f, 0.3f, + 0.1f, 0.0f, 1.0f, 1.0f, + }; + float expected_data[4 * size] = { + 0.1f, 0.0f, 1.0f, 1.0f, + 0.0f, 0.2f, 0.2f, 0.3f, + 0.8f, 0.0f, 0.0f, 0.5f, + }; + float out_data[4 * size]; + EffectChainTester tester(data, size, 1, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); + RewritingToMirrorEffect *effect = new RewritingToMirrorEffect(); + tester.get_chain()->add_effect(effect); + tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); + + Node *node = effect->mirror_node; + ASSERT_EQ(1, node->incoming_links.size()); + EXPECT_EQ(0, node->outgoing_links.size()); + EXPECT_EQ("FlatInput", node->incoming_links[0]->effect->effect_type_id()); + + expect_equal(expected_data, out_data, 4, size); +} + +// An input that outputs only blue, which has blank alpha. +class BlueInput : public Input { +public: + BlueInput() { register_int("needs_mipmaps", &needs_mipmaps); } + virtual std::string effect_type_id() const { return "IdentityEffect"; } + std::string output_fragment_shader() { return read_file("blue.frag"); } + virtual AlphaHandling alpha_handling() const { return OUTPUT_BLANK_ALPHA; } + virtual void finalize() {} + virtual bool can_output_linear_gamma() const { return true; } + virtual unsigned get_width() const { return 1; } + virtual unsigned get_height() const { return 1; } + virtual Colorspace get_color_space() const { return COLORSPACE_sRGB; } + virtual GammaCurve get_gamma_curve() const { return GAMMA_LINEAR; } + +private: + int needs_mipmaps; +}; + +// Like RewritingToInvertEffect, but splicing in a BlueInput instead, +// which outputs blank alpha. +class RewritingToBlueInput : public Input { +public: + RewritingToBlueInput() { register_int("needs_mipmaps", &needs_mipmaps); } + virtual std::string effect_type_id() const { return "RewritingToBlueInput"; } + std::string output_fragment_shader() { EXPECT_TRUE(false); return read_file("identity.frag"); } + virtual void rewrite_graph(EffectChain *graph, Node *self) { + Node *blue_node = graph->add_node(new BlueInput()); + graph->replace_receiver(self, blue_node); + graph->replace_sender(self, blue_node); + + self->disabled = true; + this->blue_node = blue_node; + } + + // Dummy values that we need to implement because we inherit from Input. + // Same as BlueInput. + virtual AlphaHandling alpha_handling() const { return OUTPUT_BLANK_ALPHA; } + virtual void finalize() {} + virtual bool can_output_linear_gamma() const { return true; } + virtual unsigned get_width() const { return 1; } + virtual unsigned get_height() const { return 1; } + virtual Colorspace get_color_space() const { return COLORSPACE_sRGB; } + virtual GammaCurve get_gamma_curve() const { return GAMMA_LINEAR; } + + Node *blue_node; + +private: + int needs_mipmaps; +}; + +TEST(EffectChainTest, NoAlphaConversionsWithBlankAlpha) { + const int size = 3; + float data[4 * size] = { + 0.0f, 0.0f, 1.0f, 1.0f, + 0.0f, 0.0f, 1.0f, 1.0f, + 0.0f, 0.0f, 1.0f, 1.0f, + }; + float out_data[4 * size]; + EffectChainTester tester(NULL, size, 1); + RewritingToBlueInput *input = new RewritingToBlueInput(); + tester.get_chain()->add_input(input); + tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR, OUTPUT_ALPHA_PREMULTIPLIED); + + Node *node = input->blue_node; + EXPECT_EQ(0, node->incoming_links.size()); + EXPECT_EQ(0, node->outgoing_links.size()); + + expect_equal(data, out_data, 4, size); +} + // Effectively scales down its input linearly by 4x (and repeating it), // which is not attainable without mipmaps. class MipmapNeedingEffect : public Effect { diff --git a/flat_input.cpp b/flat_input.cpp index 6e118bc..a9d01a7 100644 --- a/flat_input.cpp +++ b/flat_input.cpp @@ -52,13 +52,15 @@ void FlatInput::finalize() if (pixel_format == FORMAT_RGB) { format = GL_RGB; bytes_per_pixel = 3; - } else if (pixel_format == FORMAT_RGBA) { + } else if (pixel_format == FORMAT_RGBA_PREMULTIPLIED_ALPHA || + pixel_format == FORMAT_RGBA_POSTMULTIPLIED_ALPHA) { format = GL_RGBA; bytes_per_pixel = 4; } else if (pixel_format == FORMAT_BGR) { format = GL_BGR; bytes_per_pixel = 3; - } else if (pixel_format == FORMAT_BGRA) { + } else if (pixel_format == FORMAT_BGRA_PREMULTIPLIED_ALPHA || + pixel_format == FORMAT_BGRA_POSTMULTIPLIED_ALPHA) { format = GL_BGRA; bytes_per_pixel = 4; } else if (pixel_format == FORMAT_GRAYSCALE) { diff --git a/flat_input.h b/flat_input.h index 8b5a3ce..87a0d5e 100644 --- a/flat_input.h +++ b/flat_input.h @@ -1,6 +1,8 @@ #ifndef _FLAT_INPUT_H #define _FLAT_INPUT_H 1 +#include + #include "input.h" #include "init.h" @@ -24,6 +26,22 @@ public: (image_format.gamma_curve == GAMMA_LINEAR || image_format.gamma_curve == GAMMA_sRGB)); } + virtual AlphaHandling alpha_handling() const { + switch (pixel_format) { + case FORMAT_RGBA_PREMULTIPLIED_ALPHA: + case FORMAT_BGRA_PREMULTIPLIED_ALPHA: + return INPUT_AND_OUTPUT_ALPHA_PREMULTIPLIED; + case FORMAT_RGBA_POSTMULTIPLIED_ALPHA: + case FORMAT_BGRA_POSTMULTIPLIED_ALPHA: + return OUTPUT_ALPHA_POSTMULTIPLIED; + case FORMAT_RGB: + case FORMAT_BGR: + case FORMAT_GRAYSCALE: + return OUTPUT_BLANK_ALPHA; + default: + assert(false); + } + } std::string output_fragment_shader(); diff --git a/flat_input_test.cpp b/flat_input_test.cpp index b717764..db03bd0 100644 --- a/flat_input_test.cpp +++ b/flat_input_test.cpp @@ -71,7 +71,7 @@ TEST(FlatInput, RGBA) { }; float out_data[4 * size]; - EffectChainTester tester(data, 1, size, FORMAT_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); + EffectChainTester tester(data, 1, size, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); expect_equal(expected_data, out_data, 4, size); @@ -102,7 +102,7 @@ TEST(FlatInput, AlphaIsNotModifiedBySRGBConversion) { float out_data[4 * size]; EffectChainTester tester(NULL, 1, size); - tester.add_input(data, FORMAT_RGBA, COLORSPACE_sRGB, GAMMA_sRGB); + tester.add_input(data, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_sRGB); tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); expect_equal(expected_data, out_data, 4, size); @@ -152,7 +152,7 @@ TEST(FlatInput, BGRA) { }; float out_data[4 * size]; - EffectChainTester tester(data, 1, size, FORMAT_BGRA, COLORSPACE_sRGB, GAMMA_LINEAR); + EffectChainTester tester(data, 1, size, FORMAT_BGRA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); expect_equal(expected_data, out_data, 4, size); diff --git a/gamma_compression_effect.h b/gamma_compression_effect.h index 03cbb24..0155ee5 100644 --- a/gamma_compression_effect.h +++ b/gamma_compression_effect.h @@ -20,6 +20,10 @@ public: virtual bool needs_srgb_primaries() const { return false; } + // Actually needs postmultiplied input as well as outputting it. + // EffectChain will take care of that. + virtual AlphaHandling alpha_handling() const { return OUTPUT_ALPHA_POSTMULTIPLIED; } + private: GammaCurve destination_curve; float compression_curve[COMPRESSION_CURVE_SIZE]; diff --git a/gamma_expansion_effect.h b/gamma_expansion_effect.h index b93848a..ce99990 100644 --- a/gamma_expansion_effect.h +++ b/gamma_expansion_effect.h @@ -21,6 +21,10 @@ public: virtual bool needs_linear_light() const { return false; } virtual bool needs_srgb_primaries() const { return false; } + // Actually processes its input in a nonlinear fashion, + // but does not touch alpha, and we are a special case anyway. + virtual AlphaHandling alpha_handling() const { return DONT_CARE_ALPHA_TYPE; } + private: GammaCurve source_curve; float expansion_curve[EXPANSION_CURVE_SIZE]; diff --git a/gamma_expansion_effect_test.cpp b/gamma_expansion_effect_test.cpp index 71544b4..94f5749 100644 --- a/gamma_expansion_effect_test.cpp +++ b/gamma_expansion_effect_test.cpp @@ -43,7 +43,7 @@ TEST(GammaExpansionEffectTest, sRGB_AlphaIsUnchanged) { 0.0f, 0.0f, 0.0f, 1.0f, }; float out_data[5 * 4]; - EffectChainTester tester(data, 5, 1, FORMAT_RGBA, COLORSPACE_sRGB, GAMMA_sRGB); + EffectChainTester tester(data, 5, 1, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_sRGB); tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); expect_equal(data, out_data, 5, 1); @@ -88,7 +88,7 @@ TEST(GammaExpansionEffectTest, Rec709_AlphaIsUnchanged) { 0.0f, 0.0f, 0.0f, 1.0f, }; float out_data[5 * 4]; - EffectChainTester tester(data, 5, 1, FORMAT_RGBA, COLORSPACE_sRGB, GAMMA_REC_709); + EffectChainTester tester(data, 5, 1, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_REC_709); tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); expect_equal(data, out_data, 5, 1); diff --git a/glow_effect.h b/glow_effect.h index 8b21e52..c0ac9ce 100644 --- a/glow_effect.h +++ b/glow_effect.h @@ -3,6 +3,9 @@ // Glow: Cut out the highlights of the image (everything above a certain threshold), // blur them, and overlay them onto the original image. +// +// FIXME: This might be broken after MixEffect started working in premultiplied alpha. +// We need to think about how this is going to work, and then add a test. #include "effect.h" diff --git a/image_format.h b/image_format.h index 852090e..90f6034 100644 --- a/image_format.h +++ b/image_format.h @@ -1,7 +1,15 @@ #ifndef _IMAGE_FORMAT_H #define _IMAGE_FORMAT_H 1 -enum MovitPixelFormat { FORMAT_RGB, FORMAT_RGBA, FORMAT_BGR, FORMAT_BGRA, FORMAT_GRAYSCALE }; +enum MovitPixelFormat { + FORMAT_RGB, + FORMAT_RGBA_PREMULTIPLIED_ALPHA, + FORMAT_RGBA_POSTMULTIPLIED_ALPHA, + FORMAT_BGR, + FORMAT_BGRA_PREMULTIPLIED_ALPHA, + FORMAT_BGRA_POSTMULTIPLIED_ALPHA, + FORMAT_GRAYSCALE +}; enum Colorspace { COLORSPACE_INVALID = -1, // For internal use. diff --git a/lift_gamma_gain_effect.frag b/lift_gamma_gain_effect.frag index acd9d51..775941e 100644 --- a/lift_gamma_gain_effect.frag +++ b/lift_gamma_gain_effect.frag @@ -5,10 +5,12 @@ uniform vec3 PREFIX(inv_gamma_22); // 2.2 / gamma. vec4 FUNCNAME(vec2 tc) { vec4 x = INPUT(tc); + x.rgb /= x.aaa; x.rgb = pow(x.rgb, vec3(1.0/2.2)); x.rgb += PREFIX(lift) * (vec3(1) - x.rgb); x.rgb = pow(x.rgb, PREFIX(inv_gamma_22)); x.rgb *= PREFIX(gain_pow_inv_gamma); + x.rgb *= x.aaa; return x; } diff --git a/lift_gamma_gain_effect.h b/lift_gamma_gain_effect.h index 1319570..36f6a77 100644 --- a/lift_gamma_gain_effect.h +++ b/lift_gamma_gain_effect.h @@ -16,6 +16,9 @@ // rest of the curve relatively little. Thus, we actually convert to gamma 2.2 // before lift, and then back again afterwards. (Gain and gamma are, // up to constants, commutative with the de-gamma operation.) +// +// Also, gamma is a case where we would not want premultiplied alpha. +// Thus, we have to divide away alpha first, and then re-multiply it back later. #include "effect.h" diff --git a/lift_gamma_gain_effect_test.cpp b/lift_gamma_gain_effect_test.cpp index 00c96ca..3e63679 100644 --- a/lift_gamma_gain_effect_test.cpp +++ b/lift_gamma_gain_effect_test.cpp @@ -14,7 +14,7 @@ TEST(LiftGammaGainEffectTest, DefaultIsNoop) { }; float out_data[5 * 4]; - EffectChainTester tester(data, 1, 5, FORMAT_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); + EffectChainTester tester(data, 1, 5, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); tester.get_chain()->add_effect(new LiftGammaGainEffect()); tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); @@ -39,7 +39,7 @@ TEST(LiftGammaGainEffectTest, Gain) { }; float out_data[5 * 4]; - EffectChainTester tester(data, 1, 5, FORMAT_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); + EffectChainTester tester(data, 1, 5, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); Effect *lgg_effect = tester.get_chain()->add_effect(new LiftGammaGainEffect()); ASSERT_TRUE(lgg_effect->set_vec3("gain", gain)); tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); @@ -65,7 +65,7 @@ TEST(LiftGammaGainEffectTest, LiftIsDoneInApproximatelysRGB) { }; float out_data[5 * 4]; - EffectChainTester tester(data, 1, 5, FORMAT_RGBA, COLORSPACE_sRGB, GAMMA_sRGB); + EffectChainTester tester(data, 1, 5, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_sRGB); Effect *lgg_effect = tester.get_chain()->add_effect(new LiftGammaGainEffect()); ASSERT_TRUE(lgg_effect->set_vec3("lift", lift)); tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_sRGB); @@ -85,7 +85,7 @@ TEST(LiftGammaGainEffectTest, Gamma22IsApproximatelysRGB) { float gamma[3] = { 2.2f, 2.2f, 2.2f }; float out_data[5 * 4]; - EffectChainTester tester(data, 1, 5, FORMAT_RGBA, COLORSPACE_sRGB, GAMMA_sRGB); + EffectChainTester tester(data, 1, 5, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_sRGB); Effect *lgg_effect = tester.get_chain()->add_effect(new LiftGammaGainEffect()); ASSERT_TRUE(lgg_effect->set_vec3("gamma", gamma)); tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); diff --git a/mirror_effect.h b/mirror_effect.h index 0e8dd7b..03f3409 100644 --- a/mirror_effect.h +++ b/mirror_effect.h @@ -13,6 +13,7 @@ public: virtual bool needs_linear_light() const { return false; } virtual bool needs_srgb_primaries() const { return false; } + virtual AlphaHandling alpha_handling() const { return DONT_CARE_ALPHA_TYPE; } }; #endif // !defined(_MIRROR_EFFECT_H) diff --git a/mix_effect.h b/mix_effect.h index 8032062..c673965 100644 --- a/mix_effect.h +++ b/mix_effect.h @@ -1,7 +1,8 @@ #ifndef _MIX_EFFECT_H #define _MIX_EFFECT_H 1 -// Combine two images: a*x + b*y. (If you set a within [0,1] and b=1-a, you will get a fade.) +// Combine two images: a*x + b*y. If you set a within [0,1] and b=1-a, +// you will get a fade; if not, you may get surprising results (consider alpha). #include "effect.h" diff --git a/mix_effect_test.cpp b/mix_effect_test.cpp index 910182d..d49efd7 100644 --- a/mix_effect_test.cpp +++ b/mix_effect_test.cpp @@ -54,28 +54,30 @@ TEST(MixEffectTest, OnlyA) { TEST(MixEffectTest, DoesNotSumToOne) { float data_a[] = { - 1.0f, 0.5f, - 0.75f, 1.0f, + 1.0f, 0.5f, 0.75f, 0.333f, }; float data_b[] = { - 1.0f, 0.25f, - 0.15f, 0.6f, + 1.0f, 0.25f, 0.15f, 0.333f, }; + + // The fact that the RGB values don't sum but get averaged here might + // actually be a surprising result, but when you think of it, + // it does make physical sense. float expected_data[] = { - 0.0f, 0.25f, - 0.6f, 0.4f, + 1.0f, 0.375f, 0.45f, 0.666f, }; + float out_data[4]; - EffectChainTester tester(data_a, 2, 2, FORMAT_GRAYSCALE, COLORSPACE_sRGB, GAMMA_LINEAR); + EffectChainTester tester(data_a, 1, 1, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); Effect *input1 = tester.get_chain()->last_added_effect(); - Effect *input2 = tester.add_input(data_b, FORMAT_GRAYSCALE, COLORSPACE_sRGB, GAMMA_LINEAR); + Effect *input2 = tester.add_input(data_b, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); Effect *mix_effect = tester.get_chain()->add_effect(new MixEffect(), input1, input2); ASSERT_TRUE(mix_effect->set_float("strength_first", 1.0f)); - ASSERT_TRUE(mix_effect->set_float("strength_second", -1.0f)); - tester.run(out_data, GL_RED, COLORSPACE_sRGB, GAMMA_LINEAR); + ASSERT_TRUE(mix_effect->set_float("strength_second", 1.0f)); + tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); - expect_equal(expected_data, out_data, 2, 2); + expect_equal(expected_data, out_data, 4, 1); } TEST(MixEffectTest, MixesLinearlyDespitesRGBInputsAndOutputs) { diff --git a/overlay_effect.frag b/overlay_effect.frag index 38970ea..36c1a79 100644 --- a/overlay_effect.frag +++ b/overlay_effect.frag @@ -1,15 +1,22 @@ -// If we didn't have to worry about alpha in the bottom layer, -// this would be a simple mix(). However, since people might -// compose multiple layers together and we don't really have -// any control over the order, it's better to do it right. +// It's actually (but surprisingly) not correct to do a mix() here; +// it would be if we had postmultiplied alpha and didn't have to worry +// about alpha in the bottom layer, but given that we use premultiplied +// alpha all over, top shouldn't actually be multiplied by anything. // // These formulas come from Wikipedia: // // http://en.wikipedia.org/wiki/Alpha_compositing +// +// We use the associative version given. However, note that since we want +// _output_ to be premultiplied, C_o from Wikipedia is not what we want, +// but rather c_o (which is not explicitly given, but obviously is just +// C_o without the division by alpha_o). vec4 FUNCNAME(vec2 tc) { vec4 bottom = INPUT1(tc); vec4 top = INPUT2(tc); +#if 0 + // Postmultiplied version. float new_alpha = mix(bottom.a, 1.0, top.a); if (new_alpha < 1e-6) { // new_alpha = 0 only if top.a = bottom.a = 0, at least as long as @@ -23,4 +30,7 @@ vec4 FUNCNAME(vec2 tc) { vec3 color = premultiplied_color / new_alpha; return vec4(color.r, color.g, color.b, new_alpha); } +#else + return top + (1.0 - top.a) * bottom; +#endif } diff --git a/overlay_effect.h b/overlay_effect.h index 121027a..64224af 100644 --- a/overlay_effect.h +++ b/overlay_effect.h @@ -20,6 +20,11 @@ public: virtual bool needs_srgb_primaries() const { return false; } virtual unsigned num_inputs() const { return 2; } + + // Actually, if either image has blank alpha, our output will have + // blank alpha, too. However, understanding that would require changes + // to EffectChain, so postpone that optimization for later. + virtual AlphaHandling alpha_handling() const { return INPUT_AND_OUTPUT_ALPHA_PREMULTIPLIED; } }; #endif // !defined(_OVERLAY_EFFECT_H) diff --git a/overlay_effect_test.cpp b/overlay_effect_test.cpp index 009a757..865d392 100644 --- a/overlay_effect_test.cpp +++ b/overlay_effect_test.cpp @@ -32,9 +32,9 @@ TEST(OverlayEffectTest, BottomDominatesTopWhenTopIsTransparent) { 0.5f, 0.5f, 0.5f, 0.0f, }; float out_data[4]; - EffectChainTester tester(data_a, 1, 1, FORMAT_BGRA, COLORSPACE_sRGB, GAMMA_LINEAR); + EffectChainTester tester(data_a, 1, 1, FORMAT_BGRA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); Effect *input1 = tester.get_chain()->last_added_effect(); - Effect *input2 = tester.add_input(data_b, FORMAT_BGRA, COLORSPACE_sRGB, GAMMA_LINEAR); + Effect *input2 = tester.add_input(data_b, FORMAT_BGRA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); tester.get_chain()->add_effect(new OverlayEffect(), input1, input2); tester.run(out_data, GL_BGRA, COLORSPACE_sRGB, GAMMA_LINEAR); @@ -53,9 +53,9 @@ TEST(OverlayEffectTest, ZeroAlphaBecomesAllZero) { 0.0f, 0.0f, 0.0f, 0.0f }; float out_data[4]; - EffectChainTester tester(data_a, 1, 1, FORMAT_BGRA, COLORSPACE_sRGB, GAMMA_LINEAR); + EffectChainTester tester(data_a, 1, 1, FORMAT_BGRA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); Effect *input1 = tester.get_chain()->last_added_effect(); - Effect *input2 = tester.add_input(data_b, FORMAT_BGRA, COLORSPACE_sRGB, GAMMA_LINEAR); + Effect *input2 = tester.add_input(data_b, FORMAT_BGRA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); tester.get_chain()->add_effect(new OverlayEffect(), input1, input2); tester.run(out_data, GL_BGRA, COLORSPACE_sRGB, GAMMA_LINEAR); @@ -77,9 +77,9 @@ TEST(OverlayEffectTest, PhotoshopReferenceTest) { 179.0f/255.0f, 153.0f/255.0f, 51.0f/255.0f, 0.625f }; float out_data[4]; - EffectChainTester tester(data_a, 1, 1, FORMAT_BGRA, COLORSPACE_sRGB, GAMMA_LINEAR); + EffectChainTester tester(data_a, 1, 1, FORMAT_BGRA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); Effect *input1 = tester.get_chain()->last_added_effect(); - Effect *input2 = tester.add_input(data_b, FORMAT_BGRA, COLORSPACE_sRGB, GAMMA_LINEAR); + Effect *input2 = tester.add_input(data_b, FORMAT_BGRA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); tester.get_chain()->add_effect(new OverlayEffect(), input1, input2); tester.run(out_data, GL_BGRA, COLORSPACE_sRGB, GAMMA_LINEAR); diff --git a/saturation_effect.h b/saturation_effect.h index 4dd72e4..27b1d00 100644 --- a/saturation_effect.h +++ b/saturation_effect.h @@ -13,6 +13,7 @@ class SaturationEffect : public Effect { public: SaturationEffect(); virtual std::string effect_type_id() const { return "SaturationEffect"; } + virtual AlphaHandling alpha_handling() const { return DONT_CARE_ALPHA_TYPE; } std::string output_fragment_shader(); private: diff --git a/saturation_effect_test.cpp b/saturation_effect_test.cpp index 80b56a7..6712197 100644 --- a/saturation_effect_test.cpp +++ b/saturation_effect_test.cpp @@ -9,7 +9,7 @@ TEST(SaturationEffectTest, SaturationOneIsPassThrough) { 1.0f, 0.5f, 0.75f, 0.6f, }; float out_data[4]; - EffectChainTester tester(data, 1, 1, FORMAT_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); + EffectChainTester tester(data, 1, 1, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); Effect *saturation_effect = tester.get_chain()->add_effect(new SaturationEffect()); ASSERT_TRUE(saturation_effect->set_float("saturation", 1.0f)); tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); @@ -34,7 +34,7 @@ TEST(SaturationEffectTest, SaturationZeroRemovesColorButPreservesAlpha) { }; float out_data[5 * 4]; - EffectChainTester tester(data, 5, 1, FORMAT_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); + EffectChainTester tester(data, 5, 1, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); Effect *saturation_effect = tester.get_chain()->add_effect(new SaturationEffect()); ASSERT_TRUE(saturation_effect->set_float("saturation", 0.0f)); tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); @@ -55,7 +55,7 @@ TEST(SaturationEffectTest, DoubleSaturation) { }; float out_data[3 * 4]; - EffectChainTester tester(data, 3, 1, FORMAT_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); + EffectChainTester tester(data, 3, 1, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); Effect *saturation_effect = tester.get_chain()->add_effect(new SaturationEffect()); ASSERT_TRUE(saturation_effect->set_float("saturation", 2.0f)); tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); diff --git a/test_util.cpp b/test_util.cpp index 68e0b93..111f8b2 100644 --- a/test_util.cpp +++ b/test_util.cpp @@ -89,10 +89,10 @@ Input *EffectChainTester::add_input(const unsigned char *data, MovitPixelFormat return input; } -void EffectChainTester::run(float *out_data, GLenum format, Colorspace color_space, GammaCurve gamma_curve) +void EffectChainTester::run(float *out_data, GLenum format, Colorspace color_space, GammaCurve gamma_curve, OutputAlphaFormat alpha_format) { if (!finalized) { - finalize_chain(color_space, gamma_curve); + finalize_chain(color_space, gamma_curve, alpha_format); } chain.render_to_fbo(fbo, width, height); @@ -107,10 +107,10 @@ void EffectChainTester::run(float *out_data, GLenum format, Colorspace color_spa vertical_flip(out_data, width, height); } -void EffectChainTester::run(unsigned char *out_data, GLenum format, Colorspace color_space, GammaCurve gamma_curve) +void EffectChainTester::run(unsigned char *out_data, GLenum format, Colorspace color_space, GammaCurve gamma_curve, OutputAlphaFormat alpha_format) { if (!finalized) { - finalize_chain(color_space, gamma_curve); + finalize_chain(color_space, gamma_curve, alpha_format); } chain.render_to_fbo(fbo, width, height); @@ -125,13 +125,13 @@ void EffectChainTester::run(unsigned char *out_data, GLenum format, Colorspace c vertical_flip(out_data, width, height); } -void EffectChainTester::finalize_chain(Colorspace color_space, GammaCurve gamma_curve) +void EffectChainTester::finalize_chain(Colorspace color_space, GammaCurve gamma_curve, OutputAlphaFormat alpha_format) { assert(!finalized); ImageFormat image_format; image_format.color_space = color_space; image_format.gamma_curve = gamma_curve; - chain.add_output(image_format); + chain.add_output(image_format, alpha_format); chain.finalize(); finalized = true; } diff --git a/test_util.h b/test_util.h index 407824d..9066591 100644 --- a/test_util.h +++ b/test_util.h @@ -15,11 +15,11 @@ public: EffectChain *get_chain() { return &chain; } Input *add_input(const float *data, MovitPixelFormat pixel_format, Colorspace color_space, GammaCurve gamma_curve); Input *add_input(const unsigned char *data, MovitPixelFormat pixel_format, Colorspace color_space, GammaCurve gamma_curve); - void run(float *out_data, GLenum format, Colorspace color_space, GammaCurve gamma_curve); - void run(unsigned char *out_data, GLenum format, Colorspace color_space, GammaCurve gamma_curve); + void run(float *out_data, GLenum format, Colorspace color_space, GammaCurve gamma_curve, OutputAlphaFormat alpha_format = OUTPUT_ALPHA_POSTMULTIPLIED); + void run(unsigned char *out_data, GLenum format, Colorspace color_space, GammaCurve gamma_curve, OutputAlphaFormat alpha_format = OUTPUT_ALPHA_POSTMULTIPLIED); private: - void finalize_chain(Colorspace color_space, GammaCurve gamma_curve); + void finalize_chain(Colorspace color_space, GammaCurve gamma_curve, OutputAlphaFormat alpha_format); EffectChain chain; GLuint fbo, texnum; diff --git a/vignette_effect.h b/vignette_effect.h index b8634fc..a394703 100644 --- a/vignette_effect.h +++ b/vignette_effect.h @@ -13,6 +13,7 @@ public: std::string output_fragment_shader(); virtual bool needs_srgb_primaries() const { return false; } + virtual AlphaHandling alpha_handling() const { return DONT_CARE_ALPHA_TYPE; } void set_gl_state(GLuint glsl_program_num, const std::string &prefix, unsigned *sampler_num); diff --git a/white_balance_effect.h b/white_balance_effect.h index 80416cb..2f8f38e 100644 --- a/white_balance_effect.h +++ b/white_balance_effect.h @@ -9,6 +9,7 @@ class WhiteBalanceEffect : public Effect { public: WhiteBalanceEffect(); virtual std::string effect_type_id() const { return "WhiteBalanceEffect"; } + virtual AlphaHandling alpha_handling() const { return DONT_CARE_ALPHA_TYPE; } std::string output_fragment_shader(); void set_gl_state(GLuint glsl_program_num, const std::string &prefix, unsigned *sampler_num); diff --git a/white_balance_effect_test.cpp b/white_balance_effect_test.cpp index 966cba3..733b5bc 100644 --- a/white_balance_effect_test.cpp +++ b/white_balance_effect_test.cpp @@ -17,7 +17,7 @@ TEST(WhiteBalanceEffectTest, GrayNeutralDoesNothing) { }; float out_data[5 * 4]; - EffectChainTester tester(data, 1, 5, FORMAT_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); + EffectChainTester tester(data, 1, 5, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); Effect *white_balance_effect = tester.get_chain()->add_effect(new WhiteBalanceEffect()); ASSERT_TRUE(white_balance_effect->set_vec3("neutral_color", neutral)); tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); @@ -36,7 +36,7 @@ TEST(WhiteBalanceEffectTest, SettingReddishNeutralColorNeutralizesReddishColor) }; float out_data[3 * 4]; - EffectChainTester tester(data, 1, 3, FORMAT_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); + EffectChainTester tester(data, 1, 3, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); Effect *white_balance_effect = tester.get_chain()->add_effect(new WhiteBalanceEffect()); ASSERT_TRUE(white_balance_effect->set_vec3("neutral_color", neutral)); tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); @@ -73,7 +73,7 @@ TEST(WhiteBalanceEffectTest, HigherColorTemperatureIncreasesBlue) { }; float out_data[2 * 4]; - EffectChainTester tester(data, 1, 2, FORMAT_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); + EffectChainTester tester(data, 1, 2, FORMAT_RGBA_POSTMULTIPLIED_ALPHA, COLORSPACE_sRGB, GAMMA_LINEAR); Effect *white_balance_effect = tester.get_chain()->add_effect(new WhiteBalanceEffect()); ASSERT_TRUE(white_balance_effect->set_float("output_color_temperature", 10000.0f)); tester.run(out_data, GL_RGBA, COLORSPACE_sRGB, GAMMA_LINEAR); diff --git a/ycbcr_input.h b/ycbcr_input.h index fba5588..492230f 100644 --- a/ycbcr_input.h +++ b/ycbcr_input.h @@ -41,6 +41,7 @@ public: void finalize(); virtual bool can_output_linear_gamma() const { return false; } + virtual AlphaHandling alpha_handling() const { return OUTPUT_BLANK_ALPHA; } std::string output_fragment_shader(); -- 2.39.2