From: Steinar H. Gunderson Date: Sat, 27 Oct 2012 11:38:46 +0000 (+0200) Subject: Add an effect for Lanczos resampling. X-Git-Tag: 1.0~226 X-Git-Url: https://git.sesse.net/?p=movit;a=commitdiff_plain;h=fb92a4e217a92ecf83b7812cc6933f6f3048b752 Add an effect for Lanczos resampling. --- diff --git a/Makefile b/Makefile index aed37a4..edbe09b 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,7 @@ TESTS += unsharp_mask_effect_test TESTS += diffusion_effect_test TESTS += white_balance_effect_test TESTS += lift_gamma_gain_effect_test +TESTS += resample_effect_test TESTS += flat_input_test TESTS += ycbcr_input_test @@ -51,6 +52,7 @@ LIB_OBJS += glow_effect.o LIB_OBJS += unsharp_mask_effect.o LIB_OBJS += mix_effect.o LIB_OBJS += resize_effect.o +LIB_OBJS += resample_effect.o LIB_OBJS += deconvolution_sharpen_effect.o LIB_OBJS += sandbox_effect.o @@ -90,6 +92,8 @@ white_balance_effect_test: white_balance_effect_test.o $(TEST_OBJS) libmovit.a $(CXX) -o $@ $^ $(LDFLAGS) lift_gamma_gain_effect_test: lift_gamma_gain_effect_test.o $(TEST_OBJS) libmovit.a $(CXX) -o $@ $^ $(LDFLAGS) +resample_effect_test: resample_effect_test.o $(TEST_OBJS) libmovit.a + $(CXX) -o $@ $^ $(LDFLAGS) flat_input_test: flat_input_test.o $(TEST_OBJS) libmovit.a $(CXX) -o $@ $^ $(LDFLAGS) ycbcr_input_test: ycbcr_input_test.o $(TEST_OBJS) libmovit.a diff --git a/resample_effect.cpp b/resample_effect.cpp new file mode 100644 index 0000000..c0e82f1 --- /dev/null +++ b/resample_effect.cpp @@ -0,0 +1,284 @@ +// Three-lobed Lanczos, the most common choice. +#define LANCZOS_RADIUS 3.0 + +#include +#include + +#include "resample_effect.h" +#include "effect_chain.h" +#include "util.h" +#include "opengl.h" + +namespace { + +float sinc(float x) +{ + if (fabs(x) < 1e-6) { + return 1.0f - fabs(x); + } else { + return sin(x) / x; + } +} + +float lanczos_weight(float x, float a) +{ + if (fabs(x) > a) { + return 0.0f; + } else { + return sinc(M_PI * x) * sinc(M_PI * x / a); + } +} + +} // namespace + +ResampleEffect::ResampleEffect() + : input_width(1280), + input_height(720) +{ + register_int("width", &output_width); + register_int("height", &output_height); + + // The first blur pass will forward resolution information to us. + hpass = new SingleResamplePassEffect(this); + hpass->set_int("direction", SingleResamplePassEffect::HORIZONTAL); + vpass = new SingleResamplePassEffect(NULL); + vpass->set_int("direction", SingleResamplePassEffect::VERTICAL); + + update_size(); +} + +void ResampleEffect::rewrite_graph(EffectChain *graph, Node *self) +{ + Node *hpass_node = graph->add_node(hpass); + Node *vpass_node = graph->add_node(vpass); + graph->connect_nodes(hpass_node, vpass_node); + graph->replace_receiver(self, hpass_node); + graph->replace_sender(self, vpass_node); + self->disabled = true; +} + +// We get this information forwarded from the first blur pass, +// since we are not part of the chain ourselves. +void ResampleEffect::inform_input_size(unsigned input_num, unsigned width, unsigned height) +{ + assert(input_num == 0); + assert(width != 0); + assert(height != 0); + input_width = width; + input_height = height; + update_size(); +} + +void ResampleEffect::update_size() +{ + bool ok = true; + ok |= hpass->set_int("input_width", input_width); + ok |= hpass->set_int("input_height", input_height); + ok |= hpass->set_int("output_width", output_width); + ok |= hpass->set_int("output_height", input_height); + + ok |= vpass->set_int("input_width", output_width); + ok |= vpass->set_int("input_height", input_height); + ok |= vpass->set_int("output_width", output_width); + ok |= vpass->set_int("output_height", output_height); + + assert(ok); +} + +bool ResampleEffect::set_float(const std::string &key, float value) { + if (key == "width") { + output_width = value; + update_size(); + return true; + } + if (key == "height") { + output_height = value; + update_size(); + return true; + } + return false; +} + +SingleResamplePassEffect::SingleResamplePassEffect(ResampleEffect *parent) + : parent(parent), + direction(HORIZONTAL), + input_width(1280), + input_height(720), + last_input_width(-1), + last_input_height(-1), + last_output_width(-1), + last_output_height(-1) +{ + register_int("direction", (int *)&direction); + register_int("input_width", &input_width); + register_int("input_height", &input_height); + register_int("output_width", &output_width); + register_int("output_height", &output_height); + + glGenTextures(1, &texnum); +} + +SingleResamplePassEffect::~SingleResamplePassEffect() +{ + glDeleteTextures(1, &texnum); +} + +std::string SingleResamplePassEffect::output_fragment_shader() +{ + char buf[256]; + sprintf(buf, "#define DIRECTION_VERTICAL %d\n", (direction == VERTICAL)); + return buf + read_file("resample_effect.frag"); +} + +// Using vertical scaling as an example: +// +// Generally out[y] = w0 * in[yi] + w1 * in[yi + 1] + w2 * in[yi + 2] + ... +// +// Obviously, yi will depend on y (in a not-quite-linear way), but so will +// the weights w0, w1, w2, etc.. The easiest way of doing this is to encode, +// for each sample, the weight and the yi value, e.g. , , +// and so on. For each y, we encode these along the x-axis (since that is spare), +// so out[0] will read from parameters = <0,0>, <1,0>, <2,0> and so on. +// +// For horizontal scaling, we fill in the exact same texture; +// the shader just interprets is differently. +// +// TODO: Support optimization of wrapping the sample texture. +// TODO: Support optimization using free linear sampling, like in BlurEffect. +void SingleResamplePassEffect::update_texture(GLuint glsl_program_num, const std::string &prefix, unsigned *sampler_num) +{ + unsigned src_size, dst_size; + if (direction == SingleResamplePassEffect::HORIZONTAL) { + assert(input_height == output_height); + src_size = input_width; + dst_size = output_width; + } else if (direction == SingleResamplePassEffect::VERTICAL) { + assert(input_width == output_width); + src_size = input_height; + dst_size = output_height; + } else { + assert(false); + } + + // Sample the kernel in the right place. A diagram with a triangular kernel + // (corresponding to linear filtering, and obviously with radius 1) + // for easier ASCII art drawing: + // + // * + // / \ | + // / \ | + // / \ | + // x---x---x x x---x---x---x + // + // Scaling up (in this case, 2x) means sampling more densely: + // + // * + // / \ | + // / \ | + // / \ | + // x-x-x-x-x-x x x x-x-x-x-x-x-x-x + // + // When scaling up, any destination pixel will only be influenced by a few + // (in this case, two) neighboring pixels, and more importantly, the number + // will not be influenced by the scaling factor. (Note, however, that the + // pixel centers have moved, due to OpenGL's center-pixel convention.) + // The only thing that changes is the weights themselves, as the sampling + // points are at different distances from the original pixels. + // + // Scaling down is a different story: + // + // * + // / \ | + // / \ | + // / \ | + // --x------ x --x-------x-- + // + // Again, the pixel centers have moved in a maybe unintuitive fashion, + // although when you consider that there are multiple source pixels around, + // it's not so bad as at first look: + // + // * * * * + // / \ / \ / \ / \ | + // / X X X \ | + // / / \ / \ / \ \ | + // --x-------x-------x-------x-- + // + // As you can see, the new pixels become averages of the two neighboring old + // ones (the situation for Lanczos is of course more complex). + // + // Anyhow, in this case we clearly need to look at more source pixels + // to compute the destination pixel, and how many depend on the scaling factor. + // Thus, the kernel width will vary with how much we scale. + float radius_scaling_factor = std::min(float(dst_size) / float(src_size), 1.0f); + int int_radius = lrintf(LANCZOS_RADIUS / radius_scaling_factor); + src_samples = int_radius * 2 + 1; + float *weights = new float[dst_size * src_samples * 2]; + for (unsigned y = 0; y < dst_size; ++y) { + // Find the point around which we want to sample the source image, + // compensating for differing pixel centers as the scale changes. + float center_src_y = (y + 0.5f) * float(src_size) / float(dst_size) - 0.5f; + int base_src_y = lrintf(center_src_y); + + // Now sample pixels on each side around that point. + for (int i = 0; i < src_samples; ++i) { + int src_y = base_src_y + i - int_radius; + float weight = lanczos_weight(radius_scaling_factor * (src_y - center_src_y), LANCZOS_RADIUS); + weights[(y * src_samples + i) * 2 + 0] = weight * radius_scaling_factor; + weights[(y * src_samples + i) * 2 + 1] = (src_y + 0.5) / float(src_size); + } + } + + // Encode as a two-component texture. Note the GL_REPEAT, which is not relevant + // right now, but will be later. + glActiveTexture(GL_TEXTURE0 + *sampler_num); + check_error(); + glBindTexture(GL_TEXTURE_2D, texnum); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT); + check_error(); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RG16F, src_samples, dst_size, 0, GL_RG, GL_FLOAT, weights); + check_error(); + + delete[] weights; + +} + +void SingleResamplePassEffect::set_gl_state(GLuint glsl_program_num, const std::string &prefix, unsigned *sampler_num) +{ + Effect::set_gl_state(glsl_program_num, prefix, sampler_num); + + if (input_width != last_input_width || + input_height != last_input_height || + output_width != last_output_width || + output_height != last_output_height) { + update_texture(glsl_program_num, prefix, sampler_num); + last_input_width = input_width; + last_input_height = input_height; + last_output_width = output_width; + last_output_height = output_height; + } + + glActiveTexture(GL_TEXTURE0 + *sampler_num); + check_error(); + glBindTexture(GL_TEXTURE_2D, texnum); + check_error(); + + set_uniform_int(glsl_program_num, prefix, "sample_tex", *sampler_num); + ++sampler_num; + set_uniform_int(glsl_program_num, prefix, "num_samples", src_samples); + + // Instructions for how to convert integer sample numbers to positions in the weight texture. + set_uniform_float(glsl_program_num, prefix, "sample_x_scale", 1.0f / src_samples); + set_uniform_float(glsl_program_num, prefix, "sample_x_offset", 0.5f / src_samples); + + // We specifically do not want mipmaps on the input texture; + // they break minification. + glActiveTexture(GL_TEXTURE0); + check_error(); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + check_error(); +} diff --git a/resample_effect.frag b/resample_effect.frag new file mode 100644 index 0000000..6aac29d --- /dev/null +++ b/resample_effect.frag @@ -0,0 +1,38 @@ +// DIRECTION_VERTICAL will be #defined to 1 if we are scaling vertically, +// and 0 otherwise. + +uniform sampler2D PREFIX(sample_tex); +uniform int PREFIX(num_samples); +uniform float PREFIX(sample_x_scale); +uniform float PREFIX(sample_x_offset); + +// Sample a single weight. First fetch information about where to sample +// and the weight from sample_tex, and then read the pixel itself. +vec4 PREFIX(do_sample)(vec2 tc, int i) +{ + vec2 sample_tc; + sample_tc.x = float(i) * PREFIX(sample_x_scale) + PREFIX(sample_x_offset); +#if DIRECTION_VERTICAL + sample_tc.y = tc.y; +#else + sample_tc.y = tc.x; +#endif + vec2 sample = texture2D(PREFIX(sample_tex), sample_tc).rg; + +#if DIRECTION_VERTICAL + tc.y = sample.g; +#else + tc.x = sample.g; +#endif + return vec4(sample.r) * INPUT(tc); +} + +vec4 FUNCNAME(vec2 tc) { + vec4 sum = PREFIX(do_sample)(tc, 0); + for (int i = 1; i < PREFIX(num_samples); ++i) { + sum += PREFIX(do_sample)(tc, i); + } + return sum; +} + +#undef DIRECTION_VERTICAL diff --git a/resample_effect.h b/resample_effect.h new file mode 100644 index 0000000..fbe5fc5 --- /dev/null +++ b/resample_effect.h @@ -0,0 +1,93 @@ +#ifndef _RESAMPLE_EFFECT_H +#define _RESAMPLE_EFFECT_H 1 + +// High-quality image resizing, either up or down. +// +// The default scaling offered by the GPU (and as used in ResizeEffect) +// is bilinear (optionally mipmapped), which is not the highest-quality +// choice, especially for upscaling. ResampleEffect offers the three-lobed +// Lanczos kernel, which is among the most popular choices in image +// processing. While it does have its weaknesses, in particular a certain +// ringing/sharpening effect with artifacts that accumulate over several +// consecutive resizings, it is generally regarded as the best tradeoff. +// +// Works in two passes; first horizontal, then vertical (ResampleEffect, +// which is what the user is intended to use, instantiates two copies of +// SingleResamplePassEffect behind the scenes). + +#include "effect.h" + +class SingleResamplePassEffect; + +class ResampleEffect : public Effect { +public: + ResampleEffect(); + + virtual std::string effect_type_id() const { return "ResampleEffect"; } + + // We want this for the same reason as ResizeEffect; we could end up scaling + // down quite a lot. + virtual bool needs_texture_bounce() const { return true; } + virtual bool needs_srgb_primaries() const { return false; } + + virtual void inform_input_size(unsigned input_num, unsigned width, unsigned height); + + virtual std::string output_fragment_shader() { + assert(false); + } + virtual void set_gl_state(GLuint glsl_program_num, const std::string &prefix, unsigned *sampler_num) { + assert(false); + } + + virtual void rewrite_graph(EffectChain *graph, Node *self); + virtual bool set_float(const std::string &key, float value); + +private: + void update_size(); + + SingleResamplePassEffect *hpass, *vpass; + int input_width, input_height, output_width, output_height; +}; + +class SingleResamplePassEffect : public Effect { +public: + // If parent is non-NULL, calls to inform_input_size will be forwarded, + // so that it can inform both passes about the right input and output + // resolutions. + SingleResamplePassEffect(ResampleEffect *parent); + ~SingleResamplePassEffect(); + virtual std::string effect_type_id() const { return "SingleResamplePassEffect"; } + + std::string output_fragment_shader(); + + virtual bool needs_texture_bounce() const { return true; } + virtual bool needs_srgb_primaries() const { return false; } + + virtual void inform_input_size(unsigned input_num, unsigned width, unsigned height) { + if (parent != NULL) { + parent->inform_input_size(input_num, width, height); + } + } + virtual bool changes_output_size() const { return true; } + + virtual void get_output_size(unsigned *width, unsigned *height) const { + *width = this->output_width; + *height = this->output_height; + } + + void set_gl_state(GLuint glsl_program_num, const std::string &prefix, unsigned *sampler_num); + + enum Direction { HORIZONTAL = 0, VERTICAL = 1 }; + +private: + void update_texture(GLuint glsl_program_num, const std::string &prefix, unsigned *sampler_num); + + ResampleEffect *parent; + Direction direction; + GLuint texnum; + int input_width, input_height, output_width, output_height; + int last_input_width, last_input_height, last_output_width, last_output_height; + int src_samples; +}; + +#endif // !defined(_RESAMPLE_EFFECT_H) diff --git a/resample_effect_test.cpp b/resample_effect_test.cpp new file mode 100644 index 0000000..462f5c9 --- /dev/null +++ b/resample_effect_test.cpp @@ -0,0 +1,162 @@ +// Unit tests for ResampleEffect. + +#include "test_util.h" +#include "gtest/gtest.h" +#include "resample_effect.h" +#include "flat_input.h" + +namespace { + +float sinc(float x) +{ + return sin(M_PI * x) / (M_PI * x); +} + +float lanczos(float x, float a) +{ + if (fabs(x) >= a) { + return 0.0f; + } else { + return sinc(x) * sinc(x / a); + } +} + +} // namespace + +TEST(ResampleEffectTest, IdentityTransformDoesNothing) { + const int size = 4; + + float data[size * size] = { + 0.0, 1.0, 0.0, 1.0, + 0.0, 1.0, 1.0, 0.0, + 0.0, 0.5, 1.0, 0.5, + 0.0, 0.0, 0.0, 0.0, + }; + float out_data[size * size]; + + EffectChainTester tester(data, size, size, FORMAT_GRAYSCALE, COLORSPACE_sRGB, GAMMA_LINEAR); + Effect *resample_effect = tester.get_chain()->add_effect(new ResampleEffect()); + ASSERT_TRUE(resample_effect->set_int("width", 4)); + ASSERT_TRUE(resample_effect->set_int("height", 4)); + tester.run(out_data, GL_RED, COLORSPACE_sRGB, GAMMA_LINEAR); + + expect_equal(data, out_data, size, size); +} + +TEST(ResampleEffectTest, UpscaleByTwoGetsCorrectPixelCenters) { + const int size = 5; + + float data[size * size] = { + 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 1.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, + }; + float expected_data[size * size * 4], out_data[size * size * 4]; + + for (int y = 0; y < size * 2; ++y) { + for (int x = 0; x < size * 2; ++x) { + float weight = lanczos((x - size + 0.5f) * 0.5f, 3.0f); + weight *= lanczos((y - size + 0.5f) * 0.5f, 3.0f); + expected_data[y * (size * 2) + x] = weight; + } + } + + EffectChainTester tester(NULL, size * 2, size * 2, FORMAT_GRAYSCALE, COLORSPACE_sRGB, GAMMA_LINEAR); + + ImageFormat format; + format.color_space = COLORSPACE_sRGB; + format.gamma_curve = GAMMA_LINEAR; + + FlatInput *input = new FlatInput(format, FORMAT_GRAYSCALE, GL_FLOAT, size, size); + input->set_pixel_data(data); + tester.get_chain()->add_input(input); + + Effect *resample_effect = tester.get_chain()->add_effect(new ResampleEffect()); + ASSERT_TRUE(resample_effect->set_int("width", size * 2)); + ASSERT_TRUE(resample_effect->set_int("height", size * 2)); + tester.run(out_data, GL_RED, COLORSPACE_sRGB, GAMMA_LINEAR); + + expect_equal(expected_data, out_data, size * 2, size * 2); +} + +TEST(ResampleEffectTest, DownscaleByTwoGetsCorrectPixelCenters) { + const int size = 5; + + // This isn't a perfect dot, since the Lanczos filter has a slight + // sharpening effect; the most important thing is that we have kept + // the texel center right (everything is nicely symmetric). + // The approximate magnitudes have been checked against ImageMagick. + float expected_data[size * size] = { + 0.0045, -0.0067, -0.0598, -0.0067, 0.0045, + -0.0067, 0.0099, 0.0886, 0.0099, -0.0067, + -0.0598, 0.0886, 0.7930, 0.0886, -0.0598, + -0.0067, 0.0099, 0.0886, 0.0099, -0.0067, + 0.0045, -0.0067, -0.0598, -0.0067, 0.0045, + }; + float data[size * size * 4], out_data[size * size]; + + for (int y = 0; y < size * 2; ++y) { + for (int x = 0; x < size * 2; ++x) { + float weight = lanczos((x - size + 0.5f) * 0.5f, 3.0f); + weight *= lanczos((y - size + 0.5f) * 0.5f, 3.0f); + data[y * (size * 2) + x] = weight; + } + } + + EffectChainTester tester(NULL, size, size, FORMAT_GRAYSCALE, COLORSPACE_sRGB, GAMMA_LINEAR); + + ImageFormat format; + format.color_space = COLORSPACE_sRGB; + format.gamma_curve = GAMMA_LINEAR; + + FlatInput *input = new FlatInput(format, FORMAT_GRAYSCALE, GL_FLOAT, size * 2, size * 2); + input->set_pixel_data(data); + tester.get_chain()->add_input(input); + + Effect *resample_effect = tester.get_chain()->add_effect(new ResampleEffect()); + ASSERT_TRUE(resample_effect->set_int("width", size)); + ASSERT_TRUE(resample_effect->set_int("height", size)); + tester.run(out_data, GL_RED, COLORSPACE_sRGB, GAMMA_LINEAR); + + expect_equal(expected_data, out_data, size, size); +} + +TEST(ResampleEffectTest, UpscaleByThreeGetsCorrectPixelCenters) { + const int size = 5; + + float data[size * size] = { + 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 1.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0, 0.0, + }; + float out_data[size * size * 9]; + + EffectChainTester tester(NULL, size * 3, size * 3, FORMAT_GRAYSCALE, COLORSPACE_sRGB, GAMMA_LINEAR); + + ImageFormat format; + format.color_space = COLORSPACE_sRGB; + format.gamma_curve = GAMMA_LINEAR; + + FlatInput *input = new FlatInput(format, FORMAT_GRAYSCALE, GL_FLOAT, size, size); + input->set_pixel_data(data); + tester.get_chain()->add_input(input); + + Effect *resample_effect = tester.get_chain()->add_effect(new ResampleEffect()); + ASSERT_TRUE(resample_effect->set_int("width", size * 3)); + ASSERT_TRUE(resample_effect->set_int("height", size * 3)); + tester.run(out_data, GL_RED, COLORSPACE_sRGB, GAMMA_LINEAR); + + // We only bother checking that the middle pixel is still correct, + // and that symmetry holds. + EXPECT_FLOAT_EQ(1.0, out_data[7 * (size * 3) + 7]); + for (unsigned y = 0; y < size * 3; ++y) { + for (unsigned x = 0; x < size * 3; ++x) { + EXPECT_FLOAT_EQ(out_data[y * (size * 3) + x], out_data[(size * 3 - y - 1) * (size * 3) + x]); + EXPECT_FLOAT_EQ(out_data[y * (size * 3) + x], out_data[y * (size * 3) + (size * 3 - x - 1)]); + } + } +}