Add support for offsets in ResampleEffect.

This enables smooth (subpixel) panning that people frequently want for stills
and titles, but that you couldn't do in a subpixel fashion before (PaddingEffect
could only do integer pixel offsets).

The placement (ResampleEffect) might seem a bit off at first, but subpixel
offset needs resampling, and ResampleEffect already has all the logic in place
for that. We could have used the GPU's built-in bilinear resampling, of course,
but it doesn't look all that good for high-contrast situations (although working
in linear light should help some).

diff --git a/resample_effect.cpp b/resample_effect.cpp
index 50b7c6bc5c4730fc6bb25cd18c96a446c8f02986..17745bc6d564e9340d756894b5f5058b34dbc1b8 100644 (file)
--- a/resample_effect.cpp
+++ b/resample_effect.cpp
@@ -171,6 +171,13 @@ bool ResampleEffect::set_float(const string &key, float value) {
                update_size();
                return true;
        }
                update_size();
                return true;
        }

+       if (key == "top") {
+               // Compensate for the bottom-left origin.
+               return vpass->set_float("offset", -value);
+       }
+       if (key == "left") {
+               return hpass->set_float("offset", value);
+       }

        return false;
 }
 
        return false;
 }
 


@@ -179,16 +186,19 @@ SingleResamplePassEffect::SingleResamplePassEffect(ResampleEffect *parent)
          direction(HORIZONTAL),
          input_width(1280),
          input_height(720),
          direction(HORIZONTAL),
          input_width(1280),
          input_height(720),

+         offset(0.0),

          last_input_width(-1),
          last_input_height(-1),
          last_output_width(-1),
          last_input_width(-1),
          last_input_height(-1),
          last_output_width(-1),

-         last_output_height(-1)
+         last_output_height(-1),
+         last_offset(0.0 / 0.0)  // NaN.

 {
        register_int("direction", (int *)&direction);
        register_int("input_width", &input_width);
        register_int("input_height", &input_height);
        register_int("output_width", &output_width);
        register_int("output_height", &output_height);
 {
        register_int("direction", (int *)&direction);
        register_int("input_width", &input_width);
        register_int("input_height", &input_height);
        register_int("output_width", &output_width);
        register_int("output_height", &output_height);

+       register_float("offset", &offset);

 
        glGenTextures(1, &texnum);
 }
 
        glGenTextures(1, &texnum);
 }


@@ -294,10 +304,12 @@ void SingleResamplePassEffect::update_texture(GLuint glsl_program_num, const str
        int int_radius = lrintf(LANCZOS_RADIUS / radius_scaling_factor);
        int src_samples = int_radius * 2 + 1;
        float *weights = new float[dst_samples * src_samples * 2];
        int int_radius = lrintf(LANCZOS_RADIUS / radius_scaling_factor);
        int src_samples = int_radius * 2 + 1;
        float *weights = new float[dst_samples * src_samples * 2];

+       float subpixel_offset = offset - lrintf(offset);  // The part not covered by whole_pixel_offset.
+       assert(subpixel_offset >= -0.5f && subpixel_offset <= 0.5f);

        for (unsigned y = 0; y < dst_samples; ++y) {
                // Find the point around which we want to sample the source image,
                // compensating for differing pixel centers as the scale changes.
        for (unsigned y = 0; y < dst_samples; ++y) {
                // Find the point around which we want to sample the source image,
                // compensating for differing pixel centers as the scale changes.

-               float center_src_y = (y + 0.5f) * float(src_size) / float(dst_size) - 0.5f;
+               float center_src_y = (y + subpixel_offset + 0.5f) * float(src_size) / float(dst_size) - 0.5f;

                int base_src_y = lrintf(center_src_y);
 
                // Now sample <int_radius> pixels on each side around that point.
                int base_src_y = lrintf(center_src_y);
 
                // Now sample <int_radius> pixels on each side around that point.


@@ -307,7 +319,6 @@ void SingleResamplePassEffect::update_texture(GLuint glsl_program_num, const str
                        weights[(y * src_samples + i) * 2 + 0] = weight * radius_scaling_factor;
                        weights[(y * src_samples + i) * 2 + 1] = (src_y + 0.5) / float(src_size);
                }
                        weights[(y * src_samples + i) * 2 + 0] = weight * radius_scaling_factor;
                        weights[(y * src_samples + i) * 2 + 1] = (src_y + 0.5) / float(src_size);
                }

-

        }
 
        // Now make use of the bilinear filtering in the GPU to reduce the number of samples
        }
 
        // Now make use of the bilinear filtering in the GPU to reduce the number of samples


@@ -388,12 +399,14 @@ void SingleResamplePassEffect::set_gl_state(GLuint glsl_program_num, const strin
        if (input_width != last_input_width ||
            input_height != last_input_height ||
            output_width != last_output_width ||
        if (input_width != last_input_width ||
            input_height != last_input_height ||
            output_width != last_output_width ||

-           output_height != last_output_height) {
+           output_height != last_output_height ||
+           offset != last_offset) {

                update_texture(glsl_program_num, prefix, sampler_num);
                last_input_width = input_width;
                last_input_height = input_height;
                last_output_width = output_width;
                last_output_height = output_height;
                update_texture(glsl_program_num, prefix, sampler_num);
                last_input_width = input_width;
                last_input_height = input_height;
                last_output_width = output_width;
                last_output_height = output_height;

+               last_offset = offset;

        }
 
        glActiveTexture(GL_TEXTURE0 + *sampler_num);
        }
 
        glActiveTexture(GL_TEXTURE0 + *sampler_num);


@@ -411,6 +424,14 @@ void SingleResamplePassEffect::set_gl_state(GLuint glsl_program_num, const strin
        set_uniform_float(glsl_program_num, prefix, "sample_x_scale", 1.0f / src_bilinear_samples);
        set_uniform_float(glsl_program_num, prefix, "sample_x_offset", 0.5f / src_bilinear_samples);
 
        set_uniform_float(glsl_program_num, prefix, "sample_x_scale", 1.0f / src_bilinear_samples);
        set_uniform_float(glsl_program_num, prefix, "sample_x_offset", 0.5f / src_bilinear_samples);
 

+       float whole_pixel_offset;
+       if (direction == SingleResamplePassEffect::VERTICAL) {
+               whole_pixel_offset = lrintf(offset) / float(input_height);
+       } else {
+               whole_pixel_offset = lrintf(offset) / float(input_width);
+       }
+       set_uniform_float(glsl_program_num, prefix, "whole_pixel_offset", whole_pixel_offset);
+

        // We specifically do not want mipmaps on the input texture;
        // they break minification.
        Node *self = chain->find_node_for_effect(this);
        // We specifically do not want mipmaps on the input texture;
        // they break minification.
        Node *self = chain->find_node_for_effect(this);

diff --git a/resample_effect.frag b/resample_effect.frag
index d71b5f5128d60e8656b544c476a58b7ed634dfda..bce33ce369c769bd29b797def82d0e00a15b969a 100644 (file)
--- a/resample_effect.frag
+++ b/resample_effect.frag
@@ -8,6 +8,14 @@ uniform float PREFIX(sample_x_scale);
 uniform float PREFIX(sample_x_offset);
 uniform float PREFIX(slice_height);
 
 uniform float PREFIX(sample_x_offset);
 uniform float PREFIX(slice_height);
 

+// We put the fractional part of the offset (-0.5 to 0.5 pixels) in the weights
+// because we have to (otherwise they'd do nothing). However, the support texture
+// has limited numerical precision and we'd need as much of it as we can for
+// getting the subpixel sampling right, and adding a large constant to each value
+// will reduce the precision further. Thus, the non-fractional part of the offset
+// is sent in through a uniform that we simply add in the beginning of the shader.
+uniform float PREFIX(whole_pixel_offset);
+

 // Sample a single weight. First fetch information about where to sample
 // and the weight from sample_tex, and then read the pixel itself.
 vec4 PREFIX(do_sample)(vec2 tc, int i)
 // Sample a single weight. First fetch information about where to sample
 // and the weight from sample_tex, and then read the pixel itself.
 vec4 PREFIX(do_sample)(vec2 tc, int i)


@@ -30,6 +38,11 @@ vec4 PREFIX(do_sample)(vec2 tc, int i)
 }
 
 vec4 FUNCNAME(vec2 tc) {
 }
 
 vec4 FUNCNAME(vec2 tc) {

+#if DIRECTION_VERTICAL
+       tc.y += PREFIX(whole_pixel_offset);
+#else
+       tc.x += PREFIX(whole_pixel_offset);
+#endif

        vec4 sum = PREFIX(do_sample)(tc, 0);
        for (int i = 1; i < PREFIX(num_samples); ++i) {
                sum += PREFIX(do_sample)(tc, i);
        vec4 sum = PREFIX(do_sample)(tc, 0);
        for (int i = 1; i < PREFIX(num_samples); ++i) {
                sum += PREFIX(do_sample)(tc, i);

diff --git a/resample_effect.h b/resample_effect.h
index f0112b3ca7c4abcec1ba73a6ce420f65b10162dd..c52a973c18b055c2927896f9e585ea3de9bfa4d7 100644 (file)
--- a/resample_effect.h
+++ b/resample_effect.h
@@ -98,7 +98,9 @@ private:
        Direction direction;
        GLuint texnum;
        int input_width, input_height, output_width, output_height;
        Direction direction;
        GLuint texnum;
        int input_width, input_height, output_width, output_height;

+       float offset;

        int last_input_width, last_input_height, last_output_width, last_output_height;
        int last_input_width, last_input_height, last_output_width, last_output_height;

+       float last_offset;

        int src_bilinear_samples, num_loops;
        float slice_height;
 };
        int src_bilinear_samples, num_loops;
        float slice_height;
 };

diff --git a/resample_effect_test.cpp b/resample_effect_test.cpp
index 95c2bcfb9aec95693653ea65572cfdca76f79699..e4b9439ffffad9211c9b1c99d91e544e5e1a1fe0 100644 (file)
--- a/resample_effect_test.cpp
+++ b/resample_effect_test.cpp
@@ -208,4 +208,100 @@ TEST(ResampleEffectTest, HeavyResampleGetsSumRight) {
        expect_equal(expected_data, out_data, dwidth, dheight, 0.1 / 1023.0);
 }
 
        expect_equal(expected_data, out_data, dwidth, dheight, 0.1 / 1023.0);
 }
 

+TEST(ResampleEffectTest, ReadWholePixelFromLeft) {
+       const int size = 5;
+
+       float data[size * size] = {
+               0.0, 0.0, 0.0, 0.0, 0.0,
+               0.0, 0.0, 0.0, 0.0, 0.0,
+               0.0, 0.0, 1.0, 0.0, 0.0,
+               0.0, 0.0, 0.0, 0.0, 0.0,
+               0.0, 0.0, 0.0, 0.0, 0.0,
+       };
+       float expected_data[size * size] = {
+               0.0, 0.0, 0.0, 0.0, 0.0,
+               0.0, 0.0, 0.0, 0.0, 0.0,
+               0.0, 1.0, 0.0, 0.0, 0.0,
+               0.0, 0.0, 0.0, 0.0, 0.0,
+               0.0, 0.0, 0.0, 0.0, 0.0,
+       };
+       float out_data[size * size];
+
+       EffectChainTester tester(data, size, size, FORMAT_GRAYSCALE, COLORSPACE_sRGB, GAMMA_LINEAR);
+       Effect *resample_effect = tester.get_chain()->add_effect(new ResampleEffect());
+       ASSERT_TRUE(resample_effect->set_int("width", size));
+       ASSERT_TRUE(resample_effect->set_int("height", size));
+       ASSERT_TRUE(resample_effect->set_float("left", 1.0f));
+       tester.run(out_data, GL_RED, COLORSPACE_sRGB, GAMMA_LINEAR);
+
+       expect_equal(expected_data, out_data, size, size);
+}
+
+TEST(ResampleEffectTest, ReadQuarterPixelFromLeft) {
+       const int size = 5;
+
+       float data[size * size] = {
+               0.0, 0.0, 0.0, 0.0, 0.0,
+               0.0, 0.0, 0.0, 0.0, 0.0,
+               0.0, 0.0, 1.0, 0.0, 0.0,
+               0.0, 0.0, 0.0, 0.0, 0.0,
+               0.0, 0.0, 0.0, 0.0, 0.0,
+       };
+
+       float expected_data[size * size] = {
+               0.0, 0.0, 0.0, 0.0, 0.0,
+               0.0, 0.0, 0.0, 0.0, 0.0,
+
+               // sin(x*pi)/(x*pi) * sin(x*pi/3)/(x*pi/3) for
+               // x = -1.75, -0.75, 0.25, 1.25, 2.25.
+               // Note that the weight is mostly on the left side.
+               -0.06779, 0.27019, 0.89007, -0.13287, 0.03002,
+
+               0.0, 0.0, 0.0, 0.0, 0.0,
+               0.0, 0.0, 0.0, 0.0, 0.0,
+       };
+       float out_data[size * size];
+
+       EffectChainTester tester(data, size, size, FORMAT_GRAYSCALE, COLORSPACE_sRGB, GAMMA_LINEAR);
+       Effect *resample_effect = tester.get_chain()->add_effect(new ResampleEffect());
+       ASSERT_TRUE(resample_effect->set_int("width", size));
+       ASSERT_TRUE(resample_effect->set_int("height", size));
+       ASSERT_TRUE(resample_effect->set_float("left", 0.25f));
+       tester.run(out_data, GL_RED, COLORSPACE_sRGB, GAMMA_LINEAR);
+
+       expect_equal(expected_data, out_data, size, size);
+}
+
+TEST(ResampleEffectTest, ReadQuarterPixelFromTop) {
+       const int width = 3;
+       const int height = 5;
+
+       float data[width * height] = {
+               0.0, 0.0, 0.0,
+               0.0, 0.0, 0.0,
+               1.0, 0.0, 0.0,
+               0.0, 0.0, 0.0,
+               0.0, 0.0, 0.0,
+       };
+
+       // See ReadQuarterPixelFromLeft for explanation of the data.
+       float expected_data[width * height] = {
+               -0.06779, 0.0, 0.0,
+                0.27019, 0.0, 0.0,
+                0.89007, 0.0, 0.0,
+               -0.13287, 0.0, 0.0,
+                0.03002, 0.0, 0.0,
+       };
+       float out_data[width * height];
+
+       EffectChainTester tester(data, width, height, FORMAT_GRAYSCALE, COLORSPACE_sRGB, GAMMA_LINEAR);
+       Effect *resample_effect = tester.get_chain()->add_effect(new ResampleEffect());
+       ASSERT_TRUE(resample_effect->set_int("width", width));
+       ASSERT_TRUE(resample_effect->set_int("height", height));
+       ASSERT_TRUE(resample_effect->set_float("top", 0.25f));
+       tester.run(out_data, GL_RED, COLORSPACE_sRGB, GAMMA_LINEAR);
+
+       expect_equal(expected_data, out_data, width, height);
+}
+

 }  // namespace movit
 }  // namespace movit