Add proper formats for sRGB without alpha.

[movit] / resample_effect.cpp

diff --git a/resample_effect.cpp b/resample_effect.cpp
index 5b016861fafac36d0c55c4133323f1a6315b1846..50b7c6bc5c4730fc6bb25cd18c96a446c8f02986 100644 (file)
--- a/resample_effect.cpp
+++ b/resample_effect.cpp
@@ -1,13 +1,22 @@
 // Three-lobed Lanczos, the most common choice.
 #define LANCZOS_RADIUS 3.0
 
 // Three-lobed Lanczos, the most common choice.
 #define LANCZOS_RADIUS 3.0
 

-#include <math.h>
+#include <epoxy/gl.h>

 #include <assert.h>
 #include <assert.h>

+#include <limits.h>
+#include <math.h>
+#include <stdio.h>
+#include <algorithm>

 
 

-#include "resample_effect.h"

 #include "effect_chain.h"
 #include "effect_chain.h"

+#include "effect_util.h"
+#include "fp16.h"
+#include "resample_effect.h"

 #include "util.h"
 #include "util.h"

-#include "opengl.h"
+
+using namespace std;
+
+namespace movit {

 
 namespace {
 
 
 namespace {
 


@@ -40,6 +49,61 @@ unsigned gcd(unsigned a, unsigned b)
        return a;
 }
 
        return a;
 }
 

+unsigned combine_samples(float *src, float *dst, unsigned num_src_samples, unsigned max_samples_saved)
+{
+       unsigned num_samples_saved = 0;
+       for (unsigned i = 0, j = 0; i < num_src_samples; ++i, ++j) {
+               // Copy the sample directly; it will be overwritten later if we can combine.
+               if (dst != NULL) {
+                       dst[j * 2 + 0] = src[i * 2 + 0];
+                       dst[j * 2 + 1] = src[i * 2 + 1];
+               }
+
+               if (i == num_src_samples - 1) {
+                       // Last sample; cannot combine.
+                       continue;
+               }
+               assert(num_samples_saved <= max_samples_saved);
+               if (num_samples_saved == max_samples_saved) {
+                       // We could maybe save more here, but other rows can't, so don't bother.
+                       continue;
+               }
+
+               float w1 = src[i * 2 + 0];
+               float w2 = src[(i + 1) * 2 + 0];
+               if (w1 * w2 < 0.0f) {
+                       // Differing signs; cannot combine.
+                       continue;
+               }
+
+               float pos1 = src[i * 2 + 1];
+               float pos2 = src[(i + 1) * 2 + 1];
+               assert(pos2 > pos1);
+
+               float offset, total_weight, sum_sq_error;
+               combine_two_samples(w1, w2, &offset, &total_weight, &sum_sq_error);
+
+               // If the interpolation error is larger than that of about sqrt(2) of
+               // a level at 8-bit precision, don't combine. (You'd think 1.0 was enough,
+               // but since the artifacts are not really random, they can get quite
+               // visible. On the other hand, going to 0.25f, I can see no change at
+               // all with 8-bit output, so it would not seem to be worth it.)
+               if (sum_sq_error > 0.5f / (256.0f * 256.0f)) {
+                       continue;
+               }
+
+               // OK, we can combine this and the next sample.
+               if (dst != NULL) {
+                       dst[j * 2 + 0] = total_weight;
+                       dst[j * 2 + 1] = pos1 + offset * (pos2 - pos1);
+               }
+
+               ++i;  // Skip the next sample.
+               ++num_samples_saved;
+       }
+       return num_samples_saved;
+}
+

 }  // namespace
 
 ResampleEffect::ResampleEffect()
 }  // namespace
 
 ResampleEffect::ResampleEffect()


@@ -51,9 +115,9 @@ ResampleEffect::ResampleEffect()
 
        // The first blur pass will forward resolution information to us.
        hpass = new SingleResamplePassEffect(this);
 
        // The first blur pass will forward resolution information to us.
        hpass = new SingleResamplePassEffect(this);

-       hpass->set_int("direction", SingleResamplePassEffect::HORIZONTAL);
+       CHECK(hpass->set_int("direction", SingleResamplePassEffect::HORIZONTAL));

        vpass = new SingleResamplePassEffect(NULL);
        vpass = new SingleResamplePassEffect(NULL);

-       vpass->set_int("direction", SingleResamplePassEffect::VERTICAL);
+       CHECK(vpass->set_int("direction", SingleResamplePassEffect::VERTICAL));

 
        update_size();
 }
 
        update_size();
 }


@@ -79,7 +143,7 @@ void ResampleEffect::inform_input_size(unsigned input_num, unsigned width, unsig
        input_height = height;
        update_size();
 }
        input_height = height;
        update_size();
 }

-               
+

 void ResampleEffect::update_size()
 {
        bool ok = true;
 void ResampleEffect::update_size()
 {
        bool ok = true;


@@ -96,7 +160,7 @@ void ResampleEffect::update_size()
        assert(ok);
 }
 
        assert(ok);
 }
 

-bool ResampleEffect::set_float(const std::string &key, float value) {
+bool ResampleEffect::set_float(const string &key, float value) {

        if (key == "width") {
                output_width = value;
                update_size();
        if (key == "width") {
                output_width = value;
                update_size();


@@ -134,7 +198,7 @@ SingleResamplePassEffect::~SingleResamplePassEffect()
        glDeleteTextures(1, &texnum);
 }
 
        glDeleteTextures(1, &texnum);
 }
 

-std::string SingleResamplePassEffect::output_fragment_shader()
+string SingleResamplePassEffect::output_fragment_shader()

 {
        char buf[256];
        sprintf(buf, "#define DIRECTION_VERTICAL %d\n", (direction == VERTICAL));
 {
        char buf[256];
        sprintf(buf, "#define DIRECTION_VERTICAL %d\n", (direction == VERTICAL));


@@ -152,10 +216,8 @@ std::string SingleResamplePassEffect::output_fragment_shader()
 // so out[0] will read from parameters <x,y> = <0,0>, <1,0>, <2,0> and so on.
 //
 // For horizontal scaling, we fill in the exact same texture;
 // so out[0] will read from parameters <x,y> = <0,0>, <1,0>, <2,0> and so on.
 //
 // For horizontal scaling, we fill in the exact same texture;

-// the shader just interprets is differently.
-//
-// TODO: Support optimization using free linear sampling, like in BlurEffect.
-void SingleResamplePassEffect::update_texture(GLuint glsl_program_num, const std::string &prefix, unsigned *sampler_num)
+// the shader just interprets it differently.
+void SingleResamplePassEffect::update_texture(GLuint glsl_program_num, const string &prefix, unsigned *sampler_num)

 {
        unsigned src_size, dst_size;
        if (direction == SingleResamplePassEffect::HORIZONTAL) {
 {
        unsigned src_size, dst_size;
        if (direction == SingleResamplePassEffect::HORIZONTAL) {


@@ -170,7 +232,6 @@ void SingleResamplePassEffect::update_texture(GLuint glsl_program_num, const std
                assert(false);
        }
 
                assert(false);
        }
 

-

        // For many resamplings (e.g. 640 -> 1280), we will end up with the same
        // set of samples over and over again in a loop. Thus, we can compute only
        // the first such loop, and then ask the card to repeat the texture for us.
        // For many resamplings (e.g. 640 -> 1280), we will end up with the same
        // set of samples over and over again in a loop. Thus, we can compute only
        // the first such loop, and then ask the card to repeat the texture for us.


@@ -229,9 +290,9 @@ void SingleResamplePassEffect::update_texture(GLuint glsl_program_num, const std
        // Anyhow, in this case we clearly need to look at more source pixels
        // to compute the destination pixel, and how many depend on the scaling factor.
        // Thus, the kernel width will vary with how much we scale.
        // Anyhow, in this case we clearly need to look at more source pixels
        // to compute the destination pixel, and how many depend on the scaling factor.
        // Thus, the kernel width will vary with how much we scale.

-       float radius_scaling_factor = std::min(float(dst_size) / float(src_size), 1.0f);
+       float radius_scaling_factor = min(float(dst_size) / float(src_size), 1.0f);

        int int_radius = lrintf(LANCZOS_RADIUS / radius_scaling_factor);
        int int_radius = lrintf(LANCZOS_RADIUS / radius_scaling_factor);

-       src_samples = int_radius * 2 + 1;
+       int src_samples = int_radius * 2 + 1;

        float *weights = new float[dst_samples * src_samples * 2];
        for (unsigned y = 0; y < dst_samples; ++y) {
                // Find the point around which we want to sample the source image,
        float *weights = new float[dst_samples * src_samples * 2];
        for (unsigned y = 0; y < dst_samples; ++y) {
                // Find the point around which we want to sample the source image,


@@ -246,6 +307,54 @@ void SingleResamplePassEffect::update_texture(GLuint glsl_program_num, const std
                        weights[(y * src_samples + i) * 2 + 0] = weight * radius_scaling_factor;
                        weights[(y * src_samples + i) * 2 + 1] = (src_y + 0.5) / float(src_size);
                }
                        weights[(y * src_samples + i) * 2 + 0] = weight * radius_scaling_factor;
                        weights[(y * src_samples + i) * 2 + 1] = (src_y + 0.5) / float(src_size);
                }

+
+       }
+
+       // Now make use of the bilinear filtering in the GPU to reduce the number of samples
+       // we need to make. This is a bit more complex than BlurEffect since we cannot combine
+       // two neighboring samples if their weights have differing signs, so we first need to
+       // figure out the maximum number of samples. Then, we downconvert all the weights to
+       // that number -- we could have gone for a variable-length system, but this is simpler,
+       // and the gains would probably be offset by the extra cost of checking when to stop.
+       //
+       // The greedy strategy for combining samples is optimal.
+       src_bilinear_samples = 0;
+       for (unsigned y = 0; y < dst_samples; ++y) {
+               unsigned num_samples_saved = combine_samples(weights + (y * src_samples) * 2, NULL, src_samples, UINT_MAX);
+               src_bilinear_samples = max<int>(src_bilinear_samples, src_samples - num_samples_saved);
+       }
+
+       // Now that we know the right width, actually combine the samples.
+       float *bilinear_weights = new float[dst_samples * src_bilinear_samples * 2];
+       fp16_int_t *bilinear_weights_fp16 = new fp16_int_t[dst_samples * src_bilinear_samples * 2];
+       for (unsigned y = 0; y < dst_samples; ++y) {
+               float *bilinear_weights_ptr = bilinear_weights + (y * src_bilinear_samples) * 2;
+               fp16_int_t *bilinear_weights_fp16_ptr = bilinear_weights_fp16 + (y * src_bilinear_samples) * 2;
+               unsigned num_samples_saved = combine_samples(
+                       weights + (y * src_samples) * 2,
+                       bilinear_weights_ptr,
+                       src_samples,
+                       src_samples - src_bilinear_samples);
+               assert(int(src_samples) - int(num_samples_saved) == src_bilinear_samples);
+
+               // Convert to fp16.
+               for (int i = 0; i < src_bilinear_samples; ++i) {
+                       bilinear_weights_fp16_ptr[i * 2 + 0] = fp64_to_fp16(bilinear_weights_ptr[i * 2 + 0]);
+                       bilinear_weights_fp16_ptr[i * 2 + 1] = fp64_to_fp16(bilinear_weights_ptr[i * 2 + 1]);
+               }
+
+               // Normalize so that the sum becomes one. Note that we do it twice;
+               // this sometimes helps a tiny little bit when we have many samples.
+               for (int normalize_pass = 0; normalize_pass < 2; ++normalize_pass) {
+                       double sum = 0.0;
+                       for (int i = 0; i < src_bilinear_samples; ++i) {
+                               sum += fp16_to_fp64(bilinear_weights_fp16_ptr[i * 2 + 0]);
+                       }
+                       for (int i = 0; i < src_bilinear_samples; ++i) {
+                               bilinear_weights_fp16_ptr[i * 2 + 0] = fp64_to_fp16(
+                                       fp16_to_fp64(bilinear_weights_fp16_ptr[i * 2 + 0]) / sum);
+                       }
+               }

        }
 
        // Encode as a two-component texture. Note the GL_REPEAT.
        }
 
        // Encode as a two-component texture. Note the GL_REPEAT.


@@ -259,16 +368,23 @@ void SingleResamplePassEffect::update_texture(GLuint glsl_program_num, const std
        check_error();
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT);
        check_error();
        check_error();
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT);
        check_error();

-       glTexImage2D(GL_TEXTURE_2D, 0, GL_RG16F, src_samples, dst_samples, 0, GL_RG, GL_FLOAT, weights);
+       glTexImage2D(GL_TEXTURE_2D, 0, GL_RG16F, src_bilinear_samples, dst_samples, 0, GL_RG, GL_HALF_FLOAT, bilinear_weights_fp16);

        check_error();
 
        delete[] weights;
        check_error();
 
        delete[] weights;

+       delete[] bilinear_weights;
+       delete[] bilinear_weights_fp16;

 }
 
 }
 

-void SingleResamplePassEffect::set_gl_state(GLuint glsl_program_num, const std::string &prefix, unsigned *sampler_num)
+void SingleResamplePassEffect::set_gl_state(GLuint glsl_program_num, const string &prefix, unsigned *sampler_num)

 {
        Effect::set_gl_state(glsl_program_num, prefix, sampler_num);
 
 {
        Effect::set_gl_state(glsl_program_num, prefix, sampler_num);
 

+       assert(input_width > 0);
+       assert(input_height > 0);
+       assert(output_width > 0);
+       assert(output_height > 0);
+

        if (input_width != last_input_width ||
            input_height != last_input_height ||
            output_width != last_output_width ||
        if (input_width != last_input_width ||
            input_height != last_input_height ||
            output_width != last_output_width ||


@@ -287,18 +403,21 @@ void SingleResamplePassEffect::set_gl_state(GLuint glsl_program_num, const std::
 
        set_uniform_int(glsl_program_num, prefix, "sample_tex", *sampler_num);
        ++sampler_num;
 
        set_uniform_int(glsl_program_num, prefix, "sample_tex", *sampler_num);
        ++sampler_num;

-       set_uniform_int(glsl_program_num, prefix, "num_samples", src_samples);
+       set_uniform_int(glsl_program_num, prefix, "num_samples", src_bilinear_samples);

        set_uniform_float(glsl_program_num, prefix, "num_loops", num_loops);
        set_uniform_float(glsl_program_num, prefix, "slice_height", slice_height);
 
        // Instructions for how to convert integer sample numbers to positions in the weight texture.
        set_uniform_float(glsl_program_num, prefix, "num_loops", num_loops);
        set_uniform_float(glsl_program_num, prefix, "slice_height", slice_height);
 
        // Instructions for how to convert integer sample numbers to positions in the weight texture.

-       set_uniform_float(glsl_program_num, prefix, "sample_x_scale", 1.0f / src_samples);
-       set_uniform_float(glsl_program_num, prefix, "sample_x_offset", 0.5f / src_samples);
+       set_uniform_float(glsl_program_num, prefix, "sample_x_scale", 1.0f / src_bilinear_samples);
+       set_uniform_float(glsl_program_num, prefix, "sample_x_offset", 0.5f / src_bilinear_samples);

 
        // We specifically do not want mipmaps on the input texture;
        // they break minification.
 
        // We specifically do not want mipmaps on the input texture;
        // they break minification.

-       glActiveTexture(GL_TEXTURE0);
+       Node *self = chain->find_node_for_effect(this);
+       glActiveTexture(chain->get_input_sampler(self, 0));

        check_error();
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
        check_error();
 }
        check_error();
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
        check_error();
 }

+
+}  // namespace movit