Do patch normalization in densification.

diff --git a/densify.frag b/densify.frag
index d16d6304a43b6a9475f6f34be8fe4d6479482d11..f9c67ecea6c1b8000d63248de0976833813d395e 100644 (file)
--- a/densify.frag
+++ b/densify.frag
@@ -2,6 +2,7 @@
 
 in vec2 image_pos;
 flat in vec2 flow_du;
+flat in float mean_diff;
 out vec3 flow_contribution;
 
 uniform sampler2D image0_tex, image1_tex;
@@ -16,6 +17,7 @@ void main()
        // And it says L2 norm, but really, the code does absolute value even for
        // L2 error norm (it uses a square root formula for L1 norm).
        float diff = texture(image0_tex, image_pos).x - texture(image1_tex, image_pos + flow_du).x;
+       diff -= mean_diff;
        float weight = 1.0 / max(abs(diff), 2.0 / 255.0);
        flow_contribution = vec3(flow_du.x * weight, flow_du.y * weight, weight);
 }

diff --git a/densify.vert b/densify.vert
index 0506b9caf7ee000eb4616611b982fac474003451..f32b9d4151cbe3a4f587c2a6cf5225be426c7673 100644 (file)
--- a/densify.vert
+++ b/densify.vert
@@ -3,6 +3,7 @@
 in vec2 position;
 out vec2 image_pos;
 flat out vec2 flow_du;
+flat out float mean_diff;
 
 uniform int width_patches;
 uniform vec2 patch_size;  // In 0..1 coordinates.
@@ -28,7 +29,9 @@ void main()
        image_pos = patch_spacing * ivec2(patch_x, patch_y) + patch_size * grown_pos;
 
        // Find the flow value for this patch, and send it on to the fragment shader.
-       flow_du = texelFetch(flow_tex, ivec2(patch_x, patch_y), 0).xy;
+       vec3 flow_du_and_mean_diff = texelFetch(flow_tex, ivec2(patch_x, patch_y), 0).xyz;
+       flow_du = flow_du_and_mean_diff.xy;
+       mean_diff = flow_du_and_mean_diff.z;
 
        // The result of glOrtho(0.0, 1.0, 0.0, 1.0, 0.0, 1.0) is:
        //

diff --git a/flow.cpp b/flow.cpp
index fa0ab38911ee8ecb63566570710b8a26f3111dac..5673ef6f977842c57377983f58027041b44fa796 100644 (file)
--- a/flow.cpp
+++ b/flow.cpp
@@ -500,7 +500,7 @@ int main(void)
                // Create an output flow texture.
                GLuint flow_out_tex;
                glCreateTextures(GL_TEXTURE_2D, 1, &flow_out_tex);
-               glTextureStorage2D(flow_out_tex, 1, GL_RG16F, width_patches, height_patches);
+               glTextureStorage2D(flow_out_tex, 1, GL_RGB16F, width_patches, height_patches);
 
                // And draw.
                motion_search.exec(tex0_view, tex1_view, grad0_tex, prev_level_flow_tex, flow_out_tex, level_width, level_height, width_patches, height_patches);

diff --git a/motion_search.frag b/motion_search.frag
index 927ecc5634ef10488925bd4f5695cff2f02a614e..769038977bd97b5724e70716e8e4770dc33c7f9d 100644 (file)
--- a/motion_search.frag
+++ b/motion_search.frag
@@ -40,7 +40,7 @@ const uint num_iterations = 16;
 
 in vec2 flow_tc;
 in vec2 patch_bottom_left_texel;  // Center of bottom-left texel of patch.
-out vec2 out_flow;
+out vec3 out_flow;
 
 uniform sampler2D flow_tex, grad0_tex, image0_tex, image1_tex;
 uniform vec2 image_size, inv_image_size;
@@ -105,6 +105,7 @@ void main()
        // generally come out in pixels since the gradient is in pixels,
        // so we need to convert at the end.
        vec2 u = initial_u;
+       float mean_diff, first_mean_diff;
 
        for (uint i = 0; i < num_iterations; ++i) {
                vec2 du = vec2(0.0, 0.0);
@@ -131,7 +132,12 @@ void main()
                //   sum(S^T * (x - y)) = [what we calculated] - (µ1 - µ2) sum(S^T)
                //
                // so we can just subtract away the mean difference here.
-               du -= grad_sum * (warped_sum - template_sum) * (1.0 / (patch_size * patch_size));
+               mean_diff = (warped_sum - template_sum) * (1.0 / (patch_size * patch_size));
+               du -= grad_sum * mean_diff;
+
+               if (i == 0) {
+                       first_mean_diff = mean_diff;
+               }
 
                // Do the actual update.
                u -= (H_inv * du) * inv_image_size;
@@ -146,7 +152,10 @@ void main()
             u.y * image_size.y < -(patch_size * 0.5f) ||
             (1.0 - u.y) * image_size.y < -(patch_size * 0.5f)) {
                u = initial_u;
+               mean_diff = first_mean_diff;
        }
 
-       out_flow = u;
+       // NOTE: The mean patch diff will be for the second-to-last patch,
+       // not the true position of du. But hopefully, it will be very close.
+       out_flow = vec3(u.x, u.y, mean_diff);
 }