From: Steinar H. Gunderson <sgunderson@bigfoot.com>
Date: Sat, 7 Jul 2018 11:41:13 +0000 (+0200)
Subject: Do patch normalization in densification.
X-Git-Tag: 1.8.0~76^2~229
X-Git-Url: https://git.sesse.net/?a=commitdiff_plain;h=1ed10162dcc208c0f95da7f203f51be6ccea0900;p=nageru

Do patch normalization in densification.
---

diff --git a/densify.frag b/densify.frag
index d16d630..f9c67ec 100644
--- a/densify.frag
+++ b/densify.frag
@@ -2,6 +2,7 @@
 
 in vec2 image_pos;
 flat in vec2 flow_du;
+flat in float mean_diff;
 out vec3 flow_contribution;
 
 uniform sampler2D image0_tex, image1_tex;
@@ -16,6 +17,7 @@ void main()
 	// And it says L2 norm, but really, the code does absolute value even for
 	// L2 error norm (it uses a square root formula for L1 norm).
 	float diff = texture(image0_tex, image_pos).x - texture(image1_tex, image_pos + flow_du).x;
+	diff -= mean_diff;
 	float weight = 1.0 / max(abs(diff), 2.0 / 255.0);
 	flow_contribution = vec3(flow_du.x * weight, flow_du.y * weight, weight);
 }
diff --git a/densify.vert b/densify.vert
index 0506b9c..f32b9d4 100644
--- a/densify.vert
+++ b/densify.vert
@@ -3,6 +3,7 @@
 in vec2 position;
 out vec2 image_pos;
 flat out vec2 flow_du;
+flat out float mean_diff;
 
 uniform int width_patches;
 uniform vec2 patch_size;  // In 0..1 coordinates.
@@ -28,7 +29,9 @@ void main()
 	image_pos = patch_spacing * ivec2(patch_x, patch_y) + patch_size * grown_pos;
 
 	// Find the flow value for this patch, and send it on to the fragment shader.
-	flow_du = texelFetch(flow_tex, ivec2(patch_x, patch_y), 0).xy;
+	vec3 flow_du_and_mean_diff = texelFetch(flow_tex, ivec2(patch_x, patch_y), 0).xyz;
+	flow_du = flow_du_and_mean_diff.xy;
+	mean_diff = flow_du_and_mean_diff.z;
 
 	// The result of glOrtho(0.0, 1.0, 0.0, 1.0, 0.0, 1.0) is:
 	//
diff --git a/flow.cpp b/flow.cpp
index fa0ab38..5673ef6 100644
--- a/flow.cpp
+++ b/flow.cpp
@@ -500,7 +500,7 @@ int main(void)
 		// Create an output flow texture.
 		GLuint flow_out_tex;
 		glCreateTextures(GL_TEXTURE_2D, 1, &flow_out_tex);
-		glTextureStorage2D(flow_out_tex, 1, GL_RG16F, width_patches, height_patches);
+		glTextureStorage2D(flow_out_tex, 1, GL_RGB16F, width_patches, height_patches);
 
 		// And draw.
 		motion_search.exec(tex0_view, tex1_view, grad0_tex, prev_level_flow_tex, flow_out_tex, level_width, level_height, width_patches, height_patches);
diff --git a/motion_search.frag b/motion_search.frag
index 927ecc5..7690389 100644
--- a/motion_search.frag
+++ b/motion_search.frag
@@ -40,7 +40,7 @@ const uint num_iterations = 16;
 
 in vec2 flow_tc;
 in vec2 patch_bottom_left_texel;  // Center of bottom-left texel of patch.
-out vec2 out_flow;
+out vec3 out_flow;
 
 uniform sampler2D flow_tex, grad0_tex, image0_tex, image1_tex;
 uniform vec2 image_size, inv_image_size;
@@ -105,6 +105,7 @@ void main()
 	// generally come out in pixels since the gradient is in pixels,
 	// so we need to convert at the end.
 	vec2 u = initial_u;
+	float mean_diff, first_mean_diff;
 
 	for (uint i = 0; i < num_iterations; ++i) {
 		vec2 du = vec2(0.0, 0.0);
@@ -131,7 +132,12 @@ void main()
 		//   sum(S^T * (x - y)) = [what we calculated] - (µ1 - µ2) sum(S^T)
 		//
 		// so we can just subtract away the mean difference here.
-		du -= grad_sum * (warped_sum - template_sum) * (1.0 / (patch_size * patch_size));
+		mean_diff = (warped_sum - template_sum) * (1.0 / (patch_size * patch_size));
+		du -= grad_sum * mean_diff;
+
+		if (i == 0) {
+			first_mean_diff = mean_diff;
+		}
 
 		// Do the actual update.
 		u -= (H_inv * du) * inv_image_size;
@@ -146,7 +152,10 @@ void main()
 	     u.y * image_size.y < -(patch_size * 0.5f) ||
 	     (1.0 - u.y) * image_size.y < -(patch_size * 0.5f)) {
 		u = initial_u;
+		mean_diff = first_mean_diff;
 	}
 
-	out_flow = u;
+	// NOTE: The mean patch diff will be for the second-to-last patch,
+	// not the true position of du. But hopefully, it will be very close.
+	out_flow = vec3(u.x, u.y, mean_diff);
 }