From f22fffe30b28081f42fc4d5f55c6a6a95fe65c54 Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Fri, 20 Jul 2018 21:52:41 +0200 Subject: [PATCH] In the variational refinement, change the flow unit from normalized coordinates to pixels. --- equations.frag | 14 +++------- flow.cpp | 67 ++++++++++++++++++++++++++++++---------------- motion_search.frag | 4 +-- prewarp.frag | 3 +++ smoothness.frag | 10 +------ 5 files changed, 53 insertions(+), 45 deletions(-) diff --git a/equations.frag b/equations.frag index 3926351..6171a37 100644 --- a/equations.frag +++ b/equations.frag @@ -11,24 +11,16 @@ uniform sampler2D smoothness_x_tex, smoothness_y_tex; // TODO: Consider a specialized version for the case where we know that du = dv = 0, // since we run so few iterations. -// The base flow needs to be normalized. -// TODO: Should we perhaps reduce this to a separate two-component -// texture when calculating the derivatives? -vec2 normalize_flow(vec3 flow) -{ - return flow.xy / flow.z; -} - // This must be a macro, since the offset needs to be a constant expression. #define get_flow(x_offs, y_offs) \ - (normalize_flow(textureOffset(flow_tex, tc, ivec2((x_offs), (y_offs))).xyz) + \ + (textureOffset(flow_tex, tc, ivec2((x_offs), (y_offs))).xy + \ textureOffset(diff_flow_tex, tc, ivec2((x_offs), (y_offs))).xy) void main() { // Read the flow (on top of the u0/v0 flow). vec2 diff_flow = texture(diff_flow_tex, tc).xy; - float du = diff_flow.x; // FIXME: convert to pixels? + float du = diff_flow.x; float dv = diff_flow.y; // Read the first derivatives. @@ -115,7 +107,7 @@ void main() // The central term of the Laplacian, for (u0, v0) only. // (The central term for (du, dv) is what we are solving for.) - vec2 central = (smooth_l + smooth_r + smooth_d + smooth_u) * normalize_flow(texture(flow_tex, tc).xyz); + vec2 central = (smooth_l + smooth_r + smooth_d + smooth_u) * texture(flow_tex, tc).xy; b1 += central.x; b2 += central.y; diff --git a/flow.cpp b/flow.cpp index 1946569..3c69089 100644 --- a/flow.cpp +++ b/flow.cpp @@ -268,7 +268,7 @@ void Sobel::exec(GLint tex0_view, GLint grad0_tex, int level_width, int level_he class MotionSearch { public: MotionSearch(); - void exec(GLuint tex0_view, GLuint tex1_view, GLuint grad0_tex, GLuint flow_tex, GLuint flow_out_tex, int level_width, int level_height, int width_patches, int height_patches); + void exec(GLuint tex0_view, GLuint tex1_view, GLuint grad0_tex, GLuint flow_tex, GLuint flow_out_tex, int level_width, int level_height, int prev_level_width, int prev_level_height, int width_patches, int height_patches); private: GLuint motion_vs_obj; @@ -276,7 +276,7 @@ private: GLuint motion_search_program; GLuint motion_search_vao; - GLuint uniform_image_size, uniform_inv_image_size; + GLuint uniform_image_size, uniform_inv_image_size, uniform_inv_prev_level_size; GLuint uniform_image0_tex, uniform_image1_tex, uniform_grad0_tex, uniform_flow_tex; }; @@ -297,13 +297,14 @@ MotionSearch::MotionSearch() uniform_image_size = glGetUniformLocation(motion_search_program, "image_size"); uniform_inv_image_size = glGetUniformLocation(motion_search_program, "inv_image_size"); + uniform_inv_prev_level_size = glGetUniformLocation(motion_search_program, "inv_prev_level_size"); uniform_image0_tex = glGetUniformLocation(motion_search_program, "image0_tex"); uniform_image1_tex = glGetUniformLocation(motion_search_program, "image1_tex"); uniform_grad0_tex = glGetUniformLocation(motion_search_program, "grad0_tex"); uniform_flow_tex = glGetUniformLocation(motion_search_program, "flow_tex"); } -void MotionSearch::exec(GLuint tex0_view, GLuint tex1_view, GLuint grad0_tex, GLuint flow_tex, GLuint flow_out_tex, int level_width, int level_height, int width_patches, int height_patches) +void MotionSearch::exec(GLuint tex0_view, GLuint tex1_view, GLuint grad0_tex, GLuint flow_tex, GLuint flow_out_tex, int level_width, int level_height, int prev_level_width, int prev_level_height, int width_patches, int height_patches) { glUseProgram(motion_search_program); @@ -314,6 +315,7 @@ void MotionSearch::exec(GLuint tex0_view, GLuint tex1_view, GLuint grad0_tex, GL glProgramUniform2f(motion_search_program, uniform_image_size, level_width, level_height); glProgramUniform2f(motion_search_program, uniform_inv_image_size, 1.0f / level_width, 1.0f / level_height); + glProgramUniform2f(motion_search_program, uniform_inv_prev_level_size, 1.0f / prev_level_width, 1.0f / prev_level_height); GLuint flow_fbo; // TODO: cleanup glCreateFramebuffers(1, &flow_fbo); @@ -409,11 +411,14 @@ void Densify::exec(GLuint tex0_view, GLuint tex1_view, GLuint flow_tex, GLuint d // I_0 and I_w. The prewarping is what enables us to solve the variational // flow for du,dv instead of u,v. // +// Also calculates the normalized flow, ie. divides by z (this is needed because +// Densify works by additive blending) and multiplies by the image size. +// // See variational_refinement.txt for more information. class Prewarp { public: Prewarp(); - void exec(GLuint tex0_view, GLuint tex1_view, GLuint flow_tex, GLuint I_tex, GLuint I_t_tex, int level_width, int level_height); + void exec(GLuint tex0_view, GLuint tex1_view, GLuint flow_tex, GLuint normalized_flow_tex, GLuint I_tex, GLuint I_t_tex, int level_width, int level_height); private: GLuint prewarp_vs_obj; @@ -422,6 +427,7 @@ private: GLuint prewarp_vao; GLuint uniform_image0_tex, uniform_image1_tex, uniform_flow_tex; + GLuint uniform_image_size; }; Prewarp::Prewarp() @@ -442,9 +448,11 @@ Prewarp::Prewarp() uniform_image0_tex = glGetUniformLocation(prewarp_program, "image0_tex"); uniform_image1_tex = glGetUniformLocation(prewarp_program, "image1_tex"); uniform_flow_tex = glGetUniformLocation(prewarp_program, "flow_tex"); + + uniform_image_size = glGetUniformLocation(prewarp_program, "image_size"); } -void Prewarp::exec(GLuint tex0_view, GLuint tex1_view, GLuint flow_tex, GLuint I_tex, GLuint I_t_tex, int level_width, int level_height) +void Prewarp::exec(GLuint tex0_view, GLuint tex1_view, GLuint flow_tex, GLuint I_tex, GLuint I_t_tex, GLuint normalized_flow_tex, int level_width, int level_height) { glUseProgram(prewarp_program); @@ -452,12 +460,15 @@ void Prewarp::exec(GLuint tex0_view, GLuint tex1_view, GLuint flow_tex, GLuint I bind_sampler(prewarp_program, uniform_image1_tex, 1, tex1_view, linear_sampler); bind_sampler(prewarp_program, uniform_flow_tex, 2, flow_tex, nearest_sampler); + glProgramUniform2f(prewarp_program, uniform_image_size, level_width, level_height); + GLuint prewarp_fbo; // TODO: cleanup glCreateFramebuffers(1, &prewarp_fbo); - GLenum bufs[] = { GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1 }; - glNamedFramebufferDrawBuffers(prewarp_fbo, 2, bufs); + GLenum bufs[] = { GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1, GL_COLOR_ATTACHMENT2 }; + glNamedFramebufferDrawBuffers(prewarp_fbo, 3, bufs); glNamedFramebufferTexture(prewarp_fbo, GL_COLOR_ATTACHMENT0, I_tex, 0); glNamedFramebufferTexture(prewarp_fbo, GL_COLOR_ATTACHMENT1, I_t_tex, 0); + glNamedFramebufferTexture(prewarp_fbo, GL_COLOR_ATTACHMENT2, normalized_flow_tex, 0); glViewport(0, 0, level_width, level_height); glDisable(GL_BLEND); @@ -625,7 +636,6 @@ private: GLuint uniform_diff_flow_tex, uniform_flow_tex; GLuint uniform_beta_0_tex; GLuint uniform_smoothness_x_tex, uniform_smoothness_y_tex; - }; SetupEquations::SetupEquations() @@ -746,7 +756,7 @@ void SOR::exec(GLuint diff_flow_tex, GLuint equation_tex, GLuint smoothness_x_te } // Simply add the differential flow found by the variational refinement to the base flow. -// The output is in diff_flow_tex; we don't need to make a new texture. +// The output is in base_flow_tex; we don't need to make a new texture. class AddBaseFlow { public: AddBaseFlow(); @@ -758,7 +768,7 @@ private: GLuint add_flow_program; GLuint add_flow_vao; - GLuint uniform_base_flow_tex; + GLuint uniform_diff_flow_tex; }; AddBaseFlow::AddBaseFlow() @@ -776,18 +786,18 @@ AddBaseFlow::AddBaseFlow() glEnableVertexArrayAttrib(add_flow_vao, position_attrib); glVertexAttribPointer(position_attrib, 2, GL_FLOAT, GL_FALSE, 0, BUFFER_OFFSET(0)); - uniform_base_flow_tex = glGetUniformLocation(add_flow_program, "base_flow_tex"); + uniform_diff_flow_tex = glGetUniformLocation(add_flow_program, "diff_flow_tex"); } void AddBaseFlow::exec(GLuint base_flow_tex, GLuint diff_flow_tex, int level_width, int level_height) { glUseProgram(add_flow_program); - bind_sampler(add_flow_program, uniform_base_flow_tex, 0, base_flow_tex, nearest_sampler); + bind_sampler(add_flow_program, uniform_diff_flow_tex, 0, diff_flow_tex, nearest_sampler); GLuint add_flow_fbo; // TODO: cleanup glCreateFramebuffers(1, &add_flow_fbo); - glNamedFramebufferTexture(add_flow_fbo, GL_COLOR_ATTACHMENT0, diff_flow_tex, 0); + glNamedFramebufferTexture(add_flow_fbo, GL_COLOR_ATTACHMENT0, base_flow_tex, 0); glViewport(0, 0, level_width, level_height); glEnable(GL_BLEND); @@ -942,6 +952,7 @@ int main(void) GLuint initial_flow_tex; glCreateTextures(GL_TEXTURE_2D, 1, &initial_flow_tex); glTextureStorage2D(initial_flow_tex, 1, GL_RG16F, 1, 1); + int prev_level_width = 1, prev_level_height = 1; GLuint prev_level_flow_tex = initial_flow_tex; @@ -1006,7 +1017,7 @@ int main(void) // And draw. { ScopedTimer timer("Motion search", &level_timer); - motion_search.exec(tex0_view, tex1_view, grad0_tex, prev_level_flow_tex, flow_out_tex, level_width, level_height, width_patches, height_patches); + motion_search.exec(tex0_view, tex1_view, grad0_tex, prev_level_flow_tex, flow_out_tex, level_width, level_height, prev_level_width, prev_level_height, width_patches, height_patches); } // Densification. @@ -1025,15 +1036,23 @@ int main(void) // Everything below here in the loop belongs to variational refinement. ScopedTimer varref_timer("Variational refinement", &level_timer); - // Prewarping; create I and I_t. - GLuint I_tex, I_t_tex; + // Prewarping; create I and I_t, and a normalized base flow (so we don't + // have to normalize it over and over again, and also save some bandwidth). + // + // During the entire rest of the variational refinement, flow will be measured + // in pixels, not 0..1 normalized OpenGL texture coordinates. + // This is because variational refinement depends so heavily on derivatives, + // which are measured in intensity levels per pixel. + GLuint I_tex, I_t_tex, base_flow_tex; glCreateTextures(GL_TEXTURE_2D, 1, &I_tex); glCreateTextures(GL_TEXTURE_2D, 1, &I_t_tex); + glCreateTextures(GL_TEXTURE_2D, 1, &base_flow_tex); glTextureStorage2D(I_tex, 1, GL_R16F, level_width, level_height); glTextureStorage2D(I_t_tex, 1, GL_R16F, level_width, level_height); + glTextureStorage2D(base_flow_tex, 1, GL_RG16F, level_width, level_height); { ScopedTimer timer("Prewarping", &varref_timer); - prewarp.exec(tex0_view, tex1_view, dense_flow_tex, I_tex, I_t_tex, level_width, level_height); + prewarp.exec(tex0_view, tex1_view, dense_flow_tex, I_tex, I_t_tex, base_flow_tex, level_width, level_height); } // Calculate I_x and I_y. We're only calculating first derivatives; @@ -1075,13 +1094,13 @@ int main(void) // both in x and y direction. { ScopedTimer timer("Compute smoothness", &varref_timer); - compute_smoothness.exec(dense_flow_tex, du_dv_tex, smoothness_x_tex, smoothness_y_tex, level_width, level_height); + compute_smoothness.exec(base_flow_tex, du_dv_tex, smoothness_x_tex, smoothness_y_tex, level_width, level_height); } // Set up the 2x2 equation system for each pixel. { ScopedTimer timer("Set up equations", &varref_timer); - setup_equations.exec(I_x_y_tex, I_t_tex, du_dv_tex, dense_flow_tex, beta_0_tex, smoothness_x_tex, smoothness_y_tex, equation_tex, level_width, level_height); + setup_equations.exec(I_x_y_tex, I_t_tex, du_dv_tex, base_flow_tex, beta_0_tex, smoothness_x_tex, smoothness_y_tex, equation_tex, level_width, level_height); } // Run a few SOR (or quasi-SOR, since we're not really Jacobi) iterations. @@ -1095,12 +1114,15 @@ int main(void) // Add the differential flow found by the variational refinement to the base flow, // giving the final flow estimate for this level. // The output is in diff_flow_tex; we don't need to make a new texture. + // You can comment out this prat if you wish to test disabling of the variational refinement. { ScopedTimer timer("Add differential flow", &varref_timer); - add_base_flow.exec(dense_flow_tex, du_dv_tex, level_width, level_height); + add_base_flow.exec(base_flow_tex, du_dv_tex, level_width, level_height); } - prev_level_flow_tex = du_dv_tex; + prev_level_flow_tex = base_flow_tex; + prev_level_width = level_width; + prev_level_height = level_height; } total_timer.end(); @@ -1123,8 +1145,7 @@ int main(void) float du = dense_flow[(yy * level_width + x) * 2 + 0]; float dv = dense_flow[(yy * level_width + x) * 2 + 1]; - du = du * level_width; - dv = -dv * level_height; + dv = -dv; fwrite(&du, 4, 1, flowfp); fwrite(&dv, 4, 1, flowfp); diff --git a/motion_search.frag b/motion_search.frag index 188f5c3..163db2c 100644 --- a/motion_search.frag +++ b/motion_search.frag @@ -43,7 +43,7 @@ in vec2 patch_bottom_left_texel; // Center of bottom-left texel of patch. out vec3 out_flow; uniform sampler2D flow_tex, grad0_tex, image0_tex, image1_tex; -uniform vec2 image_size, inv_image_size; +uniform vec2 image_size, inv_image_size, inv_prev_level_size; void main() { @@ -91,7 +91,7 @@ void main() mat2 H_inv = inverse(H); // Fetch the initial guess for the flow. - vec2 initial_u = texture(flow_tex, flow_tc).xy; + vec2 initial_u = texture(flow_tex, flow_tc).xy * inv_prev_level_size; // Note: The flow is in OpenGL coordinates [0..1], but the calculations // generally come out in pixels since the gradient is in pixels, diff --git a/prewarp.frag b/prewarp.frag index 1bf99f7..b9645bf 100644 --- a/prewarp.frag +++ b/prewarp.frag @@ -4,8 +4,10 @@ in vec2 tc; out float I, I_t; +out vec2 normalized_flow; uniform sampler2D image0_tex, image1_tex, flow_tex; +uniform vec2 image_size; void main() { @@ -17,4 +19,5 @@ void main() I = 0.5f * (I_0 + I_w); I_t = I_w - I_0; + normalized_flow = flow.xy * image_size; } diff --git a/smoothness.frag b/smoothness.frag index 99b9791..4dc06f3 100644 --- a/smoothness.frag +++ b/smoothness.frag @@ -6,17 +6,9 @@ const float eps_sq = 0.001 * 0.001; uniform sampler2D flow_tex, diff_flow_tex; -// The base flow needs to be normalized. -// TODO: Should we perhaps reduce this to a separate two-component -// texture when calculating the derivatives? -vec2 normalize_flow(vec3 flow) -{ - return flow.xy / flow.z; -} - // This must be a macro, since the offset needs to be a constant expression. #define get_flow(x_offs, y_offs) \ - (normalize_flow(textureOffset(flow_tex, tc, ivec2((x_offs), (y_offs))).xyz) + \ + (textureOffset(flow_tex, tc, ivec2((x_offs), (y_offs))).xy + \ textureOffset(diff_flow_tex, tc, ivec2((x_offs), (y_offs))).xy) float diffusivity(float u_x, float u_y, float v_x, float v_y) -- 2.39.5