4 // Code for computing optical flow between two images, and using it to interpolate
5 // in-between frames. The main user interface is the Interpolate class.
16 // A class that caches FBOs that render to a given set of textures.
17 // It never frees anything, so it is only suitable for rendering to
18 // the same (small) set of textures over and over again.
19 template<size_t num_elements>
20 class PersistentFBOSet {
22 void render_to(const std::array<GLuint, num_elements> &textures);
24 // Convenience wrappers.
25 void render_to(GLuint texture0) {
26 render_to({{texture0}});
29 void render_to(GLuint texture0, GLuint texture1) {
30 render_to({{texture0, texture1}});
33 void render_to(GLuint texture0, GLuint texture1, GLuint texture2) {
34 render_to({{texture0, texture1, texture2}});
37 void render_to(GLuint texture0, GLuint texture1, GLuint texture2, GLuint texture3) {
38 render_to({{texture0, texture1, texture2, texture3}});
42 // TODO: Delete these on destruction.
43 std::map<std::array<GLuint, num_elements>, GLuint> fbos;
47 // Same, but with a depth texture.
48 template<size_t num_elements>
49 class PersistentFBOSetWithDepth {
51 void render_to(GLuint depth_rb, const std::array<GLuint, num_elements> &textures);
53 // Convenience wrappers.
54 void render_to(GLuint depth_rb, GLuint texture0) {
55 render_to(depth_rb, {{texture0}});
58 void render_to(GLuint depth_rb, GLuint texture0, GLuint texture1) {
59 render_to(depth_rb, {{texture0, texture1}});
62 void render_to(GLuint depth_rb, GLuint texture0, GLuint texture1, GLuint texture2) {
63 render_to(depth_rb, {{texture0, texture1, texture2}});
66 void render_to(GLuint depth_rb, GLuint texture0, GLuint texture1, GLuint texture2, GLuint texture3) {
67 render_to(depth_rb, {{texture0, texture1, texture2, texture3}});
71 // TODO: Delete these on destruction.
72 std::map<std::pair<GLuint, std::array<GLuint, num_elements>>, GLuint> fbos;
75 // Convert RGB to grayscale, using Rec. 709 coefficients.
76 class GrayscaleConversion {
78 GrayscaleConversion();
79 void exec(GLint tex, GLint gray_tex, int width, int height, int num_layers);
82 PersistentFBOSet<1> fbos;
91 // Compute gradients in every point, used for the motion search.
92 // The DIS paper doesn't actually mention how these are computed,
93 // but seemingly, a 3x3 Sobel operator is used here (at least in
94 // later versions of the code), while a [1 -8 0 8 -1] kernel is
95 // used for all the derivatives in the variational refinement part
96 // (which borrows code from DeepFlow). This is inconsistent,
97 // but I guess we're better off with staying with the original
98 // decisions until we actually know having different ones would be better.
102 void exec(GLint tex_view, GLint grad_tex, int level_width, int level_height, int num_layers);
105 PersistentFBOSet<1> fbos;
108 GLuint sobel_program;
113 // Motion search to find the initial flow. See motion_search.frag for documentation.
117 void exec(GLuint tex_view, GLuint grad_tex, GLuint flow_tex, GLuint flow_out_tex, int level_width, int level_height, int prev_level_width, int prev_level_height, int width_patches, int height_patches, int num_layers);
120 PersistentFBOSet<1> fbos;
122 GLuint motion_vs_obj;
123 GLuint motion_fs_obj;
124 GLuint motion_search_program;
126 GLuint uniform_inv_image_size, uniform_inv_prev_level_size, uniform_out_flow_size;
127 GLuint uniform_image_tex, uniform_grad_tex, uniform_flow_tex;
130 // Do “densification”, ie., upsampling of the flow patches to the flow field
131 // (the same size as the image at this level). We draw one quad per patch
132 // over its entire covered area (using instancing in the vertex shader),
133 // and then weight the contributions in the pixel shader by post-warp difference.
134 // This is equation (3) in the paper.
136 // We accumulate the flow vectors in the R/G channels (for u/v) and the total
137 // weight in the B channel. Dividing R and G by B gives the normalized values.
141 void exec(GLuint tex_view, GLuint flow_tex, GLuint dense_flow_tex, int level_width, int level_height, int width_patches, int height_patches, int num_layers);
144 PersistentFBOSet<1> fbos;
146 GLuint densify_vs_obj;
147 GLuint densify_fs_obj;
148 GLuint densify_program;
150 GLuint uniform_patch_size;
151 GLuint uniform_image_tex, uniform_flow_tex;
154 // Warp I_1 to I_w, and then compute the mean (I) and difference (I_t) of
155 // I_0 and I_w. The prewarping is what enables us to solve the variational
156 // flow for du,dv instead of u,v.
158 // Also calculates the normalized flow, ie. divides by z (this is needed because
159 // Densify works by additive blending) and multiplies by the image size.
161 // See variational_refinement.txt for more information.
165 void exec(GLuint tex_view, GLuint flow_tex, GLuint normalized_flow_tex, GLuint I_tex, GLuint I_t_tex, int level_width, int level_height, int num_layers);
168 PersistentFBOSet<3> fbos;
170 GLuint prewarp_vs_obj;
171 GLuint prewarp_fs_obj;
172 GLuint prewarp_program;
174 GLuint uniform_image_tex, uniform_flow_tex;
177 // From I, calculate the partial derivatives I_x and I_y. We use a four-tap
178 // central difference filter, since apparently, that's tradition (I haven't
179 // measured quality versus a more normal 0.5 (I[x+1] - I[x-1]).)
180 // The coefficients come from
182 // https://en.wikipedia.org/wiki/Finite_difference_coefficient
184 // Also computes β_0, since it depends only on I_x and I_y.
188 void exec(GLuint input_tex, GLuint I_x_y_tex, GLuint beta_0_tex, int level_width, int level_height, int num_layers);
191 PersistentFBOSet<2> fbos;
193 GLuint derivatives_vs_obj;
194 GLuint derivatives_fs_obj;
195 GLuint derivatives_program;
200 // Calculate the diffusivity for each pixels, g(x,y). Smoothness (s) will
201 // be calculated in the shaders on-the-fly by sampling in-between two
202 // neighboring g(x,y) pixels, plus a border tweak to make sure we get
203 // zero smoothness at the border.
205 // See variational_refinement.txt for more information.
206 class ComputeDiffusivity {
208 ComputeDiffusivity();
209 void exec(GLuint flow_tex, GLuint diff_flow_tex, GLuint diffusivity_tex, int level_width, int level_height, bool zero_diff_flow, int num_layers);
212 PersistentFBOSet<1> fbos;
214 GLuint diffusivity_vs_obj;
215 GLuint diffusivity_fs_obj;
216 GLuint diffusivity_program;
218 GLuint uniform_flow_tex, uniform_diff_flow_tex;
219 GLuint uniform_alpha, uniform_zero_diff_flow;
222 // Set up the equations set (two equations in two unknowns, per pixel).
223 // We store five floats; the three non-redundant elements of the 2x2 matrix (A)
224 // as 32-bit floats, and the two elements on the right-hand side (b) as 16-bit
225 // floats. (Actually, we store the inverse of the diagonal elements, because
226 // we only ever need to divide by them.) This fits into four u32 values;
227 // R, G, B for the matrix (the last element is symmetric) and A for the two b values.
228 // All the values of the energy term (E_I, E_G, E_S), except the smoothness
229 // terms that depend on other pixels, are calculated in one pass.
231 // The equation set is split in two; one contains only the pixels needed for
232 // the red pass, and one only for the black pass (see sor.frag). This reduces
233 // the amount of data the SOR shader has to pull in, at the cost of some
234 // complexity when the equation texture ends up with half the size and we need
235 // to adjust texture coordinates. The contraction is done along the horizontal
236 // axis, so that on even rows (0, 2, 4, ...), the “red” texture will contain
237 // pixels 0, 2, 4, 6, etc., and on odd rows 1, 3, 5, etc..
239 // See variational_refinement.txt for more information about the actual
241 class SetupEquations {
244 void exec(GLuint I_x_y_tex, GLuint I_t_tex, GLuint diff_flow_tex, GLuint flow_tex, GLuint beta_0_tex, GLuint diffusivity_tex, GLuint equation_red_tex, GLuint equation_black_tex, int level_width, int level_height, bool zero_diff_flow, int num_layers);
247 PersistentFBOSet<2> fbos;
249 GLuint equations_vs_obj;
250 GLuint equations_fs_obj;
251 GLuint equations_program;
253 GLuint uniform_I_x_y_tex, uniform_I_t_tex;
254 GLuint uniform_diff_flow_tex, uniform_base_flow_tex;
255 GLuint uniform_beta_0_tex;
256 GLuint uniform_diffusivity_tex;
257 GLuint uniform_gamma, uniform_delta, uniform_zero_diff_flow;
260 // Actually solve the equation sets made by SetupEquations, by means of
261 // successive over-relaxation (SOR).
263 // See variational_refinement.txt for more information.
267 void exec(GLuint diff_flow_tex, GLuint equation_red_tex, GLuint equation_black_tex, GLuint diffusivity_tex, int level_width, int level_height, int num_iterations, bool zero_diff_flow, int num_layers, ScopedTimer *sor_timer);
270 PersistentFBOSet<1> fbos;
276 GLuint uniform_diff_flow_tex;
277 GLuint uniform_equation_red_tex, uniform_equation_black_tex;
278 GLuint uniform_diffusivity_tex;
279 GLuint uniform_phase, uniform_num_nonzero_phases;
282 // Simply add the differential flow found by the variational refinement to the base flow.
283 // The output is in base_flow_tex; we don't need to make a new texture.
287 void exec(GLuint base_flow_tex, GLuint diff_flow_tex, int level_width, int level_height, int num_layers);
290 PersistentFBOSet<1> fbos;
292 GLuint add_flow_vs_obj;
293 GLuint add_flow_fs_obj;
294 GLuint add_flow_program;
296 GLuint uniform_diff_flow_tex;
299 // Take a copy of the flow, bilinearly interpolated and scaled up.
303 void exec(GLuint in_tex, GLuint out_tex, int input_width, int input_height, int output_width, int output_height, int num_layers);
306 PersistentFBOSet<1> fbos;
308 GLuint resize_flow_vs_obj;
309 GLuint resize_flow_fs_obj;
310 GLuint resize_flow_program;
312 GLuint uniform_flow_tex;
313 GLuint uniform_scale_factor;
318 GLuint get_texture(GLenum format, GLuint width, GLuint height, GLuint num_layers = 0);
319 void release_texture(GLuint tex_num);
320 GLuint get_renderbuffer(GLenum format, GLuint width, GLuint height);
321 void release_renderbuffer(GLuint tex_num);
327 GLuint width, height, num_layers;
329 bool is_renderbuffer = false;
331 std::vector<Texture> textures;
334 class DISComputeFlow {
336 DISComputeFlow(int width, int height);
342 enum ResizeStrategy {
344 RESIZE_FLOW_TO_FULL_SIZE
347 // The texture must have two layers (first and second frame).
348 // Returns a texture that must be released with release_texture()
350 GLuint exec(GLuint tex, FlowDirection flow_direction, ResizeStrategy resize_strategy);
352 void release_texture(GLuint tex) {
353 pool.release_texture(tex);
358 GLuint initial_flow_tex;
359 GLuint vertex_vbo, vao;
362 // The various passes.
364 MotionSearch motion_search;
367 Derivatives derivatives;
368 ComputeDiffusivity compute_diffusivity;
369 SetupEquations setup_equations;
371 AddBaseFlow add_base_flow;
372 ResizeFlow resize_flow;
375 // Forward-warp the flow half-way (or rather, by alpha). A non-zero “splatting”
376 // radius fills most of the holes.
381 // alpha is the time of the interpolated frame (0..1).
382 void exec(GLuint image_tex, GLuint bidirectional_flow_tex, GLuint flow_tex, GLuint depth_rb, int width, int height, float alpha);
385 PersistentFBOSetWithDepth<1> fbos;
389 GLuint splat_program;
391 GLuint uniform_splat_size, uniform_alpha;
392 GLuint uniform_image_tex, uniform_flow_tex;
393 GLuint uniform_inv_flow_size;
396 // Doing good and fast hole-filling on a GPU is nontrivial. We choose an option
397 // that's fairly simple (given that most holes are really small) and also hopefully
398 // cheap should the holes not be so small. Conceptually, we look for the first
399 // non-hole to the left of us (ie., shoot a ray until we hit something), then
400 // the first non-hole to the right of us, then up and down, and then average them
401 // all together. It's going to create “stars” if the holes are big, but OK, that's
404 // Our implementation here is efficient assuming that the hierarchical Z-buffer is
405 // on even for shaders that do discard (this typically kills early Z, but hopefully
406 // not hierarchical Z); we set up Z so that only holes are written to, which means
407 // that as soon as a hole is filled, the rasterizer should just skip it. Most of the
408 // fullscreen quads should just be discarded outright, really.
413 // Output will be in flow_tex, temp_tex[0, 1, 2], representing the filling
414 // from the down, left, right and up, respectively. Use HoleBlend to merge
416 void exec(GLuint flow_tex, GLuint depth_rb, GLuint temp_tex[3], int width, int height);
419 PersistentFBOSetWithDepth<1> fbos;
426 GLuint uniform_z, uniform_sample_offset;
429 // Blend the four directions from HoleFill into one pixel, so that single-pixel
430 // holes become the average of their four neighbors.
435 void exec(GLuint flow_tex, GLuint depth_rb, GLuint temp_tex[3], int width, int height);
438 PersistentFBOSetWithDepth<1> fbos;
442 GLuint blend_program;
444 GLuint uniform_left_tex, uniform_right_tex, uniform_up_tex, uniform_down_tex;
445 GLuint uniform_z, uniform_sample_offset;
451 void exec(GLuint image_tex, GLuint flow_tex, GLuint output_tex, int width, int height, float alpha);
454 PersistentFBOSet<1> fbos;
457 GLuint blend_program;
459 GLuint uniform_image_tex, uniform_flow_tex;
460 GLuint uniform_alpha, uniform_flow_consistency_tolerance;
465 Interpolate(int width, int height, int flow_level);
467 // Returns a texture that must be released with release_texture()
468 // after use. image_tex must be a two-layer RGBA8 texture with mipmaps
469 // (unless flow_level == 0).
470 GLuint exec(GLuint image_tex, GLuint bidirectional_flow_tex, GLuint width, GLuint height, float alpha);
472 void release_texture(GLuint tex) {
473 pool.release_texture(tex);
477 int width, height, flow_level;
478 GLuint vertex_vbo, vao;
483 HoleBlend hole_blend;
487 #endif // !defined(_FLOW_H)