bind_sampler(splat_program, uniform_image1_tex, 1, tex1, linear_sampler);
// FIXME: This is set to 1.0 right now so not to trigger Haswell's “PMA stall”.
- // Move to 2.0 later.
+ // Move to 2.0 later, or even 4.0.
+ // (Since we have hole filling, it's not critical, but larger values seem to do
+ // better than hole filling for large motion, blurs etc.)
float splat_size = 1.0f; // 4x4 splat means 16x overdraw, 2x2 splat means 4x overdraw.
glProgramUniform2f(splat_program, uniform_splat_size, splat_size / width, splat_size / height);
glProgramUniform1f(splat_program, uniform_alpha, alpha);
glDeleteFramebuffers(1, &fbo);
}
+// Doing good and fast hole-filling on a GPU is nontrivial. We choose an option
+// that's fairly simple (given that most holes are really small) and also hopefully
+// cheap should the holes not be so small. Conceptually, we look for the first
+// non-hole to the left of us (ie., shoot a ray until we hit something), then
+// the first non-hole to the right of us, then up and down, and then average them
+// all together. It's going to create “stars” if the holes are big, but OK, that's
+// a tradeoff.
+//
+// Our implementation here is efficient assuming that the hierarchical Z-buffer is
+// on even for shaders that do discard (this typically kills early Z, but hopefully
+// not hierarchical Z); we set up Z so that only holes are written to, which means
+// that as soon as a hole is filled, the rasterizer should just skip it. Most of the
+// fullscreen quads should just be discarded outright, really.
+class HoleFill {
+public:
+ HoleFill();
+
+ // Output will be in flow_tex, temp_tex[0, 1, 2], representing the filling
+ // from the down, left, right and up, respectively. Use HoleBlend to merge
+ // them into one.
+ void exec(GLuint flow_tex, GLuint depth_tex, GLuint temp_tex[3], int width, int height);
+
+private:
+ PersistentFBOSet<2> fbos;
+
+ GLuint fill_vs_obj;
+ GLuint fill_fs_obj;
+ GLuint fill_program;
+ GLuint fill_vao;
+
+ GLuint uniform_tex;
+ GLuint uniform_z, uniform_sample_offset;
+};
+
+HoleFill::HoleFill()
+{
+ fill_vs_obj = compile_shader(read_file("hole_fill.vert"), GL_VERTEX_SHADER);
+ fill_fs_obj = compile_shader(read_file("hole_fill.frag"), GL_FRAGMENT_SHADER);
+ fill_program = link_program(fill_vs_obj, fill_fs_obj);
+
+ // Set up the VAO containing all the required position/texcoord data.
+ glCreateVertexArrays(1, &fill_vao);
+ glBindVertexArray(fill_vao);
+ glBindBuffer(GL_ARRAY_BUFFER, vertex_vbo);
+
+ GLint position_attrib = glGetAttribLocation(fill_program, "position");
+ glEnableVertexArrayAttrib(fill_vao, position_attrib);
+ glVertexAttribPointer(position_attrib, 2, GL_FLOAT, GL_FALSE, 0, BUFFER_OFFSET(0));
+
+ uniform_tex = glGetUniformLocation(fill_program, "tex");
+ uniform_z = glGetUniformLocation(fill_program, "z");
+ uniform_sample_offset = glGetUniformLocation(fill_program, "sample_offset");
+}
+
+void HoleFill::exec(GLuint flow_tex, GLuint depth_tex, GLuint temp_tex[3], int width, int height)
+{
+ glUseProgram(fill_program);
+
+ bind_sampler(fill_program, uniform_tex, 0, flow_tex, nearest_sampler);
+
+ glProgramUniform1f(fill_program, uniform_z, 1.0f - 1.0f / 1024.0f);
+
+ glViewport(0, 0, width, height);
+ glDisable(GL_BLEND);
+ glEnable(GL_DEPTH_TEST);
+ glDepthFunc(GL_LESS); // Only update the values > 0.999f (ie., only invalid pixels).
+ glBindVertexArray(fill_vao);
+
+ // FIXME: Get this into FBOSet, so we can reuse FBOs across frames.
+ GLuint fbo;
+ glCreateFramebuffers(1, &fbo);
+ glNamedFramebufferTexture(fbo, GL_COLOR_ATTACHMENT0, flow_tex, 0); // NOTE: Reading and writing to the same texture.
+ glNamedFramebufferTexture(fbo, GL_DEPTH_ATTACHMENT, depth_tex, 0);
+ glBindFramebuffer(GL_FRAMEBUFFER, fbo);
+
+ // Fill holes from the left, by shifting 1, 2, 4, 8, etc. pixels to the right.
+ for (int offs = 1; offs < width; offs *= 2) {
+ glProgramUniform2f(fill_program, uniform_sample_offset, -offs / float(width), 0.0f);
+ glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+ glTextureBarrier();
+ }
+ glCopyImageSubData(flow_tex, GL_TEXTURE_2D, 0, 0, 0, 0, temp_tex[0], GL_TEXTURE_2D, 0, 0, 0, 0, width, height, 1);
+
+ // Similar to the right; adjust Z a bit down, so that we re-fill the pixels that
+ // were overwritten in the last algorithm.
+ glProgramUniform1f(fill_program, uniform_z, 1.0f - 2.0f / 1024.0f);
+ for (int offs = 1; offs < width; offs *= 2) {
+ glProgramUniform2f(fill_program, uniform_sample_offset, offs / float(width), 0.0f);
+ glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+ glTextureBarrier();
+ }
+ glCopyImageSubData(flow_tex, GL_TEXTURE_2D, 0, 0, 0, 0, temp_tex[1], GL_TEXTURE_2D, 0, 0, 0, 0, width, height, 1);
+
+ // Up.
+ glProgramUniform1f(fill_program, uniform_z, 1.0f - 3.0f / 1024.0f);
+ for (int offs = 1; offs < height; offs *= 2) {
+ glProgramUniform2f(fill_program, uniform_sample_offset, 0.0f, -offs / float(height));
+ glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+ glTextureBarrier();
+ }
+ glCopyImageSubData(flow_tex, GL_TEXTURE_2D, 0, 0, 0, 0, temp_tex[2], GL_TEXTURE_2D, 0, 0, 0, 0, width, height, 1);
+
+ // Down.
+ glProgramUniform1f(fill_program, uniform_z, 1.0f - 4.0f / 1024.0f);
+ for (int offs = 1; offs < height; offs *= 2) {
+ glProgramUniform2f(fill_program, uniform_sample_offset, 0.0f, offs / float(height));
+ glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+ glTextureBarrier();
+ }
+
+ glDisable(GL_DEPTH_TEST);
+
+ glDeleteFramebuffers(1, &fbo);
+}
+
+// Blend the four directions from HoleFill into one pixel, so that single-pixel
+// holes become the average of their four neighbors.
+class HoleBlend {
+public:
+ HoleBlend();
+
+ void exec(GLuint flow_tex, GLuint depth_tex, GLuint temp_tex[3], int width, int height);
+
+private:
+ PersistentFBOSet<2> fbos;
+
+ GLuint blend_vs_obj;
+ GLuint blend_fs_obj;
+ GLuint blend_program;
+ GLuint blend_vao;
+
+ GLuint uniform_left_tex, uniform_right_tex, uniform_up_tex, uniform_down_tex;
+ GLuint uniform_z, uniform_sample_offset;
+};
+
+HoleBlend::HoleBlend()
+{
+ blend_vs_obj = compile_shader(read_file("hole_fill.vert"), GL_VERTEX_SHADER); // Reuse the vertex shader from the fill.
+ blend_fs_obj = compile_shader(read_file("hole_blend.frag"), GL_FRAGMENT_SHADER);
+ blend_program = link_program(blend_vs_obj, blend_fs_obj);
+
+ // Set up the VAO containing all the required position/texcoord data.
+ glCreateVertexArrays(1, &blend_vao);
+ glBindVertexArray(blend_vao);
+ glBindBuffer(GL_ARRAY_BUFFER, vertex_vbo);
+
+ GLint position_attrib = glGetAttribLocation(blend_program, "position");
+ glEnableVertexArrayAttrib(blend_vao, position_attrib);
+ glVertexAttribPointer(position_attrib, 2, GL_FLOAT, GL_FALSE, 0, BUFFER_OFFSET(0));
+
+ uniform_left_tex = glGetUniformLocation(blend_program, "left_tex");
+ uniform_right_tex = glGetUniformLocation(blend_program, "right_tex");
+ uniform_up_tex = glGetUniformLocation(blend_program, "up_tex");
+ uniform_down_tex = glGetUniformLocation(blend_program, "down_tex");
+ uniform_z = glGetUniformLocation(blend_program, "z");
+ uniform_sample_offset = glGetUniformLocation(blend_program, "sample_offset");
+}
+
+void HoleBlend::exec(GLuint flow_tex, GLuint depth_tex, GLuint temp_tex[3], int width, int height)
+{
+ glUseProgram(blend_program);
+
+ bind_sampler(blend_program, uniform_left_tex, 0, temp_tex[0], nearest_sampler);
+ bind_sampler(blend_program, uniform_right_tex, 1, temp_tex[1], nearest_sampler);
+ bind_sampler(blend_program, uniform_up_tex, 2, temp_tex[2], nearest_sampler);
+ bind_sampler(blend_program, uniform_down_tex, 3, flow_tex, nearest_sampler);
+
+ glProgramUniform1f(blend_program, uniform_z, 1.0f - 4.0f / 1024.0f);
+ glProgramUniform2f(blend_program, uniform_sample_offset, 0.0f, 0.0f);
+
+ glViewport(0, 0, width, height);
+ glDisable(GL_BLEND);
+ glEnable(GL_DEPTH_TEST);
+ glDepthFunc(GL_LEQUAL); // Skip over all of the pixels that were never holes to begin with.
+ glBindVertexArray(blend_vao);
+
+ // FIXME: Get this into FBOSet, so we can reuse FBOs across frames.
+ GLuint fbo;
+ glCreateFramebuffers(1, &fbo);
+ glNamedFramebufferTexture(fbo, GL_COLOR_ATTACHMENT0, flow_tex, 0); // NOTE: Reading and writing to the same texture.
+ glNamedFramebufferTexture(fbo, GL_DEPTH_ATTACHMENT, depth_tex, 0);
+ glBindFramebuffer(GL_FRAMEBUFFER, fbo);
+
+ glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+
+ glDisable(GL_DEPTH_TEST);
+
+ glDeleteFramebuffers(1, &fbo);
+}
+
class Blend {
public:
Blend();
int width, height, flow_level;
TexturePool pool;
Splat splat;
+ HoleFill hole_fill;
+ HoleBlend hole_blend;
Blend blend;
};
GLuint depth_tex = pool.get_texture(GL_DEPTH_COMPONENT32F, flow_width, flow_height); // Used for ranking flows.
{
ScopedTimer timer("Clear", &total_timer);
- glClearTexImage(flow_tex, 0, GL_RG, GL_FLOAT, nullptr);
+ float invalid_flow[] = { 1000.0f, 1000.0f };
+ glClearTexImage(flow_tex, 0, GL_RG, GL_FLOAT, invalid_flow);
float infinity = 1.0f;
glClearTexImage(depth_tex, 0, GL_DEPTH_COMPONENT, GL_FLOAT, &infinity);
}
- //SDL_GL_SwapWindow(window);
{
ScopedTimer timer("Splat", &total_timer);
splat.exec(tex0_view, tex1_view, forward_flow_tex, backward_flow_tex, flow_tex, depth_tex, flow_width, flow_height, alpha);
}
- //SDL_GL_SwapWindow(window);
- pool.release_texture(depth_tex);
glDeleteTextures(1, &tex0_view);
glDeleteTextures(1, &tex1_view);
- GLuint output_tex = pool.get_texture(GL_RGB8, width, height);
+ GLuint temp_tex[3];
+ temp_tex[0] = pool.get_texture(GL_RG16F, flow_width, flow_height);
+ temp_tex[1] = pool.get_texture(GL_RG16F, flow_width, flow_height);
+ temp_tex[2] = pool.get_texture(GL_RG16F, flow_width, flow_height);
+
+ {
+ ScopedTimer timer("Fill holes", &total_timer);
+ hole_fill.exec(flow_tex, depth_tex, temp_tex, flow_width, flow_height);
+ hole_blend.exec(flow_tex, depth_tex, temp_tex, flow_width, flow_height);
+ }
+
+ pool.release_texture(temp_tex[0]);
+ pool.release_texture(temp_tex[1]);
+ pool.release_texture(temp_tex[2]);
+ pool.release_texture(depth_tex);
+
+ GLuint output_tex = pool.get_texture(GL_RGBA8, width, height);
{
ScopedTimer timer("Blend", &total_timer);
blend.exec(tex0, tex1, flow_tex, output_tex, width, height, alpha);
}
}
+// Not relevant for RGB.
+void flip_coordinate_system(uint8_t *dense_flow, unsigned width, unsigned height)
+{
+}
+
void write_flow(const char *filename, const float *dense_flow, unsigned width, unsigned height)
{
FILE *flowfp = fopen(filename, "wb");
fclose(flowfp);
}
+// Not relevant for RGB.
+void write_flow(const char *filename, const uint8_t *dense_flow, unsigned width, unsigned height)
+{
+ assert(false);
+}
+
void write_ppm(const char *filename, const float *dense_flow, unsigned width, unsigned height)
{
FILE *fp = fopen(filename, "wb");
fclose(fp);
}
+void write_ppm(const char *filename, const uint8_t *rgba, unsigned width, unsigned height)
+{
+ unique_ptr<uint8_t[]> rgb_line(new uint8_t[width * 3 + 1]);
+
+ FILE *fp = fopen(filename, "wb");
+ fprintf(fp, "P6\n%d %d\n255\n", width, height);
+ for (unsigned y = 0; y < height; ++y) {
+ unsigned y2 = height - 1 - y;
+ for (size_t x = 0; x < width; ++x) {
+ memcpy(&rgb_line[x * 3], &rgba[(y2 * width + x) * 4], 4);
+ }
+ fwrite(rgb_line.get(), width * 3, 1, fp);
+ }
+ fclose(fp);
+}
+
+struct FlowType {
+ using type = float;
+ static constexpr GLenum gl_format = GL_RG;
+ static constexpr GLenum gl_type = GL_FLOAT;
+ static constexpr int num_channels = 2;
+};
+
+struct RGBAType {
+ using type = uint8_t;
+ static constexpr GLenum gl_format = GL_RGBA;
+ static constexpr GLenum gl_type = GL_UNSIGNED_BYTE;
+ static constexpr int num_channels = 4;
+};
+
+template <class Type>
void finish_one_read(GLuint width, GLuint height)
{
+ using T = typename Type::type;
+ constexpr int bytes_per_pixel = Type::num_channels * sizeof(T);
+
assert(!reads_in_progress.empty());
ReadInProgress read = reads_in_progress.front();
reads_in_progress.pop_front();
- unique_ptr<float[]> flow(new float[width * height * 2]);
- void *buf = glMapNamedBufferRange(read.pbo, 0, width * height * 2 * sizeof(float), GL_MAP_READ_BIT); // Blocks if the read isn't done yet.
- memcpy(flow.get(), buf, width * height * 2 * sizeof(float));
+ unique_ptr<T[]> flow(new typename Type::type[width * height * Type::num_channels]);
+ void *buf = glMapNamedBufferRange(read.pbo, 0, width * height * bytes_per_pixel, GL_MAP_READ_BIT); // Blocks if the read isn't done yet.
+ memcpy(flow.get(), buf, width * height * bytes_per_pixel); // TODO: Unneeded for RGBType, since flip_coordinate_system() does nothing.:
glUnmapNamedBuffer(read.pbo);
spare_pbos.push(read.pbo);
}
}
+template <class Type>
void schedule_read(GLuint tex, GLuint width, GLuint height, const char *filename0, const char *filename1, const char *flow_filename, const char *ppm_filename)
{
+ using T = typename Type::type;
+ constexpr int bytes_per_pixel = Type::num_channels * sizeof(T);
+
if (spare_pbos.empty()) {
- finish_one_read(width, height);
+ finish_one_read<Type>(width, height);
}
assert(!spare_pbos.empty());
reads_in_progress.emplace_back(ReadInProgress{ spare_pbos.top(), filename0, filename1, flow_filename, ppm_filename });
glBindBuffer(GL_PIXEL_PACK_BUFFER, spare_pbos.top());
spare_pbos.pop();
- glGetTextureImage(tex, 0, GL_RG, GL_FLOAT, width * height * 2 * sizeof(float), nullptr);
+ glGetTextureImage(tex, 0, Type::gl_format, Type::gl_type, width * height * bytes_per_pixel, nullptr);
glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);
}
DISComputeFlow compute_flow(width1, height1);
GLuint final_tex = compute_flow.exec(tex0_gray, tex1_gray, DISComputeFlow::RESIZE_FLOW_TO_FULL_SIZE);
- schedule_read(final_tex, width1, height1, filename0, filename1, flow_filename, "flow.ppm");
+ schedule_read<FlowType>(final_tex, width1, height1, filename0, filename1, flow_filename, "flow.ppm");
compute_flow.release_texture(final_tex);
// See if there are more flows on the command line (ie., more than three arguments),
GLuint final_tex = compute_flow.exec(tex0_gray, tex1_gray, DISComputeFlow::RESIZE_FLOW_TO_FULL_SIZE);
- schedule_read(final_tex, width1, height1, filename0, filename1, flow_filename, "");
+ schedule_read<FlowType>(final_tex, width1, height1, filename0, filename1, flow_filename, "");
compute_flow.release_texture(final_tex);
}
glDeleteTextures(1, &tex0_gray);
glDeleteTextures(1, &tex1_gray);
while (!reads_in_progress.empty()) {
- finish_one_read(width1, height1);
+ finish_one_read<FlowType>(width1, height1);
}
}
GLuint pbos[5];
glCreateBuffers(5, pbos);
for (int i = 0; i < 5; ++i) {
- glNamedBufferData(pbos[i], width1 * height1 * 2 * sizeof(float), nullptr, GL_STREAM_READ);
+ glNamedBufferData(pbos[i], width1 * height1 * 4 * sizeof(uint8_t), nullptr, GL_STREAM_READ);
spare_pbos.push(pbos[i]);
}
DISComputeFlow compute_flow(width1, height1);
GrayscaleConversion gray;
Interpolate interpolate(width1, height1, finest_level);
- //Interpolate interpolate(width1, height1, 0);
int levels = find_num_levels(width1, height1);
GLuint tex0_gray, tex1_gray;
GLuint backward_flow_tex = compute_flow.exec(tex1_gray, tex0_gray, DISComputeFlow::DO_NOT_RESIZE_FLOW);
for (int frameno = 1; frameno < 60; ++frameno) {
+ char ppm_filename[256];
+ snprintf(ppm_filename, sizeof(ppm_filename), "interp%04d.ppm", frameno);
+
float alpha = frameno / 60.0f;
GLuint interpolated_tex = interpolate.exec(tex0, tex1, forward_flow_tex, backward_flow_tex, width1, height1, alpha);
- unique_ptr<uint8_t[]> rgb(new uint8_t[width1 * height1 * 3]);
- glGetTextureImage(interpolated_tex, 0, GL_RGB, GL_UNSIGNED_BYTE, width1 * height1 * 3, rgb.get());
-
- char buf[256];
- snprintf(buf, sizeof(buf), "interp%04d.ppm", frameno);
- FILE *fp = fopen(buf, "wb");
- fprintf(fp, "P6\n%d %d\n255\n", width1, height1);
- for (unsigned y = 0; y < height1; ++y) {
- unsigned y2 = height1 - 1 - y;
- fwrite(rgb.get() + y2 * width1 * 3, width1 * 3, 1, fp);
- }
- fclose(fp);
+ schedule_read<RGBAType>(interpolated_tex, width1, height1, filename0, filename1, "", ppm_filename);
+ interpolate.release_texture(interpolated_tex);
}
- //schedule_read(interpolated_tex, width1, height1, filename0, filename1, "", "halfflow.ppm");
- //interpolate.release_texture(interpolated_tex);
- //finish_one_read(width1, height1);
+ while (!reads_in_progress.empty()) {
+ finish_one_read<RGBAType>(width1, height1);
+ }
}
int main(int argc, char **argv)
projects / nageru / blobdiff