]> git.sesse.net Git - nageru/commitdiff
Start hacking in support for interpolated frames in the main application.
authorSteinar H. Gunderson <sgunderson@bigfoot.com>
Sun, 9 Sep 2018 19:21:39 +0000 (21:21 +0200)
committerSteinar H. Gunderson <sgunderson@bigfoot.com>
Sat, 15 Sep 2018 17:39:49 +0000 (19:39 +0200)
12 files changed:
Makefile
context.cpp [new file with mode: 0644]
context.h [new file with mode: 0644]
jpeg_frame_view.cpp
jpeg_frame_view.h
main.cpp
mainwindow.cpp
player.cpp
player.h
ref_counted_gl_sync.h [new file with mode: 0644]
video_stream.cpp
video_stream.h

index 4b0e275ba6c56bb5352429143df9b2deaa9ecd79..a7ae389f24e5140e4ec2aa297fc4103c1a1e842d 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,10 @@ OBJS_WITH_MOC = mainwindow.o jpeg_frame_view.o clip_list.o
 OBJS += $(OBJS_WITH_MOC)
 OBJS += $(OBJS_WITH_MOC:.o=.moc.o) 
 
-OBJS += ffmpeg_raii.o main.o player.o httpd.o mux.o metacube2.o video_stream.o
+# Flow objects
+OBJS += flow.o gpu_timers.o
+
+OBJS += ffmpeg_raii.o main.o player.o httpd.o mux.o metacube2.o video_stream.o context.o
 
 %.o: %.cpp
        $(CXX) -MMD -MP $(CPPFLAGS) $(CXXFLAGS) -o $@ -c $<
diff --git a/context.cpp b/context.cpp
new file mode 100644 (file)
index 0000000..ac8c97b
--- /dev/null
@@ -0,0 +1,68 @@
+#include <stdio.h>
+
+#include <string>
+
+#include <QGL>
+#include <QOffscreenSurface>
+#include <QOpenGLContext>
+#include <QSurface>
+#include <QSurfaceFormat>
+
+QGLWidget *global_share_widget = nullptr;
+
+using namespace std;
+
+QSurface *create_surface()
+{
+       QSurfaceFormat fmt;
+       fmt.setDepthBufferSize(0);
+       fmt.setStencilBufferSize(0);
+       fmt.setProfile(QSurfaceFormat::CoreProfile);
+       fmt.setMajorVersion(4);
+       fmt.setMinorVersion(5);
+       fmt.setSwapInterval(0);
+       QOffscreenSurface *surface = new QOffscreenSurface;
+       surface->setFormat(fmt);
+       surface->create();
+       if (!surface->isValid()) {
+               fprintf(stderr, "ERROR: surface not valid!\n");
+               exit(1);
+       }
+       return surface;
+}
+
+QSurface *create_surface(const QSurfaceFormat &format)
+{
+       QOffscreenSurface *surface = new QOffscreenSurface;
+       surface->setFormat(format);
+       surface->create();
+       if (!surface->isValid()) {
+               fprintf(stderr, "ERROR: surface not valid!\n");
+               exit(1);
+       }
+       return surface;
+}
+
+QSurface *create_surface_with_same_format(const QSurface *surface)
+{
+       return create_surface(surface->format());
+}
+
+QOpenGLContext *create_context(const QSurface *surface)
+{
+       QOpenGLContext *context = new QOpenGLContext;
+       context->setShareContext(global_share_widget->context()->contextHandle());
+       context->setFormat(surface->format());
+       context->create();
+       return context;
+}
+
+bool make_current(QOpenGLContext *context, QSurface *surface)
+{
+       return context->makeCurrent(surface);
+}
+
+void delete_context(QOpenGLContext *context)
+{
+       delete context;
+}
diff --git a/context.h b/context.h
new file mode 100644 (file)
index 0000000..aebba96
--- /dev/null
+++ b/context.h
@@ -0,0 +1,17 @@
+
+// Needs to be in its own file because Qt and libepoxy seemingly don't coexist well
+// within the same file.
+
+class QSurface;
+class QOpenGLContext;
+class QSurfaceFormat;
+class QGLWidget;
+
+extern bool using_egl;
+extern QGLWidget *global_share_widget;
+QSurface *create_surface();
+QSurface *create_surface(const QSurfaceFormat &format);
+QSurface *create_surface_with_same_format(const QSurface *surface);
+QOpenGLContext *create_context(const QSurface *surface);
+bool make_current(QOpenGLContext *context, QSurface *surface);
+void delete_context(QOpenGLContext *context);
index 16224ad3db7fbd19df2ab465e30aacae335f14e9..3d35e5e8818e11254458543c30038947efbc0d7b 100644 (file)
@@ -16,6 +16,7 @@
 
 #include "defs.h"
 #include "post_to_main_thread.h"
+#include "video_stream.h"
 
 using namespace movit;
 using namespace std;
@@ -221,14 +222,11 @@ void JPEGFrameView::initializeGL()
 {
        glDisable(GL_BLEND);
        glDisable(GL_DEPTH_TEST);
-       glDepthMask(GL_FALSE);
        check_error();
 
        static once_flag once;
        call_once(once, [] {
-               CHECK(init_movit(MOVIT_SHADER_DIR, MOVIT_DEBUG_OFF));
                resource_pool = new ResourcePool;
-
                std::thread(&jpeg_decoder_thread).detach();
        });
 
index 277241b69d70c77c5375a10dfec8b1fdfcb84f8b..1f7b2b7a67108a88e6baaaddcdbd436d0b746b4e 100644 (file)
@@ -11,8 +11,6 @@
 
 #include <memory>
 
-std::string filename_for_frame(unsigned stream_idx, int64_t pts);
-
 struct Frame {
        std::unique_ptr<uint8_t[]> y, cb, cr;
        unsigned width, height;
@@ -20,6 +18,9 @@ struct Frame {
        unsigned pitch_y, pitch_chroma;
 };
 
+std::string filename_for_frame(unsigned stream_idx, int64_t pts);
+std::shared_ptr<Frame> decode_jpeg(const std::string &filename);
+
 class JPEGFrameView : public QGLWidget {
        Q_OBJECT
 
index e8c0dc4ac1fe7e6769dea0b98377b29729c137b1..ccfb281ddecceadde96d16a15331d62e53dbb15f 100644 (file)
--- a/main.cpp
+++ b/main.cpp
@@ -16,18 +16,25 @@ extern "C" {
 
 #include <QApplication>
 
+#include <movit/init.h>
+#include <movit/util.h>
+
 #include "clip_list.h"
+#include "context.h"
 #include "defs.h"
 #include "mainwindow.h"
 #include "ffmpeg_raii.h"
 #include "httpd.h"
 #include "player.h"
 #include "post_to_main_thread.h"
+#include "ref_counted_gl_sync.h"
 #include "ui_mainwindow.h"
 
 using namespace std;
 using namespace std::chrono;
 
+std::mutex RefCountedGLsync::fence_lock;
+
 // TODO: Replace by some sort of GUI control, I guess.
 int64_t current_pts = 0;
 
@@ -40,7 +47,6 @@ string filename_for_frame(unsigned stream_idx, int64_t pts)
 
 mutex frame_mu;
 vector<int64_t> frames[MAX_STREAMS];
-QGLWidget *global_share_widget;
 HTTPD *global_httpd;
 
 int record_thread_func();
@@ -57,8 +63,8 @@ int main(int argc, char **argv)
        fmt.setDepthBufferSize(0);
        fmt.setStencilBufferSize(0);
        fmt.setProfile(QSurfaceFormat::CoreProfile);
-       fmt.setMajorVersion(3);
-       fmt.setMinorVersion(1);
+       fmt.setMajorVersion(4);
+       fmt.setMinorVersion(5);
 
        // Turn off vsync, since Qt generally gives us at most frame rate
        // (display frequency) / (number of QGLWidgets active).
@@ -71,9 +77,20 @@ int main(int argc, char **argv)
        QApplication app(argc, argv);
        global_share_widget = new QGLWidget();
        if (!global_share_widget->isValid()) {
-               fprintf(stderr, "Failed to initialize OpenGL. Futatabi needs at least OpenGL 3.1 to function properly.\n");
+               fprintf(stderr, "Failed to initialize OpenGL. Futatabi needs at least OpenGL 4.5 to function properly.\n");
                exit(1);
        }
+
+       // Initialize Movit.
+       {
+               QSurface *surface = create_surface();
+               QOpenGLContext *context = create_context(surface);
+               make_current(context, surface);
+               CHECK(movit::init_movit(MOVIT_SHADER_DIR, movit::MOVIT_DEBUG_OFF));
+               delete_context(context);
+               // TODO: Delete the surface, too.
+       }
+
        MainWindow mainWindow;
        mainWindow.show();
 
index 3b8561fac7a32fa3a14217a85ff8ff890cac23cb..0e73b990e0b7387a3931dea135c3c0d6792e0d4a 100644 (file)
@@ -87,8 +87,8 @@ MainWindow::MainWindow()
                this, &MainWindow::playlist_selection_changed);
        playlist_selection_changed();  // First time set-up.
 
-       preview_player = new Player(ui->preview_display);
-       live_player = new Player(ui->live_display);
+       preview_player = new Player(ui->preview_display, /*also_output_to_stream=*/false);
+       live_player = new Player(ui->live_display, /*also_output_to_stream=*/true);
        live_player->set_done_callback([this]{
                post_to_main_thread([this]{
                        live_player_clip_done();
index 1533977b283df0108c2b1f2246232941c89743ee..970f3402d88ddcd489cf4902bf15b5cbdf8aeb29 100644 (file)
@@ -7,13 +7,17 @@
 
 #include <stdio.h>
 
+#include <movit/util.h>
+
 #include "clip_list.h"
+#include "context.h"
 #include "defs.h"
 #include "ffmpeg_raii.h"
 #include "httpd.h"
 #include "jpeg_frame_view.h"
 #include "mux.h"
 #include "player.h"
+#include "video_stream.h"
 
 using namespace std;
 using namespace std::chrono;
@@ -22,8 +26,25 @@ extern mutex frame_mu;
 extern vector<int64_t> frames[MAX_STREAMS];
 extern HTTPD *global_httpd;
 
-void Player::thread_func()
+void Player::thread_func(bool also_output_to_stream)
 {
+       QSurface *surface = create_surface();
+       QOpenGLContext *context = create_context(surface);
+       if (!make_current(context, surface)) {
+               printf("oops\n");
+               exit(1);
+       }
+
+       check_error();
+
+       // Create the VideoStream object, now that we have an OpenGL context.
+       if (also_output_to_stream) {
+               video_stream.reset(new VideoStream);
+               video_stream->start();
+       }
+       
+       check_error();
+
        for ( ;; ) {
                // Wait until we're supposed to play something.
                {
@@ -82,11 +103,30 @@ void Player::thread_func()
 
                        destination->setFrame(stream_idx, next_pts);
 
-                       // Send the frame to the stream.
-                       // FIXME: Vaguely less crazy pts, perhaps.
-                       double pts_float = fmod(duration<double>(next_frame_start.time_since_epoch()).count(), 86400.0f);
-                       int64_t pts = lrint(pts_float * TIMEBASE);
-                       video_stream.schedule_original_frame(pts, stream_idx, next_pts);
+                       if (video_stream != nullptr) {
+                               // Send the frame to the stream.
+                               // FIXME: Vaguely less crazy pts, perhaps.
+                               double pts_float = fmod(duration<double>(next_frame_start.time_since_epoch()).count(), 86400.0f);
+                               int64_t pts = lrint(pts_float * TIMEBASE);
+                               video_stream->schedule_original_frame(pts, stream_idx, next_pts);
+
+                               // HACK: test interpolation by frame-doubling.
+                               int64_t next_next_pts = -1;
+                               {
+                                       lock_guard<mutex> lock(frame_mu);
+                                       auto it = upper_bound(frames[stream_idx].begin(),
+                                               frames[stream_idx].end(),
+                                               next_pts);
+                                       if (it == frames[stream_idx].end() || *it >= clip.pts_out) {
+                                               break;
+                                       }
+                                       next_next_pts = *it;
+                               }
+                               if (next_next_pts != -1) {
+                                       int64_t interpolated_pts = (next_pts + next_next_pts) / 2;
+                                       video_stream->schedule_interpolated_frame(interpolated_pts, stream_idx, next_pts, next_next_pts, 0.5f);
+                               }
+                       }
                }
 
                {
@@ -99,11 +139,10 @@ void Player::thread_func()
        }
 }
 
-Player::Player(JPEGFrameView *destination)
+Player::Player(JPEGFrameView *destination, bool also_output_to_stream)
        : destination(destination)
 {
-       video_stream.start();
-       thread(&Player::thread_func, this).detach();
+       thread(&Player::thread_func, this, also_output_to_stream).detach();
 }
 
 void Player::play_clip(const Clip &clip, unsigned stream_idx)
index 3321a303efc3ed6dc7910649bcb2bf6620f3bd57..10e0b6ab471bc477cf86c17d843239a16f3a590a 100644 (file)
--- a/player.h
+++ b/player.h
@@ -2,17 +2,23 @@
 #define _PLAYER_H 1
 
 #include "clip_list.h"
-#include "video_stream.h"
+
+extern "C" {
+#include <libavformat/avio.h>
+}
 
 #include <condition_variable>
 #include <functional>
 #include <mutex>
 
 class JPEGFrameView;
+class VideoStream;
+class QSurface;
+class QSurfaceFormat;
 
 class Player {
 public:
-       Player(JPEGFrameView *destination);
+       Player(JPEGFrameView *destination, bool also_output_to_stream);
 
        void play_clip(const Clip &clip, unsigned stream_idx);
        void override_angle(unsigned stream_idx);  // For the current clip only.
@@ -23,7 +29,7 @@ public:
        void set_done_callback(done_callback_func cb) { done_callback = cb; }
 
 private:
-       void thread_func();
+       void thread_func(bool also_output_to_stream);
        void open_output_stream();
        static int write_packet2_thunk(void *opaque, uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time);
        int write_packet2(uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time);
@@ -41,7 +47,7 @@ private:
        bool playing = false;  // Under queue_state_mu.
        int override_stream_idx = -1;  // Under queue_state_mu.
 
-       VideoStream video_stream;
+       std::unique_ptr<VideoStream> video_stream;  // Can be nullptr.
 };
 
 #endif  // !defined(_PLAYER_H)
diff --git a/ref_counted_gl_sync.h b/ref_counted_gl_sync.h
new file mode 100644 (file)
index 0000000..8b6db68
--- /dev/null
@@ -0,0 +1,39 @@
+#ifndef _REF_COUNTED_GL_SYNC_H
+#define _REF_COUNTED_GL_SYNC_H 1
+
+// A wrapper around GLsync (OpenGL fences) that is automatically refcounted.
+// Useful since we sometimes want to use the same fence two entirely different
+// places. (We could set two fences at the same time, but they are not an
+// unlimited hardware resource, so it would be a bit wasteful.)
+
+#include <epoxy/gl.h>
+#include <memory>
+#include <mutex>
+
+typedef std::shared_ptr<__GLsync> RefCountedGLsyncBase;
+
+class RefCountedGLsync : public RefCountedGLsyncBase {
+public:
+       RefCountedGLsync() {}
+
+       RefCountedGLsync(GLenum condition, GLbitfield flags) 
+               : RefCountedGLsyncBase(locked_glFenceSync(condition, flags), glDeleteSync) {}
+
+private:
+       // These are to work around apitrace bug #446.
+       static GLsync locked_glFenceSync(GLenum condition, GLbitfield flags)
+       {
+               std::lock_guard<std::mutex> lock(fence_lock);
+               return glFenceSync(condition, flags);
+       }
+
+       static void locked_glDeleteSync(GLsync sync)
+       {
+               std::lock_guard<std::mutex> lock(fence_lock);
+               glDeleteSync(sync);
+       }
+
+       static std::mutex fence_lock;
+};
+
+#endif  // !defined(_REF_COUNTED_GL_SYNC_H)
index 65f52c094bd70e16d41a09a9afbd0c573964e6e6..a783bdb91fe8246b1b59bb5dd97a75ad09a68b9d 100644 (file)
@@ -5,10 +5,18 @@ extern "C" {
 #include <libavformat/avio.h>
 }
 
+#include <unistd.h>
+
+#include "context.h"
+#include "flow.h"
 #include "httpd.h"
 #include "jpeg_frame_view.h"
+#include "movit/util.h"
 #include "mux.h"
 #include "player.h"
+#include "util.h"
+
+#include <epoxy/glx.h>
 
 using namespace std;
 
@@ -37,6 +45,84 @@ string read_file(const string &filename)
 
 }  // namespace
 
+VideoStream::VideoStream()
+{
+       using namespace movit;
+       // TODO: deduplicate code against JPEGFrameView?
+       ycbcr_convert_chain.reset(new EffectChain(1280, 720));
+       ImageFormat image_format;
+       image_format.color_space = COLORSPACE_sRGB;
+       image_format.gamma_curve = GAMMA_sRGB;
+       ycbcr_format.luma_coefficients = YCBCR_REC_709;
+       ycbcr_format.full_range = false;
+       ycbcr_format.num_levels = 256;
+       ycbcr_format.chroma_subsampling_x = 2;
+       ycbcr_format.chroma_subsampling_y = 1;
+       ycbcr_format.cb_x_position = 0.0f;  // H.264 -- _not_ JPEG, even though our input is MJPEG-encoded
+       ycbcr_format.cb_y_position = 0.5f;  // Irrelevant.
+       ycbcr_format.cr_x_position = 0.0f;
+       ycbcr_format.cr_y_position = 0.5f;
+       ycbcr_input = (movit::YCbCrInput *)ycbcr_convert_chain->add_input(new YCbCrInput(image_format, ycbcr_format, 1280, 720));
+
+       ImageFormat inout_format;
+       inout_format.color_space = COLORSPACE_sRGB;
+       inout_format.gamma_curve = GAMMA_sRGB;
+
+       check_error();
+       ycbcr_convert_chain->add_output(inout_format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED);
+       check_error();
+       ycbcr_convert_chain->set_dither_bits(8);
+       check_error();
+       ycbcr_convert_chain->finalize();
+       check_error();
+
+       GLuint input_tex[num_interpolate_slots], gray_tex[num_interpolate_slots];
+       glCreateTextures(GL_TEXTURE_2D_ARRAY, 10, input_tex);
+       glCreateTextures(GL_TEXTURE_2D_ARRAY, 10, gray_tex);
+       check_error();
+       constexpr size_t width = 1280, height = 720;  // FIXME: adjustable width, height
+       int levels = find_num_levels(width, height);
+       for (size_t i = 0; i < num_interpolate_slots; ++i) {
+               glTextureStorage3D(input_tex[i], levels, GL_RGBA8, width, height, 2);
+               check_error();
+               glTextureStorage3D(gray_tex[i], levels, GL_R8, width, height, 2);
+               check_error();
+
+               InterpolatedFrameResources resource;
+               resource.input_tex = input_tex[i];
+               resource.gray_tex = gray_tex[i];
+               glCreateFramebuffers(2, resource.input_fbos);
+               check_error();
+
+               glNamedFramebufferTextureLayer(resource.input_fbos[0], GL_COLOR_ATTACHMENT0, input_tex[i], 0, 0);
+               check_error();
+               glNamedFramebufferTextureLayer(resource.input_fbos[1], GL_COLOR_ATTACHMENT0, input_tex[i], 0, 1);
+               check_error();
+
+               GLuint buf = GL_COLOR_ATTACHMENT0;
+               glNamedFramebufferDrawBuffers(resource.input_fbos[0], 1, &buf);
+               check_error();
+               glNamedFramebufferDrawBuffers(resource.input_fbos[1], 1, &buf);
+               check_error();
+
+               glCreateBuffers(1, &resource.pbo);
+               check_error();
+               glNamedBufferStorage(resource.pbo, width * height * 4, nullptr, GL_MAP_READ_BIT | GL_MAP_PERSISTENT_BIT);
+               check_error();
+               resource.pbo_contents = glMapNamedBufferRange(resource.pbo, 0, width * height * 4, GL_MAP_READ_BIT | GL_MAP_PERSISTENT_BIT); 
+               interpolate_resources.push_back(resource);
+       }
+
+       check_error();
+
+       compute_flow.reset(new DISComputeFlow(width, height, operating_point3));
+       gray.reset(new GrayscaleConversion);  // NOTE: Must come after DISComputeFlow, since it sets up the VBO!
+       interpolate.reset(new Interpolate(width, height, operating_point3));
+       check_error();
+}
+
+VideoStream::~VideoStream() {}
+
 void VideoStream::start()
 {
        AVFormatContext *avctx = avformat_alloc_context();
@@ -68,17 +154,100 @@ void VideoStream::stop()
 
 void VideoStream::schedule_original_frame(int64_t output_pts, unsigned stream_idx, int64_t input_pts)
 {
-       unique_lock<mutex> lock(queue_lock);
        QueuedFrame qf;
+       qf.type = QueuedFrame::ORIGINAL;
        qf.output_pts = output_pts;
        qf.stream_idx = stream_idx;
        qf.input_first_pts = input_pts; 
+
+       unique_lock<mutex> lock(queue_lock);
+       frame_queue.push_back(qf);
+       queue_nonempty.notify_all();
+}
+
+void VideoStream::schedule_interpolated_frame(int64_t output_pts, unsigned stream_idx, int64_t input_first_pts, int64_t input_second_pts, float alpha)
+{
+       // Get the temporary OpenGL resources we need for doing the interpolation.
+       InterpolatedFrameResources resources;
+       {
+               unique_lock<mutex> lock(queue_lock);
+               if (interpolate_resources.empty()) {
+                       fprintf(stderr, "WARNING: Too many interpolated frames already in transit; dropping one.\n");
+                       return;
+               }
+               resources = interpolate_resources.front();
+               interpolate_resources.pop_front();
+       }
+
+       QueuedFrame qf;
+       qf.type = QueuedFrame::INTERPOLATED;
+       qf.output_pts = output_pts;
+       qf.stream_idx = stream_idx;
+       qf.resources = resources;
+
+       check_error();
+
+       // Convert frame0 and frame1 to OpenGL textures.
+       // TODO: Deduplicate against JPEGFrameView::setDecodedFrame?
+       for (size_t frame_no = 0; frame_no < 2; ++frame_no) {
+               shared_ptr<Frame> frame = decode_jpeg(filename_for_frame(stream_idx, frame_no == 1 ? input_second_pts : input_first_pts));
+               ycbcr_format.chroma_subsampling_x = frame->chroma_subsampling_x;
+               ycbcr_format.chroma_subsampling_y = frame->chroma_subsampling_y;
+               ycbcr_input->change_ycbcr_format(ycbcr_format);
+               ycbcr_input->set_width(frame->width);
+               ycbcr_input->set_height(frame->height);
+               ycbcr_input->set_pixel_data(0, frame->y.get());
+               ycbcr_input->set_pixel_data(1, frame->cb.get());
+               ycbcr_input->set_pixel_data(2, frame->cr.get());
+               ycbcr_input->set_pitch(0, frame->pitch_y);
+               ycbcr_input->set_pitch(1, frame->pitch_chroma);
+               ycbcr_input->set_pitch(2, frame->pitch_chroma);
+               ycbcr_convert_chain->render_to_fbo(resources.input_fbos[frame_no], 1280, 720);
+       }
+
+       glGenerateTextureMipmap(resources.input_tex);
+
+       // Compute the interpolated frame.
+       check_error();
+       gray->exec(resources.input_tex, resources.gray_tex, 1280, 720, /*num_layers=*/2);
+       check_error();
+       glGenerateTextureMipmap(resources.gray_tex);
+       check_error();
+       GLuint flow_tex = compute_flow->exec(resources.gray_tex, DISComputeFlow::FORWARD_AND_BACKWARD, DISComputeFlow::DO_NOT_RESIZE_FLOW);
+       check_error();
+
+       qf.output_tex = interpolate->exec(resources.input_tex, flow_tex, 1280, 720, alpha);
+       check_error();
+
+       // Read it down (asynchronously) to the CPU.
+       glPixelStorei(GL_PACK_ROW_LENGTH, 0);
+       glBindBuffer(GL_PIXEL_PACK_BUFFER, resources.pbo);
+       check_error();
+       glGetTextureImage(qf.output_tex, 0, GL_RGBA, GL_UNSIGNED_BYTE, 1280 * 720 * 4, nullptr);
+       check_error();
+       glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);
+
+       // Set a fence we can wait for to make sure the CPU sees the read.
+       glMemoryBarrier(GL_CLIENT_MAPPED_BUFFER_BARRIER_BIT);
+       check_error();
+       qf.fence = RefCountedGLsync(GL_SYNC_GPU_COMMANDS_COMPLETE, /*flags=*/0);
+       check_error();
+
+       unique_lock<mutex> lock(queue_lock);
        frame_queue.push_back(qf);
        queue_nonempty.notify_all();
 }
 
 void VideoStream::encode_thread_func()
 {
+       QSurface *surface = create_surface();
+       QOpenGLContext *context = create_context(surface);
+       bool ok = make_current(context, surface);
+       if (!ok) {
+               fprintf(stderr, "Video stream couldn't get an OpenGL context\n");
+               exit(1);
+       }
+
        for ( ;; ) {
                QueuedFrame qf;
                {
@@ -91,6 +260,7 @@ void VideoStream::encode_thread_func()
                }
 
                if (qf.type == QueuedFrame::ORIGINAL) {
+                       // Send the JPEG frame on, unchanged.
                        string jpeg = read_file(filename_for_frame(qf.stream_idx, qf.input_first_pts));
                        AVPacket pkt;
                        av_init_packet(&pkt);
@@ -98,7 +268,28 @@ void VideoStream::encode_thread_func()
                        pkt.data = (uint8_t *)jpeg.data();
                        pkt.size = jpeg.size();
                        stream_mux->add_packet(pkt, qf.output_pts, qf.output_pts);
-               }               
+               } else if (qf.type == QueuedFrame::INTERPOLATED) {
+                       glClientWaitSync(qf.fence.get(), /*flags=*/0, GL_TIMEOUT_IGNORED);
+
+                       // DEBUG: Writing the frame to disk.
+                       FILE *fp = fopen("inter.ppm", "wb");
+                       fprintf(fp, "P6\n%d %d\n255\n", 1280, 720);
+                       for (size_t y = 0; y < 720; ++y) {
+                               const uint8_t *ptr = (uint8_t *)qf.resources.pbo_contents + (719 - y) * 1280 * 4;
+                               for (size_t x = 0; x < 1280; ++x) {
+                                       putc(ptr[0], fp);
+                                       putc(ptr[1], fp);
+                                       putc(ptr[2], fp);
+                                       ptr += 4;
+                               }
+                       }
+                       fclose(fp);
+                       // TODO: Release flow and output textures.
+
+                       // Put the frame resources back.
+                       unique_lock<mutex> lock(queue_lock);
+                       interpolate_resources.push_back(qf.resources);
+               }
        }
 }
 
index 3a8a2f26968ecd9841226d6a406eeb9f350b7bae..953e28e661934b16105f429261494796344918b2 100644 (file)
@@ -14,10 +14,22 @@ extern "C" {
 #include <string>
 #include <thread>
 
+#include <movit/effect_chain.h>
+#include <movit/ycbcr_input.h>
+
+#include "ref_counted_gl_sync.h"
+
+class DISComputeFlow;
+class GrayscaleConversion;
+class Interpolate;
 class Mux;
+class QSurface;
+class QSurfaceFormat;
 
 class VideoStream {
 public:
+       VideoStream();
+       ~VideoStream();
        void start();
        void stop();
 
@@ -25,12 +37,25 @@ public:
        void schedule_interpolated_frame(int64_t output_pts, unsigned stream_idx, int64_t input_first_pts, int64_t input_second_pts, float alpha);
 
 private:
+
        void encode_thread_func();
        std::thread encode_thread;
 
        static int write_packet2_thunk(void *opaque, uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time);
        int write_packet2(uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time);
 
+       // Allocated at the very start; if we're empty, we start dropping frames
+       // (so that we don't build up an infinite interpolation backlog).
+       struct InterpolatedFrameResources {
+               GLuint input_tex;  // Layered (contains both input frames).
+               GLuint gray_tex;  // Same.
+               GLuint input_fbos[2];  // For rendering to the two layers of input_tex.
+               GLuint pbo;  // For reading the data back.
+               void *pbo_contents;  // Persistently mapped.
+       };
+       std::deque<InterpolatedFrameResources> interpolate_resources;  // Under <queue_lock>.
+       static constexpr size_t num_interpolate_slots = 10;
+
        struct QueuedFrame {
                int64_t output_pts;
                enum Type { ORIGINAL, INTERPOLATED } type;
@@ -40,8 +65,9 @@ private:
                // For interpolated frames only.
                int64_t input_second_pts;
                float alpha;
-               GLuint flow_tex;
-               GLuint fence;  // Set when the flow is done computing.
+               InterpolatedFrameResources resources;
+               GLuint output_tex;
+               RefCountedGLsync fence;  // Set when the interpolated image is read back to the CPU.
        };
        std::deque<QueuedFrame> frame_queue;  // Under <queue_lock>.
        std::mutex queue_lock;
@@ -50,6 +76,16 @@ private:
        std::unique_ptr<Mux> stream_mux;  // To HTTP.
        std::string stream_mux_header;
        bool seen_sync_markers = false;
+
+       QSurface *gl_surface;
+       std::unique_ptr<movit::EffectChain> ycbcr_convert_chain;  // TODO: Have a separate version with resample, for scaling?
+       movit::YCbCrInput *ycbcr_input;
+       movit::YCbCrFormat ycbcr_format;
+
+       // Frame interpolation.
+       std::unique_ptr<GrayscaleConversion> gray;
+       std::unique_ptr<DISComputeFlow> compute_flow;
+       std::unique_ptr<Interpolate> interpolate;
 };
 
 #endif  // !defined(_VIDEO_STREAM_H)