From e6f5343ffeaab71cee601f1a78937525e1947fef Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Sun, 9 Sep 2018 21:21:39 +0200 Subject: [PATCH] Start hacking in support for interpolated frames in the main application. --- Makefile | 5 +- context.cpp | 68 +++++++++++++++ context.h | 17 ++++ jpeg_frame_view.cpp | 4 +- jpeg_frame_view.h | 5 +- main.cpp | 25 +++++- mainwindow.cpp | 4 +- player.cpp | 57 ++++++++++-- player.h | 14 ++- ref_counted_gl_sync.h | 39 +++++++++ video_stream.cpp | 195 +++++++++++++++++++++++++++++++++++++++++- video_stream.h | 40 ++++++++- 12 files changed, 444 insertions(+), 29 deletions(-) create mode 100644 context.cpp create mode 100644 context.h create mode 100644 ref_counted_gl_sync.h diff --git a/Makefile b/Makefile index 4b0e275..a7ae389 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,10 @@ OBJS_WITH_MOC = mainwindow.o jpeg_frame_view.o clip_list.o OBJS += $(OBJS_WITH_MOC) OBJS += $(OBJS_WITH_MOC:.o=.moc.o) -OBJS += ffmpeg_raii.o main.o player.o httpd.o mux.o metacube2.o video_stream.o +# Flow objects +OBJS += flow.o gpu_timers.o + +OBJS += ffmpeg_raii.o main.o player.o httpd.o mux.o metacube2.o video_stream.o context.o %.o: %.cpp $(CXX) -MMD -MP $(CPPFLAGS) $(CXXFLAGS) -o $@ -c $< diff --git a/context.cpp b/context.cpp new file mode 100644 index 0000000..ac8c97b --- /dev/null +++ b/context.cpp @@ -0,0 +1,68 @@ +#include + +#include + +#include +#include +#include +#include +#include + +QGLWidget *global_share_widget = nullptr; + +using namespace std; + +QSurface *create_surface() +{ + QSurfaceFormat fmt; + fmt.setDepthBufferSize(0); + fmt.setStencilBufferSize(0); + fmt.setProfile(QSurfaceFormat::CoreProfile); + fmt.setMajorVersion(4); + fmt.setMinorVersion(5); + fmt.setSwapInterval(0); + QOffscreenSurface *surface = new QOffscreenSurface; + surface->setFormat(fmt); + surface->create(); + if (!surface->isValid()) { + fprintf(stderr, "ERROR: surface not valid!\n"); + exit(1); + } + return surface; +} + +QSurface *create_surface(const QSurfaceFormat &format) +{ + QOffscreenSurface *surface = new QOffscreenSurface; + surface->setFormat(format); + surface->create(); + if (!surface->isValid()) { + fprintf(stderr, "ERROR: surface not valid!\n"); + exit(1); + } + return surface; +} + +QSurface *create_surface_with_same_format(const QSurface *surface) +{ + return create_surface(surface->format()); +} + +QOpenGLContext *create_context(const QSurface *surface) +{ + QOpenGLContext *context = new QOpenGLContext; + context->setShareContext(global_share_widget->context()->contextHandle()); + context->setFormat(surface->format()); + context->create(); + return context; +} + +bool make_current(QOpenGLContext *context, QSurface *surface) +{ + return context->makeCurrent(surface); +} + +void delete_context(QOpenGLContext *context) +{ + delete context; +} diff --git a/context.h b/context.h new file mode 100644 index 0000000..aebba96 --- /dev/null +++ b/context.h @@ -0,0 +1,17 @@ + +// Needs to be in its own file because Qt and libepoxy seemingly don't coexist well +// within the same file. + +class QSurface; +class QOpenGLContext; +class QSurfaceFormat; +class QGLWidget; + +extern bool using_egl; +extern QGLWidget *global_share_widget; +QSurface *create_surface(); +QSurface *create_surface(const QSurfaceFormat &format); +QSurface *create_surface_with_same_format(const QSurface *surface); +QOpenGLContext *create_context(const QSurface *surface); +bool make_current(QOpenGLContext *context, QSurface *surface); +void delete_context(QOpenGLContext *context); diff --git a/jpeg_frame_view.cpp b/jpeg_frame_view.cpp index 16224ad..3d35e5e 100644 --- a/jpeg_frame_view.cpp +++ b/jpeg_frame_view.cpp @@ -16,6 +16,7 @@ #include "defs.h" #include "post_to_main_thread.h" +#include "video_stream.h" using namespace movit; using namespace std; @@ -221,14 +222,11 @@ void JPEGFrameView::initializeGL() { glDisable(GL_BLEND); glDisable(GL_DEPTH_TEST); - glDepthMask(GL_FALSE); check_error(); static once_flag once; call_once(once, [] { - CHECK(init_movit(MOVIT_SHADER_DIR, MOVIT_DEBUG_OFF)); resource_pool = new ResourcePool; - std::thread(&jpeg_decoder_thread).detach(); }); diff --git a/jpeg_frame_view.h b/jpeg_frame_view.h index 277241b..1f7b2b7 100644 --- a/jpeg_frame_view.h +++ b/jpeg_frame_view.h @@ -11,8 +11,6 @@ #include -std::string filename_for_frame(unsigned stream_idx, int64_t pts); - struct Frame { std::unique_ptr y, cb, cr; unsigned width, height; @@ -20,6 +18,9 @@ struct Frame { unsigned pitch_y, pitch_chroma; }; +std::string filename_for_frame(unsigned stream_idx, int64_t pts); +std::shared_ptr decode_jpeg(const std::string &filename); + class JPEGFrameView : public QGLWidget { Q_OBJECT diff --git a/main.cpp b/main.cpp index e8c0dc4..ccfb281 100644 --- a/main.cpp +++ b/main.cpp @@ -16,18 +16,25 @@ extern "C" { #include +#include +#include + #include "clip_list.h" +#include "context.h" #include "defs.h" #include "mainwindow.h" #include "ffmpeg_raii.h" #include "httpd.h" #include "player.h" #include "post_to_main_thread.h" +#include "ref_counted_gl_sync.h" #include "ui_mainwindow.h" using namespace std; using namespace std::chrono; +std::mutex RefCountedGLsync::fence_lock; + // TODO: Replace by some sort of GUI control, I guess. int64_t current_pts = 0; @@ -40,7 +47,6 @@ string filename_for_frame(unsigned stream_idx, int64_t pts) mutex frame_mu; vector frames[MAX_STREAMS]; -QGLWidget *global_share_widget; HTTPD *global_httpd; int record_thread_func(); @@ -57,8 +63,8 @@ int main(int argc, char **argv) fmt.setDepthBufferSize(0); fmt.setStencilBufferSize(0); fmt.setProfile(QSurfaceFormat::CoreProfile); - fmt.setMajorVersion(3); - fmt.setMinorVersion(1); + fmt.setMajorVersion(4); + fmt.setMinorVersion(5); // Turn off vsync, since Qt generally gives us at most frame rate // (display frequency) / (number of QGLWidgets active). @@ -71,9 +77,20 @@ int main(int argc, char **argv) QApplication app(argc, argv); global_share_widget = new QGLWidget(); if (!global_share_widget->isValid()) { - fprintf(stderr, "Failed to initialize OpenGL. Futatabi needs at least OpenGL 3.1 to function properly.\n"); + fprintf(stderr, "Failed to initialize OpenGL. Futatabi needs at least OpenGL 4.5 to function properly.\n"); exit(1); } + + // Initialize Movit. + { + QSurface *surface = create_surface(); + QOpenGLContext *context = create_context(surface); + make_current(context, surface); + CHECK(movit::init_movit(MOVIT_SHADER_DIR, movit::MOVIT_DEBUG_OFF)); + delete_context(context); + // TODO: Delete the surface, too. + } + MainWindow mainWindow; mainWindow.show(); diff --git a/mainwindow.cpp b/mainwindow.cpp index 3b8561f..0e73b99 100644 --- a/mainwindow.cpp +++ b/mainwindow.cpp @@ -87,8 +87,8 @@ MainWindow::MainWindow() this, &MainWindow::playlist_selection_changed); playlist_selection_changed(); // First time set-up. - preview_player = new Player(ui->preview_display); - live_player = new Player(ui->live_display); + preview_player = new Player(ui->preview_display, /*also_output_to_stream=*/false); + live_player = new Player(ui->live_display, /*also_output_to_stream=*/true); live_player->set_done_callback([this]{ post_to_main_thread([this]{ live_player_clip_done(); diff --git a/player.cpp b/player.cpp index 1533977..970f340 100644 --- a/player.cpp +++ b/player.cpp @@ -7,13 +7,17 @@ #include +#include + #include "clip_list.h" +#include "context.h" #include "defs.h" #include "ffmpeg_raii.h" #include "httpd.h" #include "jpeg_frame_view.h" #include "mux.h" #include "player.h" +#include "video_stream.h" using namespace std; using namespace std::chrono; @@ -22,8 +26,25 @@ extern mutex frame_mu; extern vector frames[MAX_STREAMS]; extern HTTPD *global_httpd; -void Player::thread_func() +void Player::thread_func(bool also_output_to_stream) { + QSurface *surface = create_surface(); + QOpenGLContext *context = create_context(surface); + if (!make_current(context, surface)) { + printf("oops\n"); + exit(1); + } + + check_error(); + + // Create the VideoStream object, now that we have an OpenGL context. + if (also_output_to_stream) { + video_stream.reset(new VideoStream); + video_stream->start(); + } + + check_error(); + for ( ;; ) { // Wait until we're supposed to play something. { @@ -82,11 +103,30 @@ void Player::thread_func() destination->setFrame(stream_idx, next_pts); - // Send the frame to the stream. - // FIXME: Vaguely less crazy pts, perhaps. - double pts_float = fmod(duration(next_frame_start.time_since_epoch()).count(), 86400.0f); - int64_t pts = lrint(pts_float * TIMEBASE); - video_stream.schedule_original_frame(pts, stream_idx, next_pts); + if (video_stream != nullptr) { + // Send the frame to the stream. + // FIXME: Vaguely less crazy pts, perhaps. + double pts_float = fmod(duration(next_frame_start.time_since_epoch()).count(), 86400.0f); + int64_t pts = lrint(pts_float * TIMEBASE); + video_stream->schedule_original_frame(pts, stream_idx, next_pts); + + // HACK: test interpolation by frame-doubling. + int64_t next_next_pts = -1; + { + lock_guard lock(frame_mu); + auto it = upper_bound(frames[stream_idx].begin(), + frames[stream_idx].end(), + next_pts); + if (it == frames[stream_idx].end() || *it >= clip.pts_out) { + break; + } + next_next_pts = *it; + } + if (next_next_pts != -1) { + int64_t interpolated_pts = (next_pts + next_next_pts) / 2; + video_stream->schedule_interpolated_frame(interpolated_pts, stream_idx, next_pts, next_next_pts, 0.5f); + } + } } { @@ -99,11 +139,10 @@ void Player::thread_func() } } -Player::Player(JPEGFrameView *destination) +Player::Player(JPEGFrameView *destination, bool also_output_to_stream) : destination(destination) { - video_stream.start(); - thread(&Player::thread_func, this).detach(); + thread(&Player::thread_func, this, also_output_to_stream).detach(); } void Player::play_clip(const Clip &clip, unsigned stream_idx) diff --git a/player.h b/player.h index 3321a30..10e0b6a 100644 --- a/player.h +++ b/player.h @@ -2,17 +2,23 @@ #define _PLAYER_H 1 #include "clip_list.h" -#include "video_stream.h" + +extern "C" { +#include +} #include #include #include class JPEGFrameView; +class VideoStream; +class QSurface; +class QSurfaceFormat; class Player { public: - Player(JPEGFrameView *destination); + Player(JPEGFrameView *destination, bool also_output_to_stream); void play_clip(const Clip &clip, unsigned stream_idx); void override_angle(unsigned stream_idx); // For the current clip only. @@ -23,7 +29,7 @@ public: void set_done_callback(done_callback_func cb) { done_callback = cb; } private: - void thread_func(); + void thread_func(bool also_output_to_stream); void open_output_stream(); static int write_packet2_thunk(void *opaque, uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time); int write_packet2(uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time); @@ -41,7 +47,7 @@ private: bool playing = false; // Under queue_state_mu. int override_stream_idx = -1; // Under queue_state_mu. - VideoStream video_stream; + std::unique_ptr video_stream; // Can be nullptr. }; #endif // !defined(_PLAYER_H) diff --git a/ref_counted_gl_sync.h b/ref_counted_gl_sync.h new file mode 100644 index 0000000..8b6db68 --- /dev/null +++ b/ref_counted_gl_sync.h @@ -0,0 +1,39 @@ +#ifndef _REF_COUNTED_GL_SYNC_H +#define _REF_COUNTED_GL_SYNC_H 1 + +// A wrapper around GLsync (OpenGL fences) that is automatically refcounted. +// Useful since we sometimes want to use the same fence two entirely different +// places. (We could set two fences at the same time, but they are not an +// unlimited hardware resource, so it would be a bit wasteful.) + +#include +#include +#include + +typedef std::shared_ptr<__GLsync> RefCountedGLsyncBase; + +class RefCountedGLsync : public RefCountedGLsyncBase { +public: + RefCountedGLsync() {} + + RefCountedGLsync(GLenum condition, GLbitfield flags) + : RefCountedGLsyncBase(locked_glFenceSync(condition, flags), glDeleteSync) {} + +private: + // These are to work around apitrace bug #446. + static GLsync locked_glFenceSync(GLenum condition, GLbitfield flags) + { + std::lock_guard lock(fence_lock); + return glFenceSync(condition, flags); + } + + static void locked_glDeleteSync(GLsync sync) + { + std::lock_guard lock(fence_lock); + glDeleteSync(sync); + } + + static std::mutex fence_lock; +}; + +#endif // !defined(_REF_COUNTED_GL_SYNC_H) diff --git a/video_stream.cpp b/video_stream.cpp index 65f52c0..a783bdb 100644 --- a/video_stream.cpp +++ b/video_stream.cpp @@ -5,10 +5,18 @@ extern "C" { #include } +#include + +#include "context.h" +#include "flow.h" #include "httpd.h" #include "jpeg_frame_view.h" +#include "movit/util.h" #include "mux.h" #include "player.h" +#include "util.h" + +#include using namespace std; @@ -37,6 +45,84 @@ string read_file(const string &filename) } // namespace +VideoStream::VideoStream() +{ + using namespace movit; + // TODO: deduplicate code against JPEGFrameView? + ycbcr_convert_chain.reset(new EffectChain(1280, 720)); + ImageFormat image_format; + image_format.color_space = COLORSPACE_sRGB; + image_format.gamma_curve = GAMMA_sRGB; + ycbcr_format.luma_coefficients = YCBCR_REC_709; + ycbcr_format.full_range = false; + ycbcr_format.num_levels = 256; + ycbcr_format.chroma_subsampling_x = 2; + ycbcr_format.chroma_subsampling_y = 1; + ycbcr_format.cb_x_position = 0.0f; // H.264 -- _not_ JPEG, even though our input is MJPEG-encoded + ycbcr_format.cb_y_position = 0.5f; // Irrelevant. + ycbcr_format.cr_x_position = 0.0f; + ycbcr_format.cr_y_position = 0.5f; + ycbcr_input = (movit::YCbCrInput *)ycbcr_convert_chain->add_input(new YCbCrInput(image_format, ycbcr_format, 1280, 720)); + + ImageFormat inout_format; + inout_format.color_space = COLORSPACE_sRGB; + inout_format.gamma_curve = GAMMA_sRGB; + + check_error(); + ycbcr_convert_chain->add_output(inout_format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED); + check_error(); + ycbcr_convert_chain->set_dither_bits(8); + check_error(); + ycbcr_convert_chain->finalize(); + check_error(); + + GLuint input_tex[num_interpolate_slots], gray_tex[num_interpolate_slots]; + glCreateTextures(GL_TEXTURE_2D_ARRAY, 10, input_tex); + glCreateTextures(GL_TEXTURE_2D_ARRAY, 10, gray_tex); + check_error(); + constexpr size_t width = 1280, height = 720; // FIXME: adjustable width, height + int levels = find_num_levels(width, height); + for (size_t i = 0; i < num_interpolate_slots; ++i) { + glTextureStorage3D(input_tex[i], levels, GL_RGBA8, width, height, 2); + check_error(); + glTextureStorage3D(gray_tex[i], levels, GL_R8, width, height, 2); + check_error(); + + InterpolatedFrameResources resource; + resource.input_tex = input_tex[i]; + resource.gray_tex = gray_tex[i]; + glCreateFramebuffers(2, resource.input_fbos); + check_error(); + + glNamedFramebufferTextureLayer(resource.input_fbos[0], GL_COLOR_ATTACHMENT0, input_tex[i], 0, 0); + check_error(); + glNamedFramebufferTextureLayer(resource.input_fbos[1], GL_COLOR_ATTACHMENT0, input_tex[i], 0, 1); + check_error(); + + GLuint buf = GL_COLOR_ATTACHMENT0; + glNamedFramebufferDrawBuffers(resource.input_fbos[0], 1, &buf); + check_error(); + glNamedFramebufferDrawBuffers(resource.input_fbos[1], 1, &buf); + check_error(); + + glCreateBuffers(1, &resource.pbo); + check_error(); + glNamedBufferStorage(resource.pbo, width * height * 4, nullptr, GL_MAP_READ_BIT | GL_MAP_PERSISTENT_BIT); + check_error(); + resource.pbo_contents = glMapNamedBufferRange(resource.pbo, 0, width * height * 4, GL_MAP_READ_BIT | GL_MAP_PERSISTENT_BIT); + interpolate_resources.push_back(resource); + } + + check_error(); + + compute_flow.reset(new DISComputeFlow(width, height, operating_point3)); + gray.reset(new GrayscaleConversion); // NOTE: Must come after DISComputeFlow, since it sets up the VBO! + interpolate.reset(new Interpolate(width, height, operating_point3)); + check_error(); +} + +VideoStream::~VideoStream() {} + void VideoStream::start() { AVFormatContext *avctx = avformat_alloc_context(); @@ -68,17 +154,100 @@ void VideoStream::stop() void VideoStream::schedule_original_frame(int64_t output_pts, unsigned stream_idx, int64_t input_pts) { - unique_lock lock(queue_lock); QueuedFrame qf; + qf.type = QueuedFrame::ORIGINAL; qf.output_pts = output_pts; qf.stream_idx = stream_idx; qf.input_first_pts = input_pts; + + unique_lock lock(queue_lock); + frame_queue.push_back(qf); + queue_nonempty.notify_all(); +} + +void VideoStream::schedule_interpolated_frame(int64_t output_pts, unsigned stream_idx, int64_t input_first_pts, int64_t input_second_pts, float alpha) +{ + // Get the temporary OpenGL resources we need for doing the interpolation. + InterpolatedFrameResources resources; + { + unique_lock lock(queue_lock); + if (interpolate_resources.empty()) { + fprintf(stderr, "WARNING: Too many interpolated frames already in transit; dropping one.\n"); + return; + } + resources = interpolate_resources.front(); + interpolate_resources.pop_front(); + } + + QueuedFrame qf; + qf.type = QueuedFrame::INTERPOLATED; + qf.output_pts = output_pts; + qf.stream_idx = stream_idx; + qf.resources = resources; + + check_error(); + + // Convert frame0 and frame1 to OpenGL textures. + // TODO: Deduplicate against JPEGFrameView::setDecodedFrame? + for (size_t frame_no = 0; frame_no < 2; ++frame_no) { + shared_ptr frame = decode_jpeg(filename_for_frame(stream_idx, frame_no == 1 ? input_second_pts : input_first_pts)); + ycbcr_format.chroma_subsampling_x = frame->chroma_subsampling_x; + ycbcr_format.chroma_subsampling_y = frame->chroma_subsampling_y; + ycbcr_input->change_ycbcr_format(ycbcr_format); + ycbcr_input->set_width(frame->width); + ycbcr_input->set_height(frame->height); + ycbcr_input->set_pixel_data(0, frame->y.get()); + ycbcr_input->set_pixel_data(1, frame->cb.get()); + ycbcr_input->set_pixel_data(2, frame->cr.get()); + ycbcr_input->set_pitch(0, frame->pitch_y); + ycbcr_input->set_pitch(1, frame->pitch_chroma); + ycbcr_input->set_pitch(2, frame->pitch_chroma); + ycbcr_convert_chain->render_to_fbo(resources.input_fbos[frame_no], 1280, 720); + } + + glGenerateTextureMipmap(resources.input_tex); + + // Compute the interpolated frame. + check_error(); + gray->exec(resources.input_tex, resources.gray_tex, 1280, 720, /*num_layers=*/2); + check_error(); + glGenerateTextureMipmap(resources.gray_tex); + check_error(); + GLuint flow_tex = compute_flow->exec(resources.gray_tex, DISComputeFlow::FORWARD_AND_BACKWARD, DISComputeFlow::DO_NOT_RESIZE_FLOW); + check_error(); + + qf.output_tex = interpolate->exec(resources.input_tex, flow_tex, 1280, 720, alpha); + check_error(); + + // Read it down (asynchronously) to the CPU. + glPixelStorei(GL_PACK_ROW_LENGTH, 0); + glBindBuffer(GL_PIXEL_PACK_BUFFER, resources.pbo); + check_error(); + glGetTextureImage(qf.output_tex, 0, GL_RGBA, GL_UNSIGNED_BYTE, 1280 * 720 * 4, nullptr); + check_error(); + glBindBuffer(GL_PIXEL_PACK_BUFFER, 0); + + // Set a fence we can wait for to make sure the CPU sees the read. + glMemoryBarrier(GL_CLIENT_MAPPED_BUFFER_BARRIER_BIT); + check_error(); + qf.fence = RefCountedGLsync(GL_SYNC_GPU_COMMANDS_COMPLETE, /*flags=*/0); + check_error(); + + unique_lock lock(queue_lock); frame_queue.push_back(qf); queue_nonempty.notify_all(); } void VideoStream::encode_thread_func() { + QSurface *surface = create_surface(); + QOpenGLContext *context = create_context(surface); + bool ok = make_current(context, surface); + if (!ok) { + fprintf(stderr, "Video stream couldn't get an OpenGL context\n"); + exit(1); + } + for ( ;; ) { QueuedFrame qf; { @@ -91,6 +260,7 @@ void VideoStream::encode_thread_func() } if (qf.type == QueuedFrame::ORIGINAL) { + // Send the JPEG frame on, unchanged. string jpeg = read_file(filename_for_frame(qf.stream_idx, qf.input_first_pts)); AVPacket pkt; av_init_packet(&pkt); @@ -98,7 +268,28 @@ void VideoStream::encode_thread_func() pkt.data = (uint8_t *)jpeg.data(); pkt.size = jpeg.size(); stream_mux->add_packet(pkt, qf.output_pts, qf.output_pts); - } + } else if (qf.type == QueuedFrame::INTERPOLATED) { + glClientWaitSync(qf.fence.get(), /*flags=*/0, GL_TIMEOUT_IGNORED); + + // DEBUG: Writing the frame to disk. + FILE *fp = fopen("inter.ppm", "wb"); + fprintf(fp, "P6\n%d %d\n255\n", 1280, 720); + for (size_t y = 0; y < 720; ++y) { + const uint8_t *ptr = (uint8_t *)qf.resources.pbo_contents + (719 - y) * 1280 * 4; + for (size_t x = 0; x < 1280; ++x) { + putc(ptr[0], fp); + putc(ptr[1], fp); + putc(ptr[2], fp); + ptr += 4; + } + } + fclose(fp); + // TODO: Release flow and output textures. + + // Put the frame resources back. + unique_lock lock(queue_lock); + interpolate_resources.push_back(qf.resources); + } } } diff --git a/video_stream.h b/video_stream.h index 3a8a2f2..953e28e 100644 --- a/video_stream.h +++ b/video_stream.h @@ -14,10 +14,22 @@ extern "C" { #include #include +#include +#include + +#include "ref_counted_gl_sync.h" + +class DISComputeFlow; +class GrayscaleConversion; +class Interpolate; class Mux; +class QSurface; +class QSurfaceFormat; class VideoStream { public: + VideoStream(); + ~VideoStream(); void start(); void stop(); @@ -25,12 +37,25 @@ public: void schedule_interpolated_frame(int64_t output_pts, unsigned stream_idx, int64_t input_first_pts, int64_t input_second_pts, float alpha); private: + void encode_thread_func(); std::thread encode_thread; static int write_packet2_thunk(void *opaque, uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time); int write_packet2(uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time); + // Allocated at the very start; if we're empty, we start dropping frames + // (so that we don't build up an infinite interpolation backlog). + struct InterpolatedFrameResources { + GLuint input_tex; // Layered (contains both input frames). + GLuint gray_tex; // Same. + GLuint input_fbos[2]; // For rendering to the two layers of input_tex. + GLuint pbo; // For reading the data back. + void *pbo_contents; // Persistently mapped. + }; + std::deque interpolate_resources; // Under . + static constexpr size_t num_interpolate_slots = 10; + struct QueuedFrame { int64_t output_pts; enum Type { ORIGINAL, INTERPOLATED } type; @@ -40,8 +65,9 @@ private: // For interpolated frames only. int64_t input_second_pts; float alpha; - GLuint flow_tex; - GLuint fence; // Set when the flow is done computing. + InterpolatedFrameResources resources; + GLuint output_tex; + RefCountedGLsync fence; // Set when the interpolated image is read back to the CPU. }; std::deque frame_queue; // Under . std::mutex queue_lock; @@ -50,6 +76,16 @@ private: std::unique_ptr stream_mux; // To HTTP. std::string stream_mux_header; bool seen_sync_markers = false; + + QSurface *gl_surface; + std::unique_ptr ycbcr_convert_chain; // TODO: Have a separate version with resample, for scaling? + movit::YCbCrInput *ycbcr_input; + movit::YCbCrFormat ycbcr_format; + + // Frame interpolation. + std::unique_ptr gray; + std::unique_ptr compute_flow; + std::unique_ptr interpolate; }; #endif // !defined(_VIDEO_STREAM_H) -- 2.39.2