From 393a2a974ea26d9eac26b52aabc12f48b0dbfeb2 Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Fri, 21 Sep 2018 00:48:13 +0200 Subject: [PATCH] Show interpolated frames in the live window. --- jpeg_frame_view.cpp | 66 ++++++++++++++++++++++++++++++++++++++------- jpeg_frame_view.h | 5 +++- main.cpp | 8 +++--- player.cpp | 18 ++++++++----- video_stream.cpp | 31 +++++++++++++++++---- 5 files changed, 102 insertions(+), 26 deletions(-) diff --git a/jpeg_frame_view.cpp b/jpeg_frame_view.cpp index 88a88dc..54a3730 100644 --- a/jpeg_frame_view.cpp +++ b/jpeg_frame_view.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -23,9 +24,18 @@ using namespace movit; using namespace std; -bool operator< (const JPEGID &a, const JPEGID &b) { - return make_pair(a.stream_idx, a.pts) < make_pair(b.stream_idx, b.pts); -} +// Just an arbitrary order for std::map. +struct JPEGIDLexicalOrder +{ + bool operator() (const JPEGID &a, const JPEGID &b) const + { + if (a.stream_idx != b.stream_idx) + return a.stream_idx < b.stream_idx; + if (a.pts != b.pts) + return a.pts < b.pts; + return a.interpolated < b.interpolated; + } +}; struct LRUFrame { shared_ptr frame; @@ -33,8 +43,8 @@ struct LRUFrame { }; mutex cache_mu; -map cache; // Under cache_mu. -condition_variable any_pending_decodes; +map cache; // Under cache_mu. +condition_variable any_pending_decodes, cache_updated; deque> pending_decodes; // Under cache_mu. atomic event_counter{0}; extern QGLWidget *global_share_widget; @@ -160,6 +170,7 @@ shared_ptr decode_jpeg_with_cache(JPEGID id, CacheMissBehavior cache_miss return nullptr; } + assert(!id.interpolated); *did_decode = true; shared_ptr frame = decode_jpeg(filename_for_frame(id.stream_idx, id.pts)); @@ -202,10 +213,33 @@ void jpeg_decoder_thread() } bool found_in_cache; - shared_ptr frame = decode_jpeg_with_cache(id, cache_miss_behavior, &found_in_cache); + shared_ptr frame; + if (id.interpolated) { + // Interpolated frames are never decoded by us, + // put directly into the cache from VideoStream. + unique_lock lock(cache_mu); + cache_updated.wait(lock, [id] { + return cache.count(id) != 0; + }); + found_in_cache = true; // Don't count it as a decode. + + auto it = cache.find(id); + assert(it != cache.end()); + + it->second.last_used = event_counter++; + frame = it->second.frame; + if (frame == nullptr) { + // We inserted a nullptr as signal that the frame was never + // interpolated and that we should stop waiting. + // But don't let it linger in the cache anymore. + cache.erase(it); + } + } else { + frame = decode_jpeg_with_cache(id, cache_miss_behavior, &found_in_cache); + } if (frame == nullptr) { - assert(cache_miss_behavior == RETURN_NULLPTR_IF_NOT_IN_CACHE); + assert(id.interpolated || cache_miss_behavior == RETURN_NULLPTR_IF_NOT_IN_CACHE); ++num_dropped; continue; } @@ -218,6 +252,7 @@ void jpeg_decoder_thread() } } + // TODO: Could we get jitter between non-interpolated and interpolated frames here? dest->setDecodedFrame(frame); } } @@ -226,15 +261,28 @@ JPEGFrameView::JPEGFrameView(QWidget *parent) : QGLWidget(parent, global_share_widget) { } -void JPEGFrameView::setFrame(unsigned stream_idx, int64_t pts) +void JPEGFrameView::setFrame(unsigned stream_idx, int64_t pts, bool interpolated) { current_stream_idx = stream_idx; unique_lock lock(cache_mu); - pending_decodes.emplace_back(JPEGID{ stream_idx, pts }, this); + pending_decodes.emplace_back(JPEGID{ stream_idx, pts, interpolated }, this); any_pending_decodes.notify_all(); } +void JPEGFrameView::insert_interpolated_frame(unsigned stream_idx, int64_t pts, shared_ptr frame) +{ + JPEGID id{ stream_idx, pts, true }; + + // We rely on the frame not being evicted from the cache before + // jpeg_decoder_thread() sees it and can display it (otherwise, + // that thread would hang). With a default cache of 1000 elements, + // that would sound like a reasonable assumption. + unique_lock lock(cache_mu); + cache[id] = LRUFrame{ std::move(frame), event_counter++ }; + cache_updated.notify_all(); +} + ResourcePool *resource_pool = nullptr; void JPEGFrameView::initializeGL() diff --git a/jpeg_frame_view.h b/jpeg_frame_view.h index 567da2d..591ff13 100644 --- a/jpeg_frame_view.h +++ b/jpeg_frame_view.h @@ -14,6 +14,7 @@ struct JPEGID { unsigned stream_idx; int64_t pts; + bool interpolated; }; struct Frame { std::unique_ptr y, cb, cr; @@ -36,7 +37,8 @@ class JPEGFrameView : public QGLWidget { public: JPEGFrameView(QWidget *parent); - void setFrame(unsigned stream_idx, int64_t pts); + void setFrame(unsigned stream_idx, int64_t pts, bool interpolated); + static void insert_interpolated_frame(unsigned stream_idx, int64_t pts, std::shared_ptr frame); void mousePressEvent(QMouseEvent *event) override; @@ -44,6 +46,7 @@ public: void setDecodedFrame(std::shared_ptr frame); + signals: void clicked(); diff --git a/main.cpp b/main.cpp index fc74285..dcb6f10 100644 --- a/main.cpp +++ b/main.cpp @@ -139,13 +139,13 @@ int record_thread_func() post_to_main_thread([pkt] { if (pkt.stream_index == 0) { - global_mainwindow->ui->input1_display->setFrame(pkt.stream_index, pkt.pts); + global_mainwindow->ui->input1_display->setFrame(pkt.stream_index, pkt.pts, /*interpolated=*/false); } else if (pkt.stream_index == 1) { - global_mainwindow->ui->input2_display->setFrame(pkt.stream_index, pkt.pts); + global_mainwindow->ui->input2_display->setFrame(pkt.stream_index, pkt.pts, /*interpolated=*/false); } else if (pkt.stream_index == 2) { - global_mainwindow->ui->input3_display->setFrame(pkt.stream_index, pkt.pts); + global_mainwindow->ui->input3_display->setFrame(pkt.stream_index, pkt.pts, /*interpolated=*/false); } else if (pkt.stream_index == 3) { - global_mainwindow->ui->input4_display->setFrame(pkt.stream_index, pkt.pts); + global_mainwindow->ui->input4_display->setFrame(pkt.stream_index, pkt.pts, /*interpolated=*/false); } }); diff --git a/player.cpp b/player.cpp index 136da21..5ff448c 100644 --- a/player.cpp +++ b/player.cpp @@ -137,7 +137,7 @@ void Player::thread_func(bool also_output_to_stream) } if (in_pts_lower == in_pts_upper) { - destination->setFrame(stream_idx, in_pts_lower); + destination->setFrame(stream_idx, in_pts_lower, /*interpolated=*/false); if (video_stream != nullptr) { video_stream->schedule_original_frame(lrint(out_pts), stream_idx, in_pts_lower); } @@ -149,14 +149,14 @@ void Player::thread_func(bool also_output_to_stream) double in_pts_lower_as_frameno = (in_pts_lower - in_pts_origin) * output_framerate / TIMEBASE / speed; double in_pts_upper_as_frameno = (in_pts_upper - in_pts_origin) * output_framerate / TIMEBASE / speed; if (fabs(in_pts_lower_as_frameno - frameno) < 0.01) { - destination->setFrame(stream_idx, in_pts_lower); + destination->setFrame(stream_idx, in_pts_lower, /*interpolated=*/false); if (video_stream != nullptr) { video_stream->schedule_original_frame(lrint(out_pts), stream_idx, in_pts_lower); } in_pts_origin += in_pts_lower - in_pts; continue; } else if (fabs(in_pts_upper_as_frameno - frameno) < 0.01) { - destination->setFrame(stream_idx, in_pts_upper); + destination->setFrame(stream_idx, in_pts_upper, /*interpolated=*/false); if (video_stream != nullptr) { video_stream->schedule_original_frame(lrint(out_pts), stream_idx, in_pts_upper); } @@ -165,10 +165,14 @@ void Player::thread_func(bool also_output_to_stream) } double alpha = double(in_pts - in_pts_lower) / (in_pts_upper - in_pts_lower); - destination->setFrame(stream_idx, in_pts_lower); // FIXME - if (video_stream != nullptr) { - // Send the frame to the stream. + if (video_stream == nullptr) { + // Previews don't do any interpolation. + destination->setFrame(stream_idx, in_pts_lower, /*interpolated=*/false); + } else { + // Calculate the interpolated frame. When it's done, the destination + // will be unblocked. + destination->setFrame(stream_idx, lrint(out_pts), /*interpolated=*/true); video_stream->schedule_interpolated_frame(lrint(out_pts), stream_idx, in_pts_lower, in_pts_upper, alpha); } } @@ -244,5 +248,5 @@ void Player::override_angle(unsigned stream_idx) if (it == frames[stream_idx].end()) { return; } - destination->setFrame(stream_idx, *it); + destination->setFrame(stream_idx, *it, /*interpolated=*/false); } diff --git a/video_stream.cpp b/video_stream.cpp index 27f0b2b..a733d46 100644 --- a/video_stream.cpp +++ b/video_stream.cpp @@ -299,6 +299,7 @@ void VideoStream::schedule_interpolated_frame(int64_t output_pts, unsigned strea unique_lock lock(queue_lock); if (interpolate_resources.empty()) { fprintf(stderr, "WARNING: Too many interpolated frames already in transit; dropping one.\n"); + JPEGFrameView::insert_interpolated_frame(stream_idx, output_pts, nullptr); return; } resources = interpolate_resources.front(); @@ -319,6 +320,7 @@ void VideoStream::schedule_interpolated_frame(int64_t output_pts, unsigned strea JPEGID jpeg_id; jpeg_id.stream_idx = stream_idx; jpeg_id.pts = frame_no == 1 ? input_second_pts : input_first_pts; + jpeg_id.interpolated = false; bool did_decode; shared_ptr frame = decode_jpeg_with_cache(jpeg_id, DECODE_IF_NOT_IN_CACHE, &did_decode); ycbcr_format.chroma_subsampling_x = frame->chroma_subsampling_x; @@ -410,11 +412,30 @@ void VideoStream::encode_thread_func() } else if (qf.type == QueuedFrame::INTERPOLATED) { glClientWaitSync(qf.fence.get(), /*flags=*/0, GL_TIMEOUT_IGNORED); - vector jpeg = encode_jpeg( - (const uint8_t *)qf.resources.pbo_contents, - (const uint8_t *)qf.resources.pbo_contents + 1280 * 720, - (const uint8_t *)qf.resources.pbo_contents + 1280 * 720 + 640 * 720, - 1280, 720); + const uint8_t *y = (const uint8_t *)qf.resources.pbo_contents; + const uint8_t *cb = (const uint8_t *)qf.resources.pbo_contents + 1280 * 720; + const uint8_t *cr = (const uint8_t *)qf.resources.pbo_contents + 1280 * 720 + 640 * 720; + + // Send a copy of the frame on to display. + shared_ptr frame(new Frame); + frame->y.reset(new uint8_t[1280 * 720]); + frame->cb.reset(new uint8_t[640 * 720]); + frame->cr.reset(new uint8_t[640 * 720]); + for (unsigned yy = 0; yy < 720; ++yy) { + memcpy(frame->y.get() + 1280 * yy, y + 1280 * (719 - yy), 1280); + memcpy(frame->cb.get() + 640 * yy, cb + 640 * (719 - yy), 640); + memcpy(frame->cr.get() + 640 * yy, cr + 640 * (719 - yy), 640); + } + frame->width = 1280; + frame->height = 720; + frame->chroma_subsampling_x = 2; + frame->chroma_subsampling_y = 1; + frame->pitch_y = 1280; + frame->pitch_chroma = 640; + JPEGFrameView::insert_interpolated_frame(qf.stream_idx, qf.output_pts, std::move(frame)); + + // Now JPEG encode it, and send it on to the stream. + vector jpeg = encode_jpeg(y, cb, cr, 1280, 720); compute_flow->release_texture(qf.flow_tex); interpolate->release_texture(qf.output_tex); interpolate->release_texture(qf.cbcr_tex); -- 2.39.2