From: Steinar H. Gunderson Date: Wed, 31 Oct 2018 17:16:08 +0000 (+0100) Subject: Stop sending interpolated images through the cache. X-Git-Tag: 1.8.0~76^2~20 X-Git-Url: https://git.sesse.net/?a=commitdiff_plain;h=78fa1508d2d831c270c748f11307b292f94f97df;p=nageru Stop sending interpolated images through the cache. They're never actually reused anyway, so there's no point in polluting the cache with them. Sending them directly to the display also takes away a large chunk of complexity. --- diff --git a/jpeg_frame_view.cpp b/jpeg_frame_view.cpp index c610227..f960d4b 100644 --- a/jpeg_frame_view.cpp +++ b/jpeg_frame_view.cpp @@ -36,9 +36,7 @@ struct JPEGIDLexicalOrder { if (a.stream_idx != b.stream_idx) return a.stream_idx < b.stream_idx; - if (a.pts != b.pts) - return a.pts < b.pts; - return a.interpolated < b.interpolated; + return a.pts < b.pts; } }; @@ -55,9 +53,16 @@ struct LRUFrame { }; struct PendingDecode { + JPEGFrameView *destination; + + // For actual decodes (only if frame below is nullptr). JPEGID primary, secondary; float fade_alpha; // Irrelevant if secondary.stream_idx == -1. - JPEGFrameView *destination; + + // Already-decoded frames are also sent through PendingDecode, + // so that they get drawn in the right order. If frame is nullptr, + // it's a real decode. + shared_ptr frame; }; } // namespace @@ -213,7 +218,6 @@ shared_ptr decode_jpeg_with_cache(JPEGID id, CacheMissBehavior cache_miss return nullptr; } - assert(!id.interpolated); *did_decode = true; shared_ptr frame = decode_jpeg(filename_for_frame(id.stream_idx, id.pts)); @@ -256,6 +260,12 @@ void jpeg_decoder_thread_func() } } + if (decode.frame != nullptr) { + // Already decoded, so just show it. + decode.destination->setDecodedFrame(decode.frame, nullptr, 1.0f); + continue; + } + shared_ptr primary_frame, secondary_frame; bool drop = false; for (int subframe_idx = 0; subframe_idx < 2; ++subframe_idx) { @@ -266,27 +276,10 @@ void jpeg_decoder_thread_func() } bool found_in_cache; - shared_ptr frame; - if (id.interpolated) { - // Interpolated frames are never decoded by us, - // put directly into the cache from VideoStream. - unique_lock lock(cache_mu); - auto it = cache.find(id); - if (it != cache.end()) { - it->second.last_used = event_counter++; - frame = it->second.frame; - } else { - // This can only really happen if it disappeared out of the - // LRU really, really fast. Which shouldn't happen. - fprintf(stderr, "WARNING: Interpolated JPEG was supposed to be in the cache, but was not\n"); - } - found_in_cache = true; // Don't count it as a decode. - } else { - frame = decode_jpeg_with_cache(id, cache_miss_behavior, &found_in_cache); - } + shared_ptr frame = decode_jpeg_with_cache(id, cache_miss_behavior, &found_in_cache); if (frame == nullptr) { - assert(id.interpolated || cache_miss_behavior == RETURN_NULLPTR_IF_NOT_IN_CACHE); + assert(cache_miss_behavior == RETURN_NULLPTR_IF_NOT_IN_CACHE); drop = true; break; } @@ -325,38 +318,29 @@ JPEGFrameView::JPEGFrameView(QWidget *parent) { } -void JPEGFrameView::setFrame(unsigned stream_idx, int64_t pts, bool interpolated, int secondary_stream_idx, int64_t secondary_pts, float fade_alpha) +void JPEGFrameView::setFrame(unsigned stream_idx, int64_t pts, int secondary_stream_idx, int64_t secondary_pts, float fade_alpha) { + if (secondary_stream_idx != -1) assert(secondary_pts != -1); current_stream_idx = stream_idx; // TODO: Does this interact with fades? unique_lock lock(cache_mu); PendingDecode decode; - if (interpolated && secondary_stream_idx != -1) { - // The frame will already be faded for us, so ask for only one; we shouldn't fade it against anything. - decode.primary = create_jpegid_for_interpolated_fade(stream_idx, pts, secondary_stream_idx, secondary_pts); - decode.secondary = JPEGID{ (unsigned)-1, -1, /*interpolated=*/false }; - } else { - decode.primary = JPEGID{ stream_idx, pts, interpolated }; - decode.secondary = JPEGID{ (unsigned)secondary_stream_idx, secondary_pts, /*interpolated=*/false }; - } + decode.primary = JPEGID{ stream_idx, pts }; + decode.secondary = JPEGID{ (unsigned)secondary_stream_idx, secondary_pts }; decode.fade_alpha = fade_alpha; decode.destination = this; pending_decodes.push_back(decode); any_pending_decodes.notify_all(); } -void JPEGFrameView::insert_interpolated_frame(JPEGID id, shared_ptr frame) +void JPEGFrameView::setFrame(shared_ptr frame) { - // We rely on the frame not being evicted from the cache before - // jpeg_decoder_thread() sees it and can display it (otherwise, - // that thread would hang). With a default cache of 1000 elements, - // that would sound like a reasonable assumption. unique_lock lock(cache_mu); - cache_bytes_used += frame_size(*frame); - cache[id] = LRUFrame{ std::move(frame), event_counter++ }; - if (cache_bytes_used > size_t(CACHE_SIZE_MB) * 1024 * 1024) { - prune_cache(); - } + PendingDecode decode; + decode.frame = std::move(frame); + decode.destination = this; + pending_decodes.push_back(decode); + any_pending_decodes.notify_all(); } ResourcePool *resource_pool = nullptr; diff --git a/jpeg_frame_view.h b/jpeg_frame_view.h index 1047e0b..65a6608 100644 --- a/jpeg_frame_view.h +++ b/jpeg_frame_view.h @@ -17,26 +17,12 @@ struct JPEGID { unsigned stream_idx; int64_t pts; - bool interpolated; }; enum CacheMissBehavior { DECODE_IF_NOT_IN_CACHE, RETURN_NULLPTR_IF_NOT_IN_CACHE }; -// This is, well, a hack. We hope for no collisions. -inline JPEGID create_jpegid_for_interpolated_fade(unsigned stream_idx, int64_t pts, unsigned secondary_stream_idx, int64_t secondary_pts) -{ - JPEGID id; - id.stream_idx = (stream_idx << 8) | secondary_stream_idx; - - uint64_t rot = secondary_stream_idx; - rot = (rot << 32) | (rot >> 32); - id.pts = pts ^ int64_t(rot); - id.interpolated = true; - return id; -} - std::string filename_for_frame(unsigned stream_idx, int64_t pts); std::shared_ptr decode_jpeg(const std::string &filename); std::shared_ptr decode_jpeg_with_cache(JPEGID id, CacheMissBehavior cache_miss_behavior, bool *did_decode); @@ -47,8 +33,8 @@ class JPEGFrameView : public QGLWidget { public: JPEGFrameView(QWidget *parent); - void setFrame(unsigned stream_idx, int64_t pts, bool interpolated, int secondary_stream_idx = -1, int64_t secondary_pts = -1, float fade_alpha = 0.0f); - static void insert_interpolated_frame(JPEGID id, std::shared_ptr frame); + void setFrame(unsigned stream_idx, int64_t pts, int secondary_stream_idx = -1, int64_t secondary_pts = -1, float fade_alpha = 0.0f); + void setFrame(std::shared_ptr frame); void mousePressEvent(QMouseEvent *event) override; diff --git a/main.cpp b/main.cpp index 6cd4741..229bbf1 100644 --- a/main.cpp +++ b/main.cpp @@ -238,13 +238,13 @@ int record_thread_func() post_to_main_thread([pkt, pts] { if (pkt.stream_index == 0) { - global_mainwindow->ui->input1_display->setFrame(pkt.stream_index, pts, /*interpolated=*/false); + global_mainwindow->ui->input1_display->setFrame(pkt.stream_index, pts); } else if (pkt.stream_index == 1) { - global_mainwindow->ui->input2_display->setFrame(pkt.stream_index, pts, /*interpolated=*/false); + global_mainwindow->ui->input2_display->setFrame(pkt.stream_index, pts); } else if (pkt.stream_index == 2) { - global_mainwindow->ui->input3_display->setFrame(pkt.stream_index, pts, /*interpolated=*/false); + global_mainwindow->ui->input3_display->setFrame(pkt.stream_index, pts); } else if (pkt.stream_index == 3) { - global_mainwindow->ui->input4_display->setFrame(pkt.stream_index, pts, /*interpolated=*/false); + global_mainwindow->ui->input4_display->setFrame(pkt.stream_index, pts); } }); diff --git a/player.cpp b/player.cpp index 3af45a0..2814623 100644 --- a/player.cpp +++ b/player.cpp @@ -232,7 +232,7 @@ got_clip: if (in_pts_lower == in_pts_upper) { auto display_func = [this, primary_stream_idx, in_pts_lower, secondary_stream_idx, secondary_pts, fade_alpha]{ - destination->setFrame(primary_stream_idx, in_pts_lower, /*interpolated=*/false, secondary_stream_idx, secondary_pts, fade_alpha); + destination->setFrame(primary_stream_idx, in_pts_lower, secondary_stream_idx, secondary_pts, fade_alpha); }; if (video_stream == nullptr) { display_func(); @@ -242,6 +242,7 @@ got_clip: next_frame_start, pts, display_func, QueueSpotHolder(this), primary_stream_idx, in_pts_lower); } else { + assert(secondary_pts != -1); video_stream->schedule_faded_frame(next_frame_start, pts, display_func, QueueSpotHolder(this), primary_stream_idx, in_pts_lower, secondary_stream_idx, secondary_pts, fade_alpha); @@ -258,7 +259,7 @@ got_clip: double snap_pts_as_frameno = (snap_pts - in_pts_origin) * output_framerate / TIMEBASE / speed; if (fabs(snap_pts_as_frameno - frameno) < 0.01) { auto display_func = [this, primary_stream_idx, snap_pts, secondary_stream_idx, secondary_pts, fade_alpha]{ - destination->setFrame(primary_stream_idx, snap_pts, /*interpolated=*/false, secondary_stream_idx, secondary_pts, fade_alpha); + destination->setFrame(primary_stream_idx, snap_pts, secondary_stream_idx, secondary_pts, fade_alpha); }; if (video_stream == nullptr) { display_func(); @@ -268,6 +269,7 @@ got_clip: next_frame_start, pts, display_func, QueueSpotHolder(this), primary_stream_idx, snap_pts); } else { + assert(secondary_pts != -1); video_stream->schedule_faded_frame( next_frame_start, pts, display_func, QueueSpotHolder(this), primary_stream_idx, snap_pts, secondary_stream_idx, secondary_pts, fade_alpha); @@ -293,10 +295,10 @@ got_clip: if (video_stream == nullptr) { // Previews don't do any interpolation. assert(secondary_stream_idx == -1); - destination->setFrame(primary_stream_idx, in_pts_lower, /*interpolated=*/false); + destination->setFrame(primary_stream_idx, in_pts_lower); } else { - auto display_func = [this, primary_stream_idx, pts, secondary_stream_idx, secondary_pts, fade_alpha]{ - destination->setFrame(primary_stream_idx, pts, /*interpolated=*/true, secondary_stream_idx, secondary_pts, fade_alpha); + auto display_func = [this](shared_ptr frame) { + destination->setFrame(frame); }; video_stream->schedule_interpolated_frame( next_frame_start, pts, display_func, QueueSpotHolder(this), @@ -429,7 +431,7 @@ void Player::override_angle(unsigned stream_idx) if (it == frames[stream_idx].end()) { return; } - destination->setFrame(stream_idx, *it, /*interpolated=*/false); + destination->setFrame(stream_idx, *it); } void Player::take_queue_spot() diff --git a/video_stream.cpp b/video_stream.cpp index bbd4616..c580b72 100644 --- a/video_stream.cpp +++ b/video_stream.cpp @@ -361,13 +361,11 @@ void VideoStream::schedule_faded_frame(steady_clock::time_point local_pts, int64 JPEGID jpeg_id1; jpeg_id1.stream_idx = stream_idx; jpeg_id1.pts = input_pts; - jpeg_id1.interpolated = false; shared_ptr frame1 = decode_jpeg_with_cache(jpeg_id1, DECODE_IF_NOT_IN_CACHE, &did_decode); JPEGID jpeg_id2; jpeg_id2.stream_idx = secondary_stream_idx; jpeg_id2.pts = secondary_input_pts; - jpeg_id2.interpolated = false; shared_ptr frame2 = decode_jpeg_with_cache(jpeg_id2, DECODE_IF_NOT_IN_CACHE, &did_decode); ycbcr_semiplanar_converter->prepare_chain_for_fade(frame1, frame2, fade_alpha)->render_to_fbo(resources->fade_fbo, 1280, 720); @@ -413,7 +411,7 @@ void VideoStream::schedule_faded_frame(steady_clock::time_point local_pts, int64 } void VideoStream::schedule_interpolated_frame(steady_clock::time_point local_pts, - int64_t output_pts, function &&display_func, + int64_t output_pts, function)> &&display_func, QueueSpotHolder &&queue_spot_holder, unsigned stream_idx, int64_t input_first_pts, int64_t input_second_pts, float alpha, @@ -426,13 +424,6 @@ void VideoStream::schedule_interpolated_frame(steady_clock::time_point local_pts fprintf(stderr, "output_pts=%ld interpolated input_pts1=%ld input_pts2=%ld alpha=%.3f\n", output_pts, input_first_pts, input_second_pts, alpha); } - JPEGID id; - if (secondary_stream_idx == -1) { - id = JPEGID{ stream_idx, output_pts, /*interpolated=*/true }; - } else { - id = create_jpegid_for_interpolated_fade(stream_idx, output_pts, secondary_stream_idx, secondary_input_pts); - } - // Get the temporary OpenGL resources we need for doing the interpolation. BorrowedInterpolatedFrameResources resources; { @@ -449,8 +440,7 @@ void VideoStream::schedule_interpolated_frame(steady_clock::time_point local_pts qf.type = (secondary_stream_idx == -1) ? QueuedFrame::INTERPOLATED : QueuedFrame::FADED_INTERPOLATED; qf.output_pts = output_pts; qf.stream_idx = stream_idx; - qf.id = id; - qf.display_func = move(display_func); + qf.display_decoded_func = move(display_func); qf.queue_spot_holder = move(queue_spot_holder); qf.local_pts = local_pts; @@ -461,7 +451,6 @@ void VideoStream::schedule_interpolated_frame(steady_clock::time_point local_pts JPEGID jpeg_id; jpeg_id.stream_idx = stream_idx; jpeg_id.pts = frame_no == 1 ? input_second_pts : input_first_pts; - jpeg_id.interpolated = false; bool did_decode; shared_ptr frame = decode_jpeg_with_cache(jpeg_id, DECODE_IF_NOT_IN_CACHE, &did_decode); ycbcr_converter->prepare_chain_for_conversion(frame)->render_to_fbo(resources->input_fbos[frame_no], 1280, 720); @@ -485,7 +474,6 @@ void VideoStream::schedule_interpolated_frame(steady_clock::time_point local_pts JPEGID jpeg_id; jpeg_id.stream_idx = secondary_stream_idx; jpeg_id.pts = secondary_input_pts; - jpeg_id.interpolated = false; bool did_decode; shared_ptr frame2 = decode_jpeg_with_cache(jpeg_id, DECODE_IF_NOT_IN_CACHE, &did_decode); @@ -646,9 +634,11 @@ void VideoStream::encode_thread_func() } else if (qf.type == QueuedFrame::INTERPOLATED || qf.type == QueuedFrame::FADED_INTERPOLATED) { glClientWaitSync(qf.fence.get(), /*flags=*/0, GL_TIMEOUT_IGNORED); - // Send a copy of the frame on to display. + // Send it on to display. shared_ptr frame = frame_from_pbo(qf.resources->pbo_contents, 1280, 720); - JPEGFrameView::insert_interpolated_frame(qf.id, frame); + if (qf.display_decoded_func != nullptr) { + qf.display_decoded_func(frame); + } // Now JPEG encode it, and send it on to the stream. vector jpeg = encode_jpeg(frame->y.get(), frame->cb.get(), frame->cr.get(), 1280, 720); diff --git a/video_stream.h b/video_stream.h index 2c0334a..22ae26a 100644 --- a/video_stream.h +++ b/video_stream.h @@ -51,7 +51,7 @@ public: unsigned stream_idx, int64_t input_pts, int secondary_stream_idx, int64_t secondary_input_pts, float fade_alpha); void schedule_interpolated_frame(std::chrono::steady_clock::time_point, int64_t output_pts, - std::function &&display_func, + std::function)> &&display_func, QueueSpotHolder &&queue_spot_holder, unsigned stream_idx, int64_t input_first_pts, int64_t input_second_pts, float alpha, int secondary_stream_idx = -1, int64_t secondary_inputs_pts = -1, @@ -121,6 +121,7 @@ private: JPEGID id; std::function display_func; // Called when the image is done decoding. + std::function)> display_decoded_func; // Same, except for INTERPOLATED and FADED_INTERPOLATED. QueueSpotHolder queue_spot_holder; };