Add CEF support to Kaeru.

[nageru] / nageru / cef_encoder_adapter.cpp

diff --git a/nageru/cef_encoder_adapter.cpp b/nageru/cef_encoder_adapter.cpp
new file mode 100644 (file)
index 0000000..a6c7ea8
--- /dev/null
+++ b/nageru/cef_encoder_adapter.cpp
@@ -0,0 +1,134 @@
+#include "cef_encoder_adapter.h"
+
+#include <bmusb/bmusb.h>
+#include <chrono>
+#include <mutex>
+#include <optional>
+#include <stddef.h>
+#include <stdint.h>
+
+extern "C" {
+#include <libavutil/rational.h>
+#include <libswscale/swscale.h>
+}
+
+#include "flags.h"
+#include "quittable_sleeper.h"
+
+using namespace bmusb;
+using namespace std;
+using namespace std::chrono;
+
+class FFmpegCapture;
+
+// In kaeru.cpp. (This is a bit of a hack, but in the interest of a cleaner split.)
+void video_frame_callback(FFmpegCapture *video, X264Encoder *x264_encoder, AudioEncoder *audio_encoder,
+                          int64_t video_pts, AVRational video_timebase,
+                          int64_t audio_pts, AVRational audio_timebase,
+                          uint16_t timecode,
+                          FrameAllocator::Frame video_frame, size_t video_offset, VideoFormat video_format,
+                          FrameAllocator::Frame audio_frame, size_t audio_offset, AudioFormat audio_format);
+
+void CEFEncoderAdapter::video_frame_callback(uint16_t timecode,
+                                             FrameAllocator::Frame video_frame, size_t video_offset, VideoFormat video_format,
+                                             FrameAllocator::Frame audio_frame, size_t audio_offset, AudioFormat audio_format)
+{
+       lock_guard lock(mu);
+       if (first_frame) {
+               start = steady_clock::now();
+               first_frame = false;
+       }
+
+       last_video_format = video_format;
+       last_audio_format = audio_format;
+
+       steady_clock::time_point now = steady_clock::now();
+       int64_t video_pts = duration_cast<microseconds>(now - start).count();
+       AVRational video_timebase{ 1, 1000000 };
+
+       FrameAllocator::Frame nv12_video_frame;
+       if (video_frame.len > 0) {
+               last_frame = now;
+
+               // Do the actual conversion, replacing the frame.
+               const uint8_t *src_pic_data[4] = { video_frame.data + video_offset, nullptr, nullptr, nullptr };
+               int src_linesizes[4] = { global_flags.width * 4, 0, 0, 0 };
+
+               uint8_t *pic_data[4] = { nv12_data.get(), nv12_data.get() + global_flags.width * global_flags.height, nullptr, nullptr };
+               int linesizes[4] = { global_flags.width, global_flags.width, 0, 0 };
+
+               sws_scale(sws_ctx.get(), src_pic_data, src_linesizes, 0, video_format.height, pic_data, linesizes);
+
+               nv12_video_frame.data = nv12_data.get();
+               nv12_video_frame.len = nv12_video_frame.size = global_flags.width * global_flags.height * 2;
+               if (video_frame.owner) {
+                       video_frame.owner->release_frame(video_frame);
+               }
+       } else {
+               nv12_video_frame = video_frame;
+               // Will be released by video_frame_callback(), so don't do that here.
+       }
+
+       // NOTE: We don't have CEF audio yet.
+
+       ::video_frame_callback(nullptr, x264_encoder, audio_encoder,
+                              video_pts, video_timebase,
+                              /*audio_pts=*/0, AVRational{ 1, 1 },
+                              timecode,
+                              nv12_video_frame, /*video_offset=*/0, video_format,
+                              audio_frame, audio_offset, audio_format);
+}
+
+// Enforce at least 15 fps by duplicating frames. This feels suboptimal
+// on the face of it, but so many things in x264 (in particular lookahead
+// and keyframe interval) work on the number of frames, so not having
+// frames for a while will mess up the latency pretty badly, especially
+// when clients do the initial join on the stream.
+//
+// Returns early (without any duplication) if should_quit becomes active.
+void CEFEncoderAdapter::duplicate_frame_if_needed(QuittableSleeper *should_quit)
+{
+       constexpr duration max_frame_interval = milliseconds(1000 / 15);
+
+       optional<steady_clock::time_point> last_frame_now = get_last_frame();
+       if (!last_frame_now) {
+               // No initial frame yet, so nothing to duplicate. Just wait a bit.
+               should_quit->sleep_for(max_frame_interval);
+               return;
+       }
+       steady_clock::time_point next_inserted_frame = *last_frame_now + max_frame_interval;
+       if (!should_quit->sleep_until(next_inserted_frame)) {
+               // Asked to quit.
+               return;
+       }
+       lock_guard lock(mu);
+       if (*last_frame_now != last_frame) {
+               // A new frame came while we were waiting, so we don't need one just yet.
+               return;
+       }
+
+       // No new frame came before the deadline, so add a duplicate. The data from last
+       // conversion is still in the buffer, so we can just use that.
+       steady_clock::time_point now = steady_clock::now();
+       int64_t video_pts = duration_cast<microseconds>(now - start).count();
+
+       last_frame = now;
+       FrameAllocator::Frame nv12_video_frame;
+       nv12_video_frame.data = nv12_data.get();
+       nv12_video_frame.len = nv12_video_frame.size = global_flags.width * global_flags.height * 2;
+       ::video_frame_callback(nullptr, x264_encoder, audio_encoder,
+                              video_pts, video_timebase,
+                              /*audio_pts=*/0, AVRational{ 1, 1 },
+                              /*timecode=*/0,
+                              nv12_video_frame, /*video_offset=*/0, last_video_format,
+                              /*audio_frame=*/{}, /*audio_offset=*/0, last_audio_format);
+}
+
+optional<steady_clock::time_point> CEFEncoderAdapter::get_last_frame() const {
+       lock_guard lock(mu);
+       if (first_frame) {
+               return nullopt;
+       } else {
+               return last_frame;
+       }
+}