From e20bf0f86703779f11d575f44173ff73544e1c2d Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Sat, 29 Dec 2018 00:23:02 +0100 Subject: [PATCH] Make Futatabi output its status as subtitles, in a hopefully fairly parseable form. --- futatabi/mainwindow.cpp | 18 ++-------- futatabi/player.cpp | 69 +++++++++++++++++++++++++++++---------- futatabi/player.h | 13 +++++++- futatabi/video_stream.cpp | 27 ++++++++++++--- futatabi/video_stream.h | 12 ++++--- shared/mux.cpp | 16 ++++++++- shared/mux.h | 8 ++++- 7 files changed, 117 insertions(+), 46 deletions(-) diff --git a/futatabi/mainwindow.cpp b/futatabi/mainwindow.cpp index d73f579..154537c 100644 --- a/futatabi/mainwindow.cpp +++ b/futatabi/mainwindow.cpp @@ -524,21 +524,6 @@ void MainWindow::live_player_done() ui->stop_btn->setEnabled(false); } -static string format_duration(double t) -{ - int t_ms = lrint(t * 1e3); - - int ms = t_ms % 1000; - t_ms /= 1000; - int s = t_ms % 60; - t_ms /= 60; - int m = t_ms; - - char buf[256]; - snprintf(buf, sizeof(buf), "%d:%02d.%03d", m, s, ms); - return buf; -} - void MainWindow::live_player_clip_progress(const map &progress, double time_remaining) { playlist_clips->set_progress(progress); @@ -1036,6 +1021,9 @@ void MainWindow::highlight_camera_input(int stream_idx) void MainWindow::set_output_status(const string &status) { ui->live_label->setText(QString::fromStdString("Current output (" + status + ")")); + if (live_player != nullptr) { + live_player->set_pause_status(status); + } lock_guard lock(queue_status_mu); queue_status = status; diff --git a/futatabi/player.cpp b/futatabi/player.cpp index 6f329ef..34beda7 100644 --- a/futatabi/player.cpp +++ b/futatabi/player.cpp @@ -113,6 +113,7 @@ void Player::play_playlist_once() vector clip_list; bool clip_ready; steady_clock::time_point before_sleep = steady_clock::now(); + string pause_status; // Wait until we're supposed to play something. { @@ -131,6 +132,8 @@ void Player::play_playlist_once() queued_clip_list.clear(); assert(!clip_list.empty()); assert(!splice_ready); // This corner case should have been handled in splice_play(). + } else { + pause_status = this->pause_status; } } @@ -140,7 +143,9 @@ void Player::play_playlist_once() if (!clip_ready) { if (video_stream != nullptr) { ++metric_refresh_frame; - video_stream->schedule_refresh_frame(steady_clock::now(), pts, /*display_func=*/nullptr, QueueSpotHolder()); + string subtitle = "Futatabi " NAGERU_VERSION ";PAUSED;" + pause_status; + video_stream->schedule_refresh_frame(steady_clock::now(), pts, /*display_func=*/nullptr, QueueSpotHolder(), + subtitle); } return; } @@ -250,18 +255,18 @@ void Player::play_playlist_once() } } + // NOTE: None of this will take into account any snapping done below. + double clip_progress = calc_progress(*clip, in_pts_for_progress); + map progress{ { clip_list[clip_idx].id, clip_progress } }; + double time_remaining; + if (next_clip != nullptr && time_left_this_clip <= next_clip_fade_time) { + double next_clip_progress = calc_progress(*next_clip, in_pts_secondary_for_progress); + progress[clip_list[clip_idx + 1].id] = next_clip_progress; + time_remaining = compute_time_left(clip_list, clip_idx + 1, next_clip_progress); + } else { + time_remaining = compute_time_left(clip_list, clip_idx, clip_progress); + } if (progress_callback != nullptr) { - // NOTE: None of this will take into account any snapping done below. - double clip_progress = calc_progress(*clip, in_pts_for_progress); - map progress{ { clip_list[clip_idx].id, clip_progress } }; - double time_remaining; - if (next_clip != nullptr && time_left_this_clip <= next_clip_fade_time) { - double next_clip_progress = calc_progress(*next_clip, in_pts_secondary_for_progress); - progress[clip_list[clip_idx + 1].id] = next_clip_progress; - time_remaining = compute_time_left(clip_list, clip_idx + 1, next_clip_progress); - } else { - time_remaining = compute_time_left(clip_list, clip_idx, clip_progress); - } progress_callback(progress, time_remaining); } @@ -315,11 +320,23 @@ void Player::play_playlist_once() } } + string subtitle; + { + stringstream ss; + ss.imbue(locale("C")); + ss.precision(3); + ss << "Futatabi " NAGERU_VERSION ";PLAYING;"; + ss << fixed << time_remaining; + ss << ";" << format_duration(time_remaining) << " left"; + subtitle = ss.str(); + } + // If there's nothing to interpolate between, or if interpolation is turned off, // or we're a preview, then just display the frame. if (frame_lower.pts == frame_upper.pts || global_flags.interpolation_quality == 0 || video_stream == nullptr) { display_single_frame(primary_stream_idx, frame_lower, secondary_stream_idx, - secondary_frame, fade_alpha, next_frame_start, /*snapped=*/false); + secondary_frame, fade_alpha, next_frame_start, /*snapped=*/false, + subtitle); continue; } @@ -331,7 +348,8 @@ void Player::play_playlist_once() for (FrameOnDisk snap_frame : { frame_lower, frame_upper }) { if (fabs(snap_frame.pts - in_pts) < pts_snap_tolerance) { display_single_frame(primary_stream_idx, snap_frame, secondary_stream_idx, - secondary_frame, fade_alpha, next_frame_start, /*snapped=*/true); + secondary_frame, fade_alpha, next_frame_start, /*snapped=*/true, + subtitle); in_pts_origin += snap_frame.pts - in_pts; snapped = true; break; @@ -387,7 +405,7 @@ void Player::play_playlist_once() video_stream->schedule_interpolated_frame( next_frame_start, pts, display_func, QueueSpotHolder(this), frame_lower, frame_upper, alpha, - secondary_frame, fade_alpha); + secondary_frame, fade_alpha, subtitle); last_pts_played = in_pts; // Not really needed; only previews use last_pts_played. } @@ -408,7 +426,7 @@ void Player::play_playlist_once() } } -void Player::display_single_frame(int primary_stream_idx, const FrameOnDisk &primary_frame, int secondary_stream_idx, const FrameOnDisk &secondary_frame, double fade_alpha, steady_clock::time_point frame_start, bool snapped) +void Player::display_single_frame(int primary_stream_idx, const FrameOnDisk &primary_frame, int secondary_stream_idx, const FrameOnDisk &secondary_frame, double fade_alpha, steady_clock::time_point frame_start, bool snapped, const std::string &subtitle) { auto display_func = [this, primary_stream_idx, primary_frame, secondary_frame, fade_alpha] { if (destination != nullptr) { @@ -427,7 +445,7 @@ void Player::display_single_frame(int primary_stream_idx, const FrameOnDisk &pri } video_stream->schedule_original_frame( frame_start, pts, display_func, QueueSpotHolder(this), - primary_frame); + primary_frame, subtitle); } else { assert(secondary_frame.pts != -1); // NOTE: We could be increasing unused metrics for previews, but that's harmless. @@ -438,7 +456,7 @@ void Player::display_single_frame(int primary_stream_idx, const FrameOnDisk &pri } video_stream->schedule_faded_frame(frame_start, pts, display_func, QueueSpotHolder(this), primary_frame, - secondary_frame, fade_alpha); + secondary_frame, fade_alpha, subtitle); } } last_pts_played = primary_frame.pts; @@ -590,3 +608,18 @@ double compute_time_left(const vector &clips, size_t currently_playi } return remaining; } + +string format_duration(double t) +{ + int t_ms = lrint(t * 1e3); + + int ms = t_ms % 1000; + t_ms /= 1000; + int s = t_ms % 60; + t_ms /= 60; + int m = t_ms; + + char buf[256]; + snprintf(buf, sizeof(buf), "%d:%02d.%03d", m, s, ms); + return buf; +} diff --git a/futatabi/player.h b/futatabi/player.h index 33c1723..a097fd9 100644 --- a/futatabi/player.h +++ b/futatabi/player.h @@ -53,6 +53,14 @@ public: // If nothing is playing, the call will be ignored. void splice_play(const std::vector &clips); + // Set the status string that will be used for the video stream's status subtitles + // whenever we are not playing anything. + void set_pause_status(const std::string &status) + { + std::lock_guard lock(queue_state_mu); + pause_status = status; + } + // Not thread-safe to set concurrently with playing. // Will be called back from the player thread. using done_callback_func = std::function; @@ -71,7 +79,7 @@ public: private: void thread_func(AVFormatContext *file_avctx); void play_playlist_once(); - void display_single_frame(int primary_stream_idx, const FrameOnDisk &primary_frame, int secondary_stream_idx, const FrameOnDisk &secondary_frame, double fade_alpha, std::chrono::steady_clock::time_point frame_start, bool snapped); + void display_single_frame(int primary_stream_idx, const FrameOnDisk &primary_frame, int secondary_stream_idx, const FrameOnDisk &secondary_frame, double fade_alpha, std::chrono::steady_clock::time_point frame_start, bool snapped, const std::string &subtitle); void open_output_stream(); static int write_packet2_thunk(void *opaque, uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time); int write_packet2(uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time); @@ -97,6 +105,7 @@ private: bool splice_ready = false; // Under queue_state_mu. std::vector to_splice_clip_list; // Under queue_state_mu. + std::string pause_status = "paused"; // Under queue_state_mu. std::unique_ptr video_stream; // Can be nullptr. @@ -127,4 +136,6 @@ static inline double compute_total_time(const std::vector &clips) return compute_time_left(clips, 0, 0.0); } +std::string format_duration(double t); + #endif // !defined(_PLAYER_H) diff --git a/futatabi/video_stream.cpp b/futatabi/video_stream.cpp index 66323b8..745da27 100644 --- a/futatabi/video_stream.cpp +++ b/futatabi/video_stream.cpp @@ -264,7 +264,7 @@ void VideoStream::start() size_t width = global_flags.width, height = global_flags.height; // Doesn't matter for MJPEG. mux.reset(new Mux(avctx, width, height, Mux::CODEC_MJPEG, /*video_extradata=*/"", /*audio_codec_parameters=*/nullptr, - AVCOL_SPC_BT709, COARSE_TIMEBASE, /*write_callback=*/nullptr, Mux::WRITE_FOREGROUND, {})); + AVCOL_SPC_BT709, COARSE_TIMEBASE, /*write_callback=*/nullptr, Mux::WRITE_FOREGROUND, {}, Mux::WITH_SUBTITLES)); encode_thread = thread(&VideoStream::encode_thread_func, this); } @@ -307,7 +307,7 @@ void VideoStream::clear_queue() void VideoStream::schedule_original_frame(steady_clock::time_point local_pts, int64_t output_pts, function &&display_func, QueueSpotHolder &&queue_spot_holder, - FrameOnDisk frame) + FrameOnDisk frame, const string &subtitle) { fprintf(stderr, "output_pts=%ld original input_pts=%ld\n", output_pts, frame.pts); @@ -322,6 +322,7 @@ void VideoStream::schedule_original_frame(steady_clock::time_point local_pts, qf.frame1 = frame; qf.display_func = move(display_func); qf.queue_spot_holder = move(queue_spot_holder); + qf.subtitle = subtitle; lock_guard lock(queue_lock); frame_queue.push_back(move(qf)); @@ -332,7 +333,7 @@ void VideoStream::schedule_faded_frame(steady_clock::time_point local_pts, int64 function &&display_func, QueueSpotHolder &&queue_spot_holder, FrameOnDisk frame1_spec, FrameOnDisk frame2_spec, - float fade_alpha) + float fade_alpha, const string &subtitle) { fprintf(stderr, "output_pts=%ld faded input_pts=%ld,%ld fade_alpha=%.2f\n", output_pts, frame1_spec.pts, frame2_spec.pts, fade_alpha); @@ -365,6 +366,7 @@ void VideoStream::schedule_faded_frame(steady_clock::time_point local_pts, int64 qf.frame1 = frame1_spec; qf.display_func = move(display_func); qf.queue_spot_holder = move(queue_spot_holder); + qf.subtitle = subtitle; qf.secondary_frame = frame2_spec; @@ -400,7 +402,7 @@ void VideoStream::schedule_interpolated_frame(steady_clock::time_point local_pts int64_t output_pts, function)> &&display_func, QueueSpotHolder &&queue_spot_holder, FrameOnDisk frame1, FrameOnDisk frame2, - float alpha, FrameOnDisk secondary_frame, float fade_alpha) + float alpha, FrameOnDisk secondary_frame, float fade_alpha, const string &subtitle) { if (secondary_frame.pts != -1) { fprintf(stderr, "output_pts=%ld interpolated input_pts1=%ld input_pts2=%ld alpha=%.3f secondary_pts=%ld fade_alpha=%.2f\n", output_pts, frame1.pts, frame2.pts, alpha, secondary_frame.pts, fade_alpha); @@ -426,6 +428,7 @@ void VideoStream::schedule_interpolated_frame(steady_clock::time_point local_pts qf.display_decoded_func = move(display_func); qf.queue_spot_holder = move(queue_spot_holder); qf.local_pts = local_pts; + qf.subtitle = subtitle; check_error(); @@ -524,13 +527,14 @@ void VideoStream::schedule_interpolated_frame(steady_clock::time_point local_pts void VideoStream::schedule_refresh_frame(steady_clock::time_point local_pts, int64_t output_pts, function &&display_func, - QueueSpotHolder &&queue_spot_holder) + QueueSpotHolder &&queue_spot_holder, const string &subtitle) { QueuedFrame qf; qf.type = QueuedFrame::REFRESH; qf.output_pts = output_pts; qf.display_func = move(display_func); qf.queue_spot_holder = move(queue_spot_holder); + qf.subtitle = subtitle; lock_guard lock(queue_lock); frame_queue.push_back(move(qf)); @@ -677,6 +681,19 @@ void VideoStream::encode_thread_func() } else { assert(false); } + + if (!qf.subtitle.empty()) { + AVPacket pkt; + av_init_packet(&pkt); + pkt.stream_index = mux->get_subtitle_stream_idx(); + assert(pkt.stream_index != -1); + pkt.data = (uint8_t *)qf.subtitle.data(); + pkt.size = qf.subtitle.size(); + pkt.flags = 0; + pkt.duration = lrint(TIMEBASE / global_flags.output_framerate); // Doesn't really matter for Nageru. + mux->add_packet(pkt, qf.output_pts, qf.output_pts); + } + if (qf.display_func != nullptr) { qf.display_func(); } diff --git a/futatabi/video_stream.h b/futatabi/video_stream.h index 906cd77..03f82d6 100644 --- a/futatabi/video_stream.h +++ b/futatabi/video_stream.h @@ -47,21 +47,21 @@ public: void schedule_original_frame(std::chrono::steady_clock::time_point, int64_t output_pts, std::function &&display_func, QueueSpotHolder &&queue_spot_holder, - FrameOnDisk frame); + FrameOnDisk frame, const std::string &subtitle); void schedule_faded_frame(std::chrono::steady_clock::time_point, int64_t output_pts, std::function &&display_func, QueueSpotHolder &&queue_spot_holder, FrameOnDisk frame1, FrameOnDisk frame2, - float fade_alpha); + float fade_alpha, const std::string &subtitle); void schedule_interpolated_frame(std::chrono::steady_clock::time_point, int64_t output_pts, std::function)> &&display_func, QueueSpotHolder &&queue_spot_holder, FrameOnDisk frame1, FrameOnDisk frame2, - float alpha, FrameOnDisk secondary_frame = {}, // Empty = no secondary (fade) frame. - float fade_alpha = 0.0f); + float alpha, FrameOnDisk secondary_frame, // Empty = no secondary (fade) frame. + float fade_alpha, const std::string &subtitle); void schedule_refresh_frame(std::chrono::steady_clock::time_point, int64_t output_pts, std::function &&display_func, - QueueSpotHolder &&queue_spot_holder); + QueueSpotHolder &&queue_spot_holder, const std::string &subtitle); private: FrameReader frame_reader; @@ -127,6 +127,8 @@ private: std::function display_func; // Called when the image is done decoding. std::function)> display_decoded_func; // Same, except for INTERPOLATED and FADED_INTERPOLATED. + std::string subtitle; // Blank for none. + QueueSpotHolder queue_spot_holder; }; std::deque frame_queue; // Under . diff --git a/shared/mux.cpp b/shared/mux.cpp index 48bfaa1..e122e2b 100644 --- a/shared/mux.cpp +++ b/shared/mux.cpp @@ -47,7 +47,7 @@ struct PacketBefore { const AVFormatContext * const ctx; }; -Mux::Mux(AVFormatContext *avctx, int width, int height, Codec video_codec, const string &video_extradata, const AVCodecParameters *audio_codecpar, AVColorSpace color_space, int time_base, function write_callback, WriteStrategy write_strategy, const vector &metrics) +Mux::Mux(AVFormatContext *avctx, int width, int height, Codec video_codec, const string &video_extradata, const AVCodecParameters *audio_codecpar, AVColorSpace color_space, int time_base, function write_callback, WriteStrategy write_strategy, const vector &metrics, WithSubtitles with_subtitles) : write_strategy(write_strategy), avctx(avctx), write_callback(write_callback), metrics(metrics) { AVStream *avstream_video = avformat_new_stream(avctx, nullptr); @@ -104,6 +104,20 @@ Mux::Mux(AVFormatContext *avctx, int width, int height, Codec video_codec, const streams.push_back(avstream_audio); } + if (with_subtitles == WITH_SUBTITLES) { + AVStream *avstream_subtitles = avformat_new_stream(avctx, nullptr); + if (avstream_subtitles == nullptr) { + fprintf(stderr, "avformat_new_stream() failed\n"); + exit(1); + } + avstream_subtitles->time_base = AVRational{1, time_base}; + avstream_subtitles->codecpar->codec_type = AVMEDIA_TYPE_SUBTITLE; + avstream_subtitles->codecpar->codec_id = AV_CODEC_ID_WEBVTT; + avstream_subtitles->disposition = AV_DISPOSITION_METADATA; + streams.push_back(avstream_subtitles); + subtitle_stream_idx = streams.size() - 1; + } + AVDictionary *options = NULL; vector> opts = MUX_OPTS; for (pair opt : opts) { diff --git a/shared/mux.h b/shared/mux.h index d3d7798..1b9fe93 100644 --- a/shared/mux.h +++ b/shared/mux.h @@ -64,6 +64,10 @@ public: // higher overhead. WRITE_BACKGROUND, }; + enum WithSubtitles { + WITH_SUBTITLES, + WITHOUT_SUBTITLES + }; // Takes ownership of avctx. will be called every time // a write has been made to the video stream (id 0), with the pts of @@ -72,9 +76,10 @@ public: // will be added to. // // If audio_codecpar is nullptr, there will be no audio stream. - Mux(AVFormatContext *avctx, int width, int height, Codec video_codec, const std::string &video_extradata, const AVCodecParameters *audio_codecpar, AVColorSpace color_space, int time_base, std::function write_callback, WriteStrategy write_strategy, const std::vector &metrics); + Mux(AVFormatContext *avctx, int width, int height, Codec video_codec, const std::string &video_extradata, const AVCodecParameters *audio_codecpar, AVColorSpace color_space, int time_base, std::function write_callback, WriteStrategy write_strategy, const std::vector &metrics, WithSubtitles with_subtitles = WITHOUT_SUBTITLES); ~Mux(); void add_packet(const AVPacket &pkt, int64_t pts, int64_t dts, AVRational timebase = { 1, TIMEBASE }, int stream_index_override = -1); + int get_subtitle_stream_idx() const { return subtitle_stream_idx; } // As long as the mux is plugged, it will not actually write anything to disk, // just queue the packets. Once it is unplugged, the packets are reordered by pts @@ -113,6 +118,7 @@ private: std::condition_variable packet_queue_ready; std::vector streams; + int subtitle_stream_idx = -1; std::function write_callback; std::vector metrics; -- 2.39.2