From 1c72c1bdbe20edb994ab39b80306a0184de58c7e Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Sun, 30 Dec 2018 12:08:26 +0100 Subject: [PATCH] Add support to Nageru for reading the subtitles from FFmpeg streams. --- nageru/ffmpeg_capture.cpp | 9 +++++++-- nageru/ffmpeg_capture.h | 22 +++++++++++++++++++++- nageru/mixer.cpp | 5 +++++ nageru/pbo_frame_allocator.h | 2 ++ nageru/theme.cpp | 19 +++++++++++++++++++ 5 files changed, 54 insertions(+), 3 deletions(-) diff --git a/nageru/ffmpeg_capture.cpp b/nageru/ffmpeg_capture.cpp index 8f15973..1c7a30f 100644 --- a/nageru/ffmpeg_capture.cpp +++ b/nageru/ffmpeg_capture.cpp @@ -394,6 +394,8 @@ bool FFmpegCapture::play_video(const string &pathname) } int audio_stream_index = find_stream_index(format_ctx.get(), AVMEDIA_TYPE_AUDIO); + int subtitle_stream_index = find_stream_index(format_ctx.get(), AVMEDIA_TYPE_SUBTITLE); + has_last_subtitle = false; // Open video decoder. const AVCodecParameters *video_codecpar = format_ctx->streams[video_stream_index]->codecpar; @@ -455,7 +457,7 @@ bool FFmpegCapture::play_video(const string &pathname) int64_t audio_pts; bool error; AVFrameWithDeleter frame = decode_frame(format_ctx.get(), video_codec_ctx.get(), audio_codec_ctx.get(), - pathname, video_stream_index, audio_stream_index, audio_frame.get(), &audio_format, &audio_pts, &error); + pathname, video_stream_index, audio_stream_index, subtitle_stream_index, audio_frame.get(), &audio_format, &audio_pts, &error); if (error) { return false; } @@ -636,7 +638,7 @@ namespace { } // namespace AVFrameWithDeleter FFmpegCapture::decode_frame(AVFormatContext *format_ctx, AVCodecContext *video_codec_ctx, AVCodecContext *audio_codec_ctx, - const std::string &pathname, int video_stream_index, int audio_stream_index, + const std::string &pathname, int video_stream_index, int audio_stream_index, int subtitle_stream_index, FrameAllocator::Frame *audio_frame, AudioFormat *audio_format, int64_t *audio_pts, bool *error) { *error = false; @@ -672,6 +674,9 @@ AVFrameWithDeleter FFmpegCapture::decode_frame(AVFormatContext *format_ctx, AVCo *error = true; return AVFrameWithDeleter(nullptr); } + } else if (pkt.stream_index == subtitle_stream_index) { + last_subtitle = string(reinterpret_cast(pkt.data), pkt.size); + has_last_subtitle = true; } } else { eof = true; // Or error, but ignore that for the time being. diff --git a/nageru/ffmpeg_capture.h b/nageru/ffmpeg_capture.h index 468c213..a9a8371 100644 --- a/nageru/ffmpeg_capture.h +++ b/nageru/ffmpeg_capture.h @@ -17,6 +17,11 @@ // changes parameters midway, which is allowed in some formats. // // You can get out the audio either as decoded or in raw form (Kaeru uses this). +// +// If there's a subtitle track, you can also get out the last subtitle at the +// point of the frame. Note that once we get a video frame, we don't look for +// subtitle, so if subtitles and a frame comes at the same time, you might not +// see the subtitle until the next frame. #include #include @@ -164,6 +169,18 @@ public: return current_frame_ycbcr_format; } + // Only valid to call during the frame callback. + std::string get_last_subtitle() const + { + return last_subtitle; + } + + // Same. + bool get_has_last_subtitle() const + { + return has_last_subtitle; + } + void set_dequeue_thread_callbacks(std::function init, std::function cleanup) override { dequeue_init_callback = init; @@ -218,7 +235,7 @@ private: // Returns nullptr if no frame was decoded (e.g. EOF). AVFrameWithDeleter decode_frame(AVFormatContext *format_ctx, AVCodecContext *video_codec_ctx, AVCodecContext *audio_codec_ctx, - const std::string &pathname, int video_stream_index, int audio_stream_index, + const std::string &pathname, int video_stream_index, int audio_stream_index, int subtitle_stream_index, bmusb::FrameAllocator::Frame *audio_frame, bmusb::AudioFormat *audio_format, int64_t *audio_pts, bool *error); void convert_audio(const AVFrame *audio_avframe, bmusb::FrameAllocator::Frame *audio_frame, bmusb::AudioFormat *audio_format); @@ -276,6 +293,9 @@ private: int64_t last_channel_layout; int last_sample_rate; + // Subtitles (no decoding done, really). + bool has_last_subtitle = false; + std::string last_subtitle; }; #endif // !defined(_FFMPEG_CAPTURE_H) diff --git a/nageru/mixer.cpp b/nageru/mixer.cpp index 7ef2a4c..9e28d4c 100644 --- a/nageru/mixer.cpp +++ b/nageru/mixer.cpp @@ -795,6 +795,11 @@ void Mixer::bm_frame(unsigned card_index, uint16_t timecode, card->last_timecode = timecode; PBOFrameAllocator::Userdata *userdata = (PBOFrameAllocator::Userdata *)video_frame.userdata; + if (card->type == CardType::FFMPEG_INPUT) { + FFmpegCapture *ffmpeg_capture = static_cast(card->capture.get()); + userdata->has_last_subtitle = ffmpeg_capture->get_has_last_subtitle(); + userdata->last_subtitle = ffmpeg_capture->get_last_subtitle(); + } size_t cbcr_width, cbcr_height, cbcr_offset, y_offset; size_t expected_length = video_format.stride * (video_format.height + video_format.extra_lines_top + video_format.extra_lines_bottom); diff --git a/nageru/pbo_frame_allocator.h b/nageru/pbo_frame_allocator.h index eead7f4..ab51f6b 100644 --- a/nageru/pbo_frame_allocator.h +++ b/nageru/pbo_frame_allocator.h @@ -52,6 +52,8 @@ public: GLuint last_v210_width[2]; // PixelFormat_10BitYCbCr. bool last_interlaced, last_has_signal, last_is_connected; unsigned last_frame_rate_nom, last_frame_rate_den; + bool has_last_subtitle = false; + std::string last_subtitle; }; private: diff --git a/nageru/theme.cpp b/nageru/theme.cpp index 386a425..ead38fa 100644 --- a/nageru/theme.cpp +++ b/nageru/theme.cpp @@ -72,6 +72,8 @@ struct InputStateInfo { unsigned last_width[MAX_VIDEO_CARDS], last_height[MAX_VIDEO_CARDS]; bool last_interlaced[MAX_VIDEO_CARDS], last_has_signal[MAX_VIDEO_CARDS], last_is_connected[MAX_VIDEO_CARDS]; unsigned last_frame_rate_nom[MAX_VIDEO_CARDS], last_frame_rate_den[MAX_VIDEO_CARDS]; + bool has_last_subtitle[MAX_VIDEO_CARDS]; + std::string last_subtitle[MAX_VIDEO_CARDS]; }; InputStateInfo::InputStateInfo(const InputState &input_state) @@ -93,6 +95,8 @@ InputStateInfo::InputStateInfo(const InputState &input_state) last_is_connected[signal_num] = userdata->last_is_connected; last_frame_rate_nom[signal_num] = userdata->last_frame_rate_nom; last_frame_rate_den[signal_num] = userdata->last_frame_rate_den; + has_last_subtitle[signal_num] = userdata->has_last_subtitle; + last_subtitle[signal_num] = userdata->last_subtitle; } } @@ -634,6 +638,20 @@ int InputStateInfo_get_frame_rate_den(lua_State* L) return 1; } +int InputStateInfo_get_last_subtitle(lua_State* L) +{ + assert(lua_gettop(L) == 2); + InputStateInfo *input_state_info = get_input_state_info(L, 1); + Theme *theme = get_theme_updata(L); + int signal_num = theme->map_signal(luaL_checknumber(L, 2)); + if (!input_state_info->has_last_subtitle[signal_num]) { + lua_pushnil(L); + } else { + lua_pushstring(L, input_state_info->last_subtitle[signal_num].c_str()); + } + return 1; +} + int Effect_set_float(lua_State *L) { assert(lua_gettop(L) == 3); @@ -829,6 +847,7 @@ const luaL_Reg InputStateInfo_funcs[] = { { "get_is_connected", InputStateInfo_get_is_connected }, { "get_frame_rate_nom", InputStateInfo_get_frame_rate_nom }, { "get_frame_rate_den", InputStateInfo_get_frame_rate_den }, + { "get_last_subtitle", InputStateInfo_get_last_subtitle }, { NULL, NULL } }; -- 2.39.2