Add support to Nageru for reading the subtitles from FFmpeg streams.

diff --git a/nageru/ffmpeg_capture.cpp b/nageru/ffmpeg_capture.cpp
index 8f15973ba7eba8fc5d05d21268b873140023655a..1c7a30f2e82e1245574174420a145c7fa8b8ee0f 100644 (file)
--- a/nageru/ffmpeg_capture.cpp
+++ b/nageru/ffmpeg_capture.cpp
@@ -394,6 +394,8 @@ bool FFmpegCapture::play_video(const string &pathname)
        }
 
        int audio_stream_index = find_stream_index(format_ctx.get(), AVMEDIA_TYPE_AUDIO);
+       int subtitle_stream_index = find_stream_index(format_ctx.get(), AVMEDIA_TYPE_SUBTITLE);
+       has_last_subtitle = false;
 
        // Open video decoder.
        const AVCodecParameters *video_codecpar = format_ctx->streams[video_stream_index]->codecpar;
@@ -455,7 +457,7 @@ bool FFmpegCapture::play_video(const string &pathname)
                int64_t audio_pts;
                bool error;
                AVFrameWithDeleter frame = decode_frame(format_ctx.get(), video_codec_ctx.get(), audio_codec_ctx.get(),
-                       pathname, video_stream_index, audio_stream_index, audio_frame.get(), &audio_format, &audio_pts, &error);
+                       pathname, video_stream_index, audio_stream_index, subtitle_stream_index, audio_frame.get(), &audio_format, &audio_pts, &error);
                if (error) {
                        return false;
                }
@@ -636,7 +638,7 @@ namespace {
 }  // namespace
 
 AVFrameWithDeleter FFmpegCapture::decode_frame(AVFormatContext *format_ctx, AVCodecContext *video_codec_ctx, AVCodecContext *audio_codec_ctx,
-       const std::string &pathname, int video_stream_index, int audio_stream_index,
+       const std::string &pathname, int video_stream_index, int audio_stream_index, int subtitle_stream_index,
        FrameAllocator::Frame *audio_frame, AudioFormat *audio_format, int64_t *audio_pts, bool *error)
 {
        *error = false;
@@ -672,6 +674,9 @@ AVFrameWithDeleter FFmpegCapture::decode_frame(AVFormatContext *format_ctx, AVCo
                                        *error = true;
                                        return AVFrameWithDeleter(nullptr);
                                }
+                       } else if (pkt.stream_index == subtitle_stream_index) {
+                               last_subtitle = string(reinterpret_cast<const char *>(pkt.data), pkt.size);
+                               has_last_subtitle = true;
                        }
                } else {
                        eof = true;  // Or error, but ignore that for the time being.

diff --git a/nageru/ffmpeg_capture.h b/nageru/ffmpeg_capture.h
index 468c213d17f677a9235242a5d9f7b1bcecdeaba3..a9a837115977348265115bb77ca0b59a935eecc9 100644 (file)
--- a/nageru/ffmpeg_capture.h
+++ b/nageru/ffmpeg_capture.h
@@ -17,6 +17,11 @@
 // changes parameters midway, which is allowed in some formats.
 //
 // You can get out the audio either as decoded or in raw form (Kaeru uses this).
+//
+// If there's a subtitle track, you can also get out the last subtitle at the
+// point of the frame. Note that once we get a video frame, we don't look for
+// subtitle, so if subtitles and a frame comes at the same time, you might not
+// see the subtitle until the next frame.
 
 #include <assert.h>
 #include <stdint.h>
@@ -164,6 +169,18 @@ public:
                return current_frame_ycbcr_format;
        }
 
+       // Only valid to call during the frame callback.
+       std::string get_last_subtitle() const
+       {
+               return last_subtitle;
+       }
+
+       // Same.
+       bool get_has_last_subtitle() const
+       {
+               return has_last_subtitle;
+       }
+
        void set_dequeue_thread_callbacks(std::function<void()> init, std::function<void()> cleanup) override
        {
                dequeue_init_callback = init;
@@ -218,7 +235,7 @@ private:
 
        // Returns nullptr if no frame was decoded (e.g. EOF).
        AVFrameWithDeleter decode_frame(AVFormatContext *format_ctx, AVCodecContext *video_codec_ctx, AVCodecContext *audio_codec_ctx,
-                                       const std::string &pathname, int video_stream_index, int audio_stream_index,
+                                       const std::string &pathname, int video_stream_index, int audio_stream_index, int subtitle_stream_index,
                                        bmusb::FrameAllocator::Frame *audio_frame, bmusb::AudioFormat *audio_format, int64_t *audio_pts, bool *error);
        void convert_audio(const AVFrame *audio_avframe, bmusb::FrameAllocator::Frame *audio_frame, bmusb::AudioFormat *audio_format);
 
@@ -276,6 +293,9 @@ private:
        int64_t last_channel_layout;
        int last_sample_rate;
 
+       // Subtitles (no decoding done, really).
+       bool has_last_subtitle = false;
+       std::string last_subtitle;
 };
 
 #endif  // !defined(_FFMPEG_CAPTURE_H)

diff --git a/nageru/mixer.cpp b/nageru/mixer.cpp
index 7ef2a4c576d185fb72a9e0895cb9cd3142248a8b..9e28d4c25c5abfd3ef622646bcc8dbfe0606cf67 100644 (file)
--- a/nageru/mixer.cpp
+++ b/nageru/mixer.cpp
@@ -795,6 +795,11 @@ void Mixer::bm_frame(unsigned card_index, uint16_t timecode,
        card->last_timecode = timecode;
 
        PBOFrameAllocator::Userdata *userdata = (PBOFrameAllocator::Userdata *)video_frame.userdata;
+       if (card->type == CardType::FFMPEG_INPUT) {
+               FFmpegCapture *ffmpeg_capture = static_cast<FFmpegCapture *>(card->capture.get());
+               userdata->has_last_subtitle = ffmpeg_capture->get_has_last_subtitle();
+               userdata->last_subtitle = ffmpeg_capture->get_last_subtitle();
+       }
 
        size_t cbcr_width, cbcr_height, cbcr_offset, y_offset;
        size_t expected_length = video_format.stride * (video_format.height + video_format.extra_lines_top + video_format.extra_lines_bottom);

diff --git a/nageru/pbo_frame_allocator.h b/nageru/pbo_frame_allocator.h
index eead7f4a32f979c8203a184f535862c6f1dab0e2..ab51f6bc108839297363dd0ede80860dacfb4481 100644 (file)
--- a/nageru/pbo_frame_allocator.h
+++ b/nageru/pbo_frame_allocator.h
@@ -52,6 +52,8 @@ public:
                GLuint last_v210_width[2];  // PixelFormat_10BitYCbCr.
                bool last_interlaced, last_has_signal, last_is_connected;
                unsigned last_frame_rate_nom, last_frame_rate_den;
+               bool has_last_subtitle = false;
+               std::string last_subtitle;
        };
 
 private:

diff --git a/nageru/theme.cpp b/nageru/theme.cpp
index 386a425fd024be011e1d4591f05b655db92007c8..ead38fa819e877fceee43ec32cf4aa52a183dc92 100644 (file)
--- a/nageru/theme.cpp
+++ b/nageru/theme.cpp
@@ -72,6 +72,8 @@ struct InputStateInfo {
        unsigned last_width[MAX_VIDEO_CARDS], last_height[MAX_VIDEO_CARDS];
        bool last_interlaced[MAX_VIDEO_CARDS], last_has_signal[MAX_VIDEO_CARDS], last_is_connected[MAX_VIDEO_CARDS];
        unsigned last_frame_rate_nom[MAX_VIDEO_CARDS], last_frame_rate_den[MAX_VIDEO_CARDS];
+       bool has_last_subtitle[MAX_VIDEO_CARDS];
+       std::string last_subtitle[MAX_VIDEO_CARDS];
 };
 
 InputStateInfo::InputStateInfo(const InputState &input_state)
@@ -93,6 +95,8 @@ InputStateInfo::InputStateInfo(const InputState &input_state)
                last_is_connected[signal_num] = userdata->last_is_connected;
                last_frame_rate_nom[signal_num] = userdata->last_frame_rate_nom;
                last_frame_rate_den[signal_num] = userdata->last_frame_rate_den;
+               has_last_subtitle[signal_num] = userdata->has_last_subtitle;
+               last_subtitle[signal_num] = userdata->last_subtitle;
        }
 }
 
@@ -634,6 +638,20 @@ int InputStateInfo_get_frame_rate_den(lua_State* L)
        return 1;
 }
 
+int InputStateInfo_get_last_subtitle(lua_State* L)
+{
+       assert(lua_gettop(L) == 2);
+       InputStateInfo *input_state_info = get_input_state_info(L, 1);
+       Theme *theme = get_theme_updata(L);
+       int signal_num = theme->map_signal(luaL_checknumber(L, 2));
+       if (!input_state_info->has_last_subtitle[signal_num]) {
+               lua_pushnil(L);
+       } else {
+               lua_pushstring(L, input_state_info->last_subtitle[signal_num].c_str());
+       }
+       return 1;
+}
+
 int Effect_set_float(lua_State *L)
 {
        assert(lua_gettop(L) == 3);
@@ -829,6 +847,7 @@ const luaL_Reg InputStateInfo_funcs[] = {
        { "get_is_connected", InputStateInfo_get_is_connected },
        { "get_frame_rate_nom", InputStateInfo_get_frame_rate_nom },
        { "get_frame_rate_den", InputStateInfo_get_frame_rate_den },
+       { "get_last_subtitle", InputStateInfo_get_last_subtitle },
        { NULL, NULL }
 };