Fix an issue where the mixer lagging too much behind CEF would cause us to display...

[nageru] / ffmpeg_capture.h

diff --git a/ffmpeg_capture.h b/ffmpeg_capture.h
index afca641e951e20031bf37bf2f9ed2d0cad9413a6..f718c70a23ff4dccceb7df40877f389e44815266 100644 (file)
--- a/ffmpeg_capture.h
+++ b/ffmpeg_capture.h
@@ -16,7 +16,8 @@
 // but it would require some more plumbing, and it would also fail if the file
 // changes parameters midway, which is allowed in some formats.
 //
-// There is currently no audio support.
+// You can get out the audio either as decoded or in raw form (Kaeru uses this).
+// However, the rest of Nageru can't really use the audio for anything yet.
 
 #include <assert.h>
 #include <stdint.h>
@@ -31,12 +32,15 @@
 #include <movit/ycbcr.h>
 
 extern "C" {
+#include <libavresample/avresample.h>
 #include <libavutil/pixfmt.h>
 #include <libavutil/rational.h>
+#include <libavutil/samplefmt.h>
 }
 
 #include "bmusb/bmusb.h"
 #include "ffmpeg_raii.h"
+#include "ref_counted_frame.h"
 #include "quittable_sleeper.h"
 
 struct AVFormatContext;
@@ -104,7 +108,8 @@ public:
 
        // FFmpegCapture-specific overload of set_frame_callback that also gives
        // the raw original pts from the video. Negative pts means a dummy frame.
-       typedef std::function<void(int64_t pts, AVRational timebase, uint16_t timecode,
+       typedef std::function<void(int64_t video_pts, AVRational video_timebase, int64_t audio_pts, AVRational audio_timebase,
+                                  uint16_t timecode,
                                   bmusb::FrameAllocator::Frame video_frame, size_t video_offset, bmusb::VideoFormat video_format,
                                   bmusb::FrameAllocator::Frame audio_frame, size_t audio_offset, bmusb::AudioFormat audio_format)>
                frame_callback_t;
@@ -117,13 +122,13 @@ public:
        {
                frame_callback = std::bind(
                        callback,
-                       std::placeholders::_3,
-                       std::placeholders::_4,
                        std::placeholders::_5,
                        std::placeholders::_6,
                        std::placeholders::_7,
                        std::placeholders::_8,
-                       std::placeholders::_9);
+                       std::placeholders::_9,
+                       std::placeholders::_10,
+                       std::placeholders::_11);
        }
 
        // FFmpegCapture-specific callback that gives the raw audio.
@@ -194,10 +199,13 @@ private:
        bool process_queued_commands(AVFormatContext *format_ctx, const std::string &pathname, timespec last_modified, bool *rewound);
 
        // Returns nullptr if no frame was decoded (e.g. EOF).
-       AVFrameWithDeleter decode_frame(AVFormatContext *format_ctx, AVCodecContext *codec_ctx, const std::string &pathname, int video_stream_index, int audio_stream_index, bool *error);
+       AVFrameWithDeleter decode_frame(AVFormatContext *format_ctx, AVCodecContext *video_codec_ctx, AVCodecContext *audio_codec_ctx,
+                                       const std::string &pathname, int video_stream_index, int audio_stream_index,
+                                       bmusb::FrameAllocator::Frame *audio_frame, bmusb::AudioFormat *audio_format, int64_t *audio_pts, bool *error);
+       void convert_audio(const AVFrame *audio_avframe, bmusb::FrameAllocator::Frame *audio_frame, bmusb::AudioFormat *audio_format);
 
        bmusb::VideoFormat construct_video_format(const AVFrame *frame, AVRational video_timebase);
-       bmusb::FrameAllocator::Frame make_video_frame(const AVFrame *frame, const std::string &pathname, bool *error);
+       UniqueFrame make_video_frame(const AVFrame *frame, const std::string &pathname, bool *error);
 
        std::string description, filename;
        uint16_t timecode = 0;
@@ -222,7 +230,7 @@ private:
        SwsContextWithDeleter sws_ctx;
        int sws_last_width = -1, sws_last_height = -1, sws_last_src_format = -1;
        AVPixelFormat sws_dst_format = AVPixelFormat(-1);  // In practice, always initialized.
-       AVRational video_timebase;
+       AVRational video_timebase, audio_timebase;
 
        QuittableSleeper producer_thread_should_quit;
        std::thread producer_thread;
@@ -236,6 +244,13 @@ private:
                double new_rate;  // For CHANGE_RATE.
        };
        std::vector<QueuedCommand> command_queue;  // Protected by <queue_mu>.
+
+       // Audio resampler.
+       AVAudioResampleContext *resampler = nullptr;
+       AVSampleFormat last_src_format, last_dst_format;
+       int64_t last_channel_layout;
+       int last_sample_rate;
+
 };
 
 #endif  // !defined(_FFMPEG_CAPTURE_H)