Make multitrack export include audio.

diff --git a/futatabi/export.cpp b/futatabi/export.cpp
index 5b8da136370a7dd8b649bce379e436daa7e8b176..1b7c59c90ae75f853b1d9eaf6b7f27a190d06996 100644 (file)
--- a/futatabi/export.cpp
+++ b/futatabi/export.cpp
@@ -6,6 +6,7 @@
 #include "frame_on_disk.h"
 #include "player.h"
 #include "shared/ffmpeg_raii.h"
 #include "frame_on_disk.h"
 #include "player.h"
 #include "shared/ffmpeg_raii.h"

+#include "shared/shared_defs.h"

 #include "shared/timebase.h"
 
 #include <QMessageBox>
 #include "shared/timebase.h"
 
 #include <QMessageBox>


@@ -23,22 +24,22 @@ using namespace std;
 namespace {
 
 // Only used in export_cliplist_clip_multitrack_triggered.
 namespace {
 
 // Only used in export_cliplist_clip_multitrack_triggered.

-struct BufferedJPEG {
+struct BufferedFrame {

        int64_t pts;
        int64_t pts;

-       unsigned stream_idx;
-       string jpeg;
+       unsigned video_stream_idx;
+       string data;

 };
 
 };
 

-bool write_buffered_jpegs(AVFormatContext *avctx, const vector<BufferedJPEG> &buffered_jpegs)
+bool write_buffered_frames(AVFormatContext *avctx, const vector<BufferedFrame> &buffered_frames)

 {
 {

-       for (const BufferedJPEG &jpeg : buffered_jpegs) {
+       for (const BufferedFrame &frame : buffered_frames) {

                AVPacket pkt;
                av_init_packet(&pkt);
                AVPacket pkt;
                av_init_packet(&pkt);

-               pkt.stream_index = jpeg.stream_idx;
-               pkt.data = (uint8_t *)jpeg.jpeg.data();
-               pkt.size = jpeg.jpeg.size();
-               pkt.pts = jpeg.pts;
-               pkt.dts = jpeg.pts;
+               pkt.stream_index = frame.video_stream_idx;
+               pkt.data = (uint8_t *)frame.data.data();
+               pkt.size = frame.data.size();
+               pkt.pts = frame.pts;
+               pkt.dts = frame.pts;

                pkt.flags = AV_PKT_FLAG_KEY;
 
                if (av_write_frame(avctx, &pkt) < 0) {
                pkt.flags = AV_PKT_FLAG_KEY;
 
                if (av_write_frame(avctx, &pkt) < 0) {


@@ -123,6 +124,23 @@ void export_multitrack_clip(const string &filename, const Clip &clip)
                video_streams.push_back(avstream_video);
        }
 
                video_streams.push_back(avstream_video);
        }
 

+       // Similar, for audio streams.
+       vector<AVStream *> audio_streams;
+       for (unsigned stream_idx = 0; stream_idx <= last_stream_idx; ++stream_idx) {
+               AVStream *avstream_audio = avformat_new_stream(avctx, nullptr);
+               if (avstream_audio == nullptr) {
+                       fprintf(stderr, "avformat_new_stream() failed\n");
+                       abort();
+               }
+               avstream_audio->time_base = AVRational{ 1, TIMEBASE };
+               avstream_audio->codecpar->codec_type = AVMEDIA_TYPE_AUDIO;
+               avstream_audio->codecpar->codec_id = AV_CODEC_ID_PCM_S32LE;
+               avstream_audio->codecpar->channel_layout = AV_CH_LAYOUT_STEREO;
+               avstream_audio->codecpar->channels = 2;
+               avstream_audio->codecpar->sample_rate = OUTPUT_FREQUENCY;
+               audio_streams.push_back(avstream_audio);
+       }
+

        if (avformat_write_header(avctx, nullptr) < 0) {
                QMessageBox msgbox;
                msgbox.setText("Writing header failed");
        if (avformat_write_header(avctx, nullptr) < 0) {
                QMessageBox msgbox;
                msgbox.setText("Writing header failed");


@@ -140,7 +158,7 @@ void export_multitrack_clip(const string &filename, const Clip &clip)
 
        // We buffer up to 1000 frames at a time, in a hope that we can reduce
        // the amount of seeking needed on rotational media.
 
        // We buffer up to 1000 frames at a time, in a hope that we can reduce
        // the amount of seeking needed on rotational media.

-       vector<BufferedJPEG> buffered_jpegs;
+       vector<BufferedFrame> buffered_frames;

        size_t frames_written = 0;
        while (num_streams_with_frames_left > 0) {
                // Find the stream with the lowest frame. Lower stream indexes win.
        size_t frames_written = 0;
        while (num_streams_with_frames_left > 0) {
                // Find the stream with the lowest frame. Lower stream indexes win.


@@ -163,21 +181,35 @@ void export_multitrack_clip(const string &filename, const Clip &clip)
                                --num_streams_with_frames_left;
                        }
                }
                                --num_streams_with_frames_left;
                        }
                }

-               string jpeg = readers[first_frame_stream_idx].read_frame(first_frame);
-               int64_t scaled_pts = av_rescale_q(first_frame.pts, AVRational{ 1, TIMEBASE },
-                                                 video_streams[first_frame_stream_idx]->time_base);
-               buffered_jpegs.emplace_back(BufferedJPEG{ scaled_pts, first_frame_stream_idx, std::move(jpeg) });
-               if (buffered_jpegs.size() >= 1000) {
-                       if (!write_buffered_jpegs(avctx, buffered_jpegs)) {
+
+               FrameReader::Frame frame = readers[first_frame_stream_idx].read_frame(first_frame, /*read_audio=*/true);
+
+               // Write audio. (Before video, since that's what we expect on input.)
+               if (!frame.audio.empty()) {
+                       unsigned audio_stream_idx = first_frame_stream_idx + video_streams.size();
+                       int64_t scaled_audio_pts = av_rescale_q(first_frame.pts, AVRational{ 1, TIMEBASE },
+                                                               audio_streams[first_frame_stream_idx]->time_base);
+                       buffered_frames.emplace_back(BufferedFrame{ scaled_audio_pts, audio_stream_idx, std::move(frame.audio) });
+               }
+
+               // Write video.
+               unsigned video_stream_idx = first_frame_stream_idx;
+               int64_t scaled_video_pts = av_rescale_q(first_frame.pts, AVRational{ 1, TIMEBASE },
+                                                       video_streams[first_frame_stream_idx]->time_base);
+               buffered_frames.emplace_back(BufferedFrame{ scaled_video_pts, video_stream_idx, std::move(frame.video) });
+
+               // Flush to disk if required.
+               if (buffered_frames.size() >= 1000) {
+                       if (!write_buffered_frames(avctx, buffered_frames)) {

                                QMessageBox msgbox;
                                msgbox.setText("Writing frames failed");
                                msgbox.exec();
                                unlink(filename.c_str());
                                return;
                        }
                                QMessageBox msgbox;
                                msgbox.setText("Writing frames failed");
                                msgbox.exec();
                                unlink(filename.c_str());
                                return;
                        }

-                       frames_written += buffered_jpegs.size();
+                       frames_written += buffered_frames.size();

                        progress.setValue(frames_written);
                        progress.setValue(frames_written);

-                       buffered_jpegs.clear();
+                       buffered_frames.clear();

                }
                if (progress.wasCanceled()) {
                        unlink(filename.c_str());
                }
                if (progress.wasCanceled()) {
                        unlink(filename.c_str());


@@ -185,14 +217,14 @@ void export_multitrack_clip(const string &filename, const Clip &clip)
                }
        }
 
                }
        }
 

-       if (!write_buffered_jpegs(avctx, buffered_jpegs)) {
+       if (!write_buffered_frames(avctx, buffered_frames)) {

                QMessageBox msgbox;
                msgbox.setText("Writing frames failed");
                msgbox.exec();
                unlink(filename.c_str());
                return;
        }
                QMessageBox msgbox;
                msgbox.setText("Writing frames failed");
                msgbox.exec();
                unlink(filename.c_str());
                return;
        }

-       frames_written += buffered_jpegs.size();
+       frames_written += buffered_frames.size();

        progress.setValue(frames_written);
 }
 
        progress.setValue(frames_written);
 }
 

diff --git a/futatabi/frame_on_disk.cpp b/futatabi/frame_on_disk.cpp
index 9a870947b4b84c171de2897ae5ad9523a88f6e52..6bdaf23a25569edb97d691a26ec1f3949dc792b1 100644 (file)
--- a/futatabi/frame_on_disk.cpp
+++ b/futatabi/frame_on_disk.cpp
@@ -47,7 +47,28 @@ FrameReader::~FrameReader()
        }
 }
 
        }
 }
 

-string FrameReader::read_frame(FrameOnDisk frame)
+namespace {
+
+string read_string(int fd, size_t size, off_t offset)
+{
+       string str;
+       str.resize(size);
+       size_t str_offset = 0;
+       while (str_offset < size) {
+               int ret = pread(fd, &str[str_offset], size - str_offset, offset + str_offset);
+               if (ret <= 0) {
+                       perror("pread");
+                       abort();
+               }
+
+               str_offset += ret;
+       }
+       return str;
+}
+
+}  // namespace
+
+FrameReader::Frame FrameReader::read_frame(FrameOnDisk frame, bool read_audio)

 {
        steady_clock::time_point start = steady_clock::now();
 
 {
        steady_clock::time_point start = steady_clock::now();
 


@@ -76,20 +97,10 @@ string FrameReader::read_frame(FrameOnDisk frame)
                ++metric_frame_opened_files;
        }
 
                ++metric_frame_opened_files;
        }
 

-       // TODO: Read the audio.
-
-       string str;
-       str.resize(frame.size);
-       off_t offset = 0;
-       while (offset < frame.size) {
-               int ret = pread(fd, &str[offset], frame.size - offset, frame.offset + offset);
-               if (ret <= 0) {
-                       string filename = frame_filenames[frame.filename_idx];
-                       perror("pread");
-                       abort();
-               }
-
-               offset += ret;
+       Frame ret;
+       ret.video = read_string(fd, frame.size, frame.offset);
+       if (read_audio) {
+               ret.audio = read_string(fd, frame.audio_size, frame.offset + frame.size);

        }
 
        steady_clock::time_point stop = steady_clock::now();
        }
 
        steady_clock::time_point stop = steady_clock::now();


@@ -98,5 +109,5 @@ string FrameReader::read_frame(FrameOnDisk frame)
        metric_frame_read_bytes += frame.size;
        ++metric_frame_read_frames;
 
        metric_frame_read_bytes += frame.size;
        ++metric_frame_read_frames;
 

-       return str;
+       return ret;

 }
 }

diff --git a/futatabi/frame_on_disk.h b/futatabi/frame_on_disk.h
index dbe121113f185bb9011ce3c58c9d0a835a5daa0c..360bd23ce1222271cfef1f82df5006de0b141a60 100644 (file)
--- a/futatabi/frame_on_disk.h
+++ b/futatabi/frame_on_disk.h
@@ -41,7 +41,12 @@ class FrameReader {
 public:
        FrameReader();
        ~FrameReader();
 public:
        FrameReader();
        ~FrameReader();

-       std::string read_frame(FrameOnDisk frame);
+
+       struct Frame {
+               std::string video;
+               std::string audio;
+       };
+       Frame read_frame(FrameOnDisk frame, bool read_audio);

 
 private:
        int fd = -1;
 
 private:
        int fd = -1;

diff --git a/futatabi/jpeg_frame_view.cpp b/futatabi/jpeg_frame_view.cpp
index 198affd38ecd5324cc58977dc75c94d2741ac930..c1afafd765ca32f803c35c23b821c949bb807e91 100644 (file)
--- a/futatabi/jpeg_frame_view.cpp
+++ b/futatabi/jpeg_frame_view.cpp
@@ -238,7 +238,7 @@ shared_ptr<Frame> decode_jpeg_with_cache(FrameOnDisk frame_spec, CacheMissBehavi
        ++metric_jpeg_cache_miss_frames;
 
        *did_decode = true;
        ++metric_jpeg_cache_miss_frames;
 
        *did_decode = true;

-       shared_ptr<Frame> frame = decode_jpeg(frame_reader->read_frame(frame_spec));
+       shared_ptr<Frame> frame = decode_jpeg(frame_reader->read_frame(frame_spec, /*read_audio=*/false).video);

 
        lock_guard<mutex> lock(cache_mu);
        cache_bytes_used += frame_size(*frame);
 
        lock_guard<mutex> lock(cache_mu);
        cache_bytes_used += frame_size(*frame);

diff --git a/futatabi/video_stream.cpp b/futatabi/video_stream.cpp
index 4b0336ce3b8384a68673e7a5cf6fd7a0f603e9b2..06acfd2601a43ebe4ff17af8b0f7fffee788fc25 100644 (file)
--- a/futatabi/video_stream.cpp
+++ b/futatabi/video_stream.cpp
@@ -335,6 +335,8 @@ void VideoStream::schedule_original_frame(steady_clock::time_point local_pts,
 {
        fprintf(stderr, "output_pts=%" PRId64 "  original      input_pts=%" PRId64 "\n", output_pts, frame.pts);
 
 {
        fprintf(stderr, "output_pts=%" PRId64 "  original      input_pts=%" PRId64 "\n", output_pts, frame.pts);
 

+       // TODO: Write audio if at the right speed.
+

        QueuedFrame qf;
        qf.local_pts = local_pts;
        qf.type = QueuedFrame::ORIGINAL;
        QueuedFrame qf;
        qf.local_pts = local_pts;
        qf.type = QueuedFrame::ORIGINAL;


@@ -342,7 +344,7 @@ void VideoStream::schedule_original_frame(steady_clock::time_point local_pts,
        qf.display_func = move(display_func);
        qf.queue_spot_holder = move(queue_spot_holder);
        qf.subtitle = subtitle;
        qf.display_func = move(display_func);
        qf.queue_spot_holder = move(queue_spot_holder);
        qf.subtitle = subtitle;

-       qf.encoded_jpeg.reset(new string(frame_reader.read_frame(frame)));
+       qf.encoded_jpeg.reset(new string(frame_reader.read_frame(frame, /*read_audio=*/false).video));

 
        lock_guard<mutex> lock(queue_lock);
        frame_queue.push_back(move(qf));
 
        lock_guard<mutex> lock(queue_lock);
        frame_queue.push_back(move(qf));

diff --git a/nageru/defs.h b/nageru/defs.h
index a990330759348796474d219bd1a7b3c4d405effa..6d684e79415bc71a9befa302cafccf42d1809df1 100644 (file)
--- a/nageru/defs.h
+++ b/nageru/defs.h
@@ -3,7 +3,6 @@
 
 #include <libavformat/version.h>
 
 
 #include <libavformat/version.h>
 

-#define OUTPUT_FREQUENCY 48000  // Currently needs to be exactly 48000, since bmusb outputs in that.

 #define MAX_FPS 60
 #define FAKE_FPS 25  // Must be an integer.
 #define MAX_VIDEO_CARDS 16
 #define MAX_FPS 60
 #define FAKE_FPS 25  // Must be an integer.
 #define MAX_VIDEO_CARDS 16

diff --git a/shared/shared_defs.h b/shared/shared_defs.h
index 20c56bf4eb0ed02f0c39adfc3a2e6980ca11d082..62b719dddaf8ad3c795fe8a5bc5be6643b264f9d 100644 (file)
--- a/shared/shared_defs.h
+++ b/shared/shared_defs.h
@@ -1,6 +1,8 @@
 #ifndef _SHARED_DEFS_H
 #define _SHARED_DEFS_H 1
 
 #ifndef _SHARED_DEFS_H
 #define _SHARED_DEFS_H 1
 

+#define OUTPUT_FREQUENCY 48000  // Currently needs to be exactly 48000, since bmusb outputs in that.
+

 #define MUX_OPTS { \
        /* Make seekable .mov files, and keep MP4 muxer from using unlimited amounts of memory. */ \
        { "movflags", "empty_moov+frag_keyframe+default_base_moof+skip_trailer" }, \
 #define MUX_OPTS { \
        /* Make seekable .mov files, and keep MP4 muxer from using unlimited amounts of memory. */ \
        { "movflags", "empty_moov+frag_keyframe+default_base_moof+skip_trailer" }, \