]> git.sesse.net Git - nageru/blob - mux.cpp
Fix non-transcoded audio in Kaeru in streams where the audio does not have index 1.
[nageru] / mux.cpp
1 #include "mux.h"
2
3 #include <assert.h>
4 #include <stdint.h>
5 #include <stdio.h>
6 #include <stdlib.h>
7 #include <string.h>
8 #include <algorithm>
9 #include <mutex>
10 #include <string>
11 #include <utility>
12 #include <vector>
13
14 extern "C" {
15 #include <libavformat/avio.h>
16 #include <libavutil/avutil.h>
17 #include <libavutil/dict.h>
18 #include <libavutil/mathematics.h>
19 #include <libavutil/mem.h>
20 #include <libavutil/pixfmt.h>
21 #include <libavutil/rational.h>
22 }
23
24 #include "defs.h"
25 #include "flags.h"
26 #include "metrics.h"
27 #include "timebase.h"
28
29 using namespace std;
30
31 struct PacketBefore {
32         PacketBefore(const AVFormatContext *ctx) : ctx(ctx) {}
33
34         bool operator() (const Mux::QueuedPacket &a_qp, const Mux::QueuedPacket &b_qp) const {
35                 const AVPacket *a = a_qp.pkt;
36                 const AVPacket *b = b_qp.pkt;
37                 int64_t a_dts = (a->dts == AV_NOPTS_VALUE ? a->pts : a->dts);
38                 int64_t b_dts = (b->dts == AV_NOPTS_VALUE ? b->pts : b->dts);
39                 AVRational a_timebase = ctx->streams[a->stream_index]->time_base;
40                 AVRational b_timebase = ctx->streams[b->stream_index]->time_base;
41                 if (av_compare_ts(a_dts, a_timebase, b_dts, b_timebase) != 0) {
42                         return av_compare_ts(a_dts, a_timebase, b_dts, b_timebase) < 0;
43                 } else {
44                         return av_compare_ts(a->pts, a_timebase, b->pts, b_timebase) < 0;
45                 }
46         }
47
48         const AVFormatContext * const ctx;
49 };
50
51 Mux::Mux(AVFormatContext *avctx, int width, int height, Codec video_codec, const string &video_extradata, const AVCodecParameters *audio_codecpar, int time_base, std::function<void(int64_t)> write_callback, WriteStrategy write_strategy, const vector<MuxMetrics *> &metrics)
52         : write_strategy(write_strategy), avctx(avctx), write_callback(write_callback), metrics(metrics)
53 {
54         avstream_video = avformat_new_stream(avctx, nullptr);
55         if (avstream_video == nullptr) {
56                 fprintf(stderr, "avformat_new_stream() failed\n");
57                 exit(1);
58         }
59         avstream_video->time_base = AVRational{1, time_base};
60         avstream_video->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
61         if (video_codec == CODEC_H264) {
62                 avstream_video->codecpar->codec_id = AV_CODEC_ID_H264;
63         } else {
64                 assert(video_codec == CODEC_NV12);
65                 avstream_video->codecpar->codec_id = AV_CODEC_ID_RAWVIDEO;
66                 avstream_video->codecpar->codec_tag = avcodec_pix_fmt_to_codec_tag(AV_PIX_FMT_NV12);
67         }
68         avstream_video->codecpar->width = width;
69         avstream_video->codecpar->height = height;
70
71         // Colorspace details. Closely correspond to settings in EffectChain_finalize,
72         // as noted in each comment.
73         // Note that the H.264 stream also contains this information and depending on the
74         // mux, this might simply get ignored. See sps_rbsp().
75         // Note that there's no way to change this per-frame as the H.264 stream
76         // would like to be able to.
77         avstream_video->codecpar->color_primaries = AVCOL_PRI_BT709;  // RGB colorspace (inout_format.color_space).
78         avstream_video->codecpar->color_trc = AVCOL_TRC_IEC61966_2_1;  // Gamma curve (inout_format.gamma_curve).
79         // YUV colorspace (output_ycbcr_format.luma_coefficients).
80         if (global_flags.ycbcr_rec709_coefficients) {
81                 avstream_video->codecpar->color_space = AVCOL_SPC_BT709;
82         } else {
83                 avstream_video->codecpar->color_space = AVCOL_SPC_SMPTE170M;
84         }
85         avstream_video->codecpar->color_range = AVCOL_RANGE_MPEG;  // Full vs. limited range (output_ycbcr_format.full_range).
86         avstream_video->codecpar->chroma_location = AVCHROMA_LOC_LEFT;  // Chroma sample location. See chroma_offset_0[] in Mixer::subsample_chroma().
87         avstream_video->codecpar->field_order = AV_FIELD_PROGRESSIVE;
88
89         if (!video_extradata.empty()) {
90                 avstream_video->codecpar->extradata = (uint8_t *)av_malloc(video_extradata.size());
91                 avstream_video->codecpar->extradata_size = video_extradata.size();
92                 memcpy(avstream_video->codecpar->extradata, video_extradata.data(), video_extradata.size());
93         }
94
95         avstream_audio = avformat_new_stream(avctx, nullptr);
96         if (avstream_audio == nullptr) {
97                 fprintf(stderr, "avformat_new_stream() failed\n");
98                 exit(1);
99         }
100         avstream_audio->time_base = AVRational{1, time_base};
101         if (avcodec_parameters_copy(avstream_audio->codecpar, audio_codecpar) < 0) {
102                 fprintf(stderr, "avcodec_parameters_copy() failed\n");
103                 exit(1);
104         }
105
106         AVDictionary *options = NULL;
107         vector<pair<string, string>> opts = MUX_OPTS;
108         for (pair<string, string> opt : opts) {
109                 av_dict_set(&options, opt.first.c_str(), opt.second.c_str(), 0);
110         }
111         if (avformat_write_header(avctx, &options) < 0) {
112                 fprintf(stderr, "avformat_write_header() failed\n");
113                 exit(1);
114         }
115         for (MuxMetrics *metric : metrics) {
116                 metric->metric_written_bytes += avctx->pb->pos;
117         }
118
119         // Make sure the header is written before the constructor exits.
120         avio_flush(avctx->pb);
121
122         if (write_strategy == WRITE_BACKGROUND) {
123                 writer_thread = thread(&Mux::thread_func, this);
124         }
125 }
126
127 Mux::~Mux()
128 {
129         assert(plug_count == 0);
130         if (write_strategy == WRITE_BACKGROUND) {
131                 writer_thread_should_quit = true;
132                 packet_queue_ready.notify_all();
133                 writer_thread.join();
134         }
135         int64_t old_pos = avctx->pb->pos;
136         av_write_trailer(avctx);
137         for (MuxMetrics *metric : metrics) {
138                 metric->metric_written_bytes += avctx->pb->pos - old_pos;
139         }
140
141         if (!(avctx->oformat->flags & AVFMT_NOFILE) &&
142             !(avctx->flags & AVFMT_FLAG_CUSTOM_IO)) {
143                 avio_closep(&avctx->pb);
144         }
145         avformat_free_context(avctx);
146 }
147
148 void Mux::add_packet(const AVPacket &pkt, int64_t pts, int64_t dts, AVRational timebase, int stream_index_override)
149 {
150         AVPacket pkt_copy;
151         av_init_packet(&pkt_copy);
152         if (av_packet_ref(&pkt_copy, &pkt) < 0) {
153                 fprintf(stderr, "av_copy_packet() failed\n");
154                 exit(1);
155         }
156         if (stream_index_override != -1) {
157                 pkt_copy.stream_index = stream_index_override;
158         }
159         if (pkt_copy.stream_index == 0) {
160                 pkt_copy.pts = av_rescale_q(pts, timebase, avstream_video->time_base);
161                 pkt_copy.dts = av_rescale_q(dts, timebase, avstream_video->time_base);
162                 pkt_copy.duration = av_rescale_q(pkt.duration, timebase, avstream_video->time_base);
163         } else if (pkt_copy.stream_index == 1) {
164                 pkt_copy.pts = av_rescale_q(pts, timebase, avstream_audio->time_base);
165                 pkt_copy.dts = av_rescale_q(dts, timebase, avstream_audio->time_base);
166                 pkt_copy.duration = av_rescale_q(pkt.duration, timebase, avstream_audio->time_base);
167         } else {
168                 assert(false);
169         }
170
171         {
172                 lock_guard<mutex> lock(mu);
173                 if (write_strategy == WriteStrategy::WRITE_BACKGROUND) {
174                         packet_queue.push_back(QueuedPacket{ av_packet_clone(&pkt_copy), pts });
175                         if (plug_count == 0) packet_queue_ready.notify_all();
176                 } else if (plug_count > 0) {
177                         packet_queue.push_back(QueuedPacket{ av_packet_clone(&pkt_copy), pts });
178                 } else {
179                         write_packet_or_die(pkt_copy, pts);
180                 }
181         }
182
183         av_packet_unref(&pkt_copy);
184 }
185
186 void Mux::write_packet_or_die(const AVPacket &pkt, int64_t unscaled_pts)
187 {
188         for (MuxMetrics *metric : metrics) {
189                 if (pkt.stream_index == 0) {
190                         metric->metric_video_bytes += pkt.size;
191                 } else if (pkt.stream_index == 1) {
192                         metric->metric_audio_bytes += pkt.size;
193                 } else {
194                         assert(false);
195                 }
196         }
197         int64_t old_pos = avctx->pb->pos;
198         if (av_interleaved_write_frame(avctx, const_cast<AVPacket *>(&pkt)) < 0) {
199                 fprintf(stderr, "av_interleaved_write_frame() failed\n");
200                 exit(1);
201         }
202         avio_flush(avctx->pb);
203         for (MuxMetrics *metric : metrics) {
204                 metric->metric_written_bytes += avctx->pb->pos - old_pos;
205         }
206
207         if (pkt.stream_index == 0 && write_callback != nullptr) {
208                 write_callback(unscaled_pts);
209         }
210 }
211
212 void Mux::plug()
213 {
214         lock_guard<mutex> lock(mu);
215         ++plug_count;
216 }
217
218 void Mux::unplug()
219 {
220         lock_guard<mutex> lock(mu);
221         if (--plug_count > 0) {
222                 return;
223         }
224         assert(plug_count >= 0);
225
226         sort(packet_queue.begin(), packet_queue.end(), PacketBefore(avctx));
227
228         if (write_strategy == WRITE_BACKGROUND) {
229                 packet_queue_ready.notify_all();
230         } else {
231                 for (QueuedPacket &qp : packet_queue) {
232                         write_packet_or_die(*qp.pkt, qp.unscaled_pts);
233                         av_packet_free(&qp.pkt);
234                 }
235                 packet_queue.clear();
236         }
237 }
238
239 void Mux::thread_func()
240 {
241         unique_lock<mutex> lock(mu);
242         for ( ;; ) {
243                 packet_queue_ready.wait(lock, [this]() {
244                         return writer_thread_should_quit || (!packet_queue.empty() && plug_count == 0);
245                 });
246                 if (writer_thread_should_quit && packet_queue.empty()) {
247                         // All done.
248                         break;
249                 }
250
251                 assert(!packet_queue.empty() && plug_count == 0);
252                 vector<QueuedPacket> packets;
253                 swap(packets, packet_queue);
254
255                 lock.unlock();
256                 for (QueuedPacket &qp : packets) {
257                         write_packet_or_die(*qp.pkt, qp.unscaled_pts);
258                         av_packet_free(&qp.pkt);
259                 }
260                 lock.lock();
261         }
262 }
263
264 void MuxMetrics::init(const vector<pair<string, string>> &labels)
265 {
266         vector<pair<string, string>> labels_video = labels;
267         labels_video.emplace_back("stream", "video");
268         global_metrics.add("mux_stream_bytes", labels_video, &metric_video_bytes);
269
270         vector<pair<string, string>> labels_audio = labels;
271         labels_audio.emplace_back("stream", "audio");
272         global_metrics.add("mux_stream_bytes", labels_audio, &metric_audio_bytes);
273
274         global_metrics.add("mux_written_bytes", labels, &metric_written_bytes);
275 }