1 #include "shared/mux.h"
15 #include <libavformat/avio.h>
16 #include <libavutil/avutil.h>
17 #include <libavutil/dict.h>
18 #include <libavutil/mathematics.h>
19 #include <libavutil/mem.h>
20 #include <libavutil/pixfmt.h>
21 #include <libavutil/rational.h>
24 #include "shared/metrics.h"
25 #include "shared/shared_defs.h"
26 #include "shared/timebase.h"
31 PacketBefore(const AVFormatContext *ctx) : ctx(ctx) {}
33 bool operator() (const Mux::QueuedPacket &a_qp, const Mux::QueuedPacket &b_qp) const {
34 const AVPacket *a = a_qp.pkt;
35 const AVPacket *b = b_qp.pkt;
36 int64_t a_dts = (a->dts == AV_NOPTS_VALUE ? a->pts : a->dts);
37 int64_t b_dts = (b->dts == AV_NOPTS_VALUE ? b->pts : b->dts);
38 AVRational a_timebase = ctx->streams[a->stream_index]->time_base;
39 AVRational b_timebase = ctx->streams[b->stream_index]->time_base;
40 if (av_compare_ts(a_dts, a_timebase, b_dts, b_timebase) != 0) {
41 return av_compare_ts(a_dts, a_timebase, b_dts, b_timebase) < 0;
43 return av_compare_ts(a->pts, a_timebase, b->pts, b_timebase) < 0;
47 const AVFormatContext * const ctx;
50 Mux::Mux(AVFormatContext *avctx, int width, int height, Codec video_codec, const string &video_extradata, const AVCodecParameters *audio_codecpar, AVColorSpace color_space, int time_base, function<void(int64_t)> write_callback, WriteStrategy write_strategy, const vector<MuxMetrics *> &metrics, WithSubtitles with_subtitles)
51 : write_strategy(write_strategy), avctx(avctx), write_callback(write_callback), metrics(metrics)
53 // MPEG-TS ostensibly needs some conversions (e.g. for differing start codes),
54 // so let FFmpeg insert them as needed in case we are muxing to that.
55 // Curiously enough, things actually seem to go quite fine without
56 // (and it also seems FFmpeg's MPEG-TS muxer automatically does stuff like
57 // repeat PPS/SPS before keyframes for us), but it can't hurt.
58 avctx->flags |= AVFMT_FLAG_AUTO_BSF;
60 AVStream *avstream_video = avformat_new_stream(avctx, nullptr);
61 if (avstream_video == nullptr) {
62 fprintf(stderr, "avformat_new_stream() failed\n");
65 avstream_video->time_base = AVRational{1, time_base};
66 avstream_video->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
67 if (video_codec == CODEC_H264) {
68 avstream_video->codecpar->codec_id = AV_CODEC_ID_H264;
69 } else if (video_codec == CODEC_AV1) {
70 avstream_video->codecpar->codec_id = AV_CODEC_ID_AV1;
72 assert(video_codec == CODEC_MJPEG);
73 avstream_video->codecpar->codec_id = AV_CODEC_ID_MJPEG;
75 avstream_video->codecpar->width = width;
76 avstream_video->codecpar->height = height;
78 // Colorspace details. Closely correspond to settings in EffectChain_finalize,
79 // as noted in each comment.
80 // Note that the H.264 stream also contains this information and depending on the
81 // mux, this might simply get ignored. See sps_rbsp().
82 // Note that there's no way to change this per-frame as the H.264 stream
83 // would like to be able to.
84 avstream_video->codecpar->color_primaries = AVCOL_PRI_BT709; // RGB colorspace (inout_format.color_space).
85 avstream_video->codecpar->color_trc = AVCOL_TRC_IEC61966_2_1; // Gamma curve (inout_format.gamma_curve).
86 // YUV colorspace (output_ycbcr_format.luma_coefficients).
87 avstream_video->codecpar->color_space = color_space;
88 avstream_video->codecpar->color_range = AVCOL_RANGE_MPEG; // Full vs. limited range (output_ycbcr_format.full_range).
89 avstream_video->codecpar->chroma_location = AVCHROMA_LOC_LEFT; // Chroma sample location. See chroma_offset_0[] in Mixer::subsample_chroma().
90 avstream_video->codecpar->field_order = AV_FIELD_PROGRESSIVE;
92 if (!video_extradata.empty()) {
93 avstream_video->codecpar->extradata = (uint8_t *)av_malloc(video_extradata.size() + AV_INPUT_BUFFER_PADDING_SIZE);
94 avstream_video->codecpar->extradata_size = video_extradata.size();
95 memcpy(avstream_video->codecpar->extradata, video_extradata.data(), video_extradata.size());
97 streams.push_back(avstream_video);
99 if (audio_codecpar != nullptr) {
100 AVStream *avstream_audio = avformat_new_stream(avctx, nullptr);
101 if (avstream_audio == nullptr) {
102 fprintf(stderr, "avformat_new_stream() failed\n");
105 avstream_audio->time_base = AVRational{1, time_base};
106 if (avcodec_parameters_copy(avstream_audio->codecpar, audio_codecpar) < 0) {
107 fprintf(stderr, "avcodec_parameters_copy() failed\n");
110 streams.push_back(avstream_audio);
113 if (with_subtitles == WITH_SUBTITLES) {
114 AVStream *avstream_subtitles = avformat_new_stream(avctx, nullptr);
115 if (avstream_subtitles == nullptr) {
116 fprintf(stderr, "avformat_new_stream() failed\n");
119 avstream_subtitles->time_base = AVRational{1, time_base};
120 avstream_subtitles->codecpar->codec_type = AVMEDIA_TYPE_SUBTITLE;
121 avstream_subtitles->codecpar->codec_id = AV_CODEC_ID_WEBVTT;
122 avstream_subtitles->disposition = AV_DISPOSITION_METADATA;
123 streams.push_back(avstream_subtitles);
124 subtitle_stream_idx = streams.size() - 1;
127 if (write_strategy == WRITE_BACKGROUND) {
128 writer_thread = thread(&Mux::thread_func, this);
136 assert(plug_count == 0);
137 if (write_strategy == WRITE_BACKGROUND) {
138 writer_thread_should_quit = true;
139 packet_queue_ready.notify_all();
140 writer_thread.join();
142 int64_t old_pos = avctx->pb->pos;
143 av_write_trailer(avctx);
144 for (MuxMetrics *metric : metrics) {
145 metric->metric_written_bytes += avctx->pb->pos - old_pos;
148 if (!(avctx->oformat->flags & AVFMT_NOFILE) &&
149 !(avctx->flags & AVFMT_FLAG_CUSTOM_IO)) {
150 avio_closep(&avctx->pb);
152 avformat_free_context(avctx);
155 void Mux::add_packet(const AVPacket &pkt, int64_t pts, int64_t dts, AVRational timebase, int stream_index_override)
160 if (av_packet_ref(&pkt_copy, &pkt) < 0) {
161 fprintf(stderr, "av_copy_packet() failed\n");
164 if (stream_index_override != -1) {
165 pkt_copy.stream_index = stream_index_override;
167 assert(size_t(pkt_copy.stream_index) < streams.size());
168 AVRational time_base = streams[pkt_copy.stream_index]->time_base;
169 pkt_copy.pts = av_rescale_q(pts, timebase, time_base);
170 pkt_copy.dts = av_rescale_q(dts, timebase, time_base);
171 pkt_copy.duration = av_rescale_q(pkt.duration, timebase, time_base);
174 lock_guard<mutex> lock(mu);
176 // Just drop the packet on the floor.
177 } else if (write_strategy == WriteStrategy::WRITE_BACKGROUND) {
178 packet_queue.push_back(QueuedPacket{ av_packet_clone(&pkt_copy), pts });
180 packet_queue_ready.notify_all();
181 } else if (plug_count > 0) {
182 packet_queue.push_back(QueuedPacket{ av_packet_clone(&pkt_copy), pts });
184 write_packet_or_die(pkt_copy, pts);
188 av_packet_unref(&pkt_copy);
191 void Mux::write_packet_or_die(const AVPacket &pkt, int64_t unscaled_pts)
193 for (MuxMetrics *metric : metrics) {
194 if (pkt.stream_index == 0) {
195 metric->metric_video_bytes += pkt.size;
196 } else if (pkt.stream_index == 1) {
197 metric->metric_audio_bytes += pkt.size;
202 int64_t old_pos = avctx->pb->pos;
203 int err = av_interleaved_write_frame(avctx, const_cast<AVPacket *>(&pkt));
205 char errbuf[AV_ERROR_MAX_STRING_SIZE];
206 av_strerror(err, errbuf, sizeof(errbuf));
207 fprintf(stderr, "av_interleaved_write_frame() failed: %s\n", errbuf);
210 avio_flush(avctx->pb);
211 for (MuxMetrics *metric : metrics) {
212 metric->metric_written_bytes += avctx->pb->pos - old_pos;
215 if (pkt.stream_index == 0 && write_callback != nullptr) {
216 write_callback(unscaled_pts);
222 lock_guard<mutex> lock(mu);
229 lock_guard<mutex> lock(mu);
230 if (--plug_count > 0) {
233 assert(plug_count >= 0);
235 sort(packet_queue.begin(), packet_queue.end(), PacketBefore(avctx));
237 if (write_strategy == WRITE_BACKGROUND) {
238 packet_queue_ready.notify_all();
240 for (QueuedPacket &qp : packet_queue) {
241 write_packet_or_die(*qp.pkt, qp.unscaled_pts);
242 av_packet_free(&qp.pkt);
244 packet_queue.clear();
250 lock_guard<mutex> lock(mu);
252 assert(plug_count == 0);
253 for (QueuedPacket &qp : packet_queue) {
254 av_packet_free(&qp.pkt);
256 packet_queue.clear();
262 lock_guard<mutex> lock(mu);
267 void Mux::thread_func()
269 pthread_setname_np(pthread_self(), "Mux");
273 unique_lock<mutex> lock(mu);
275 packet_queue_ready.wait(lock, [this]() {
276 return writer_thread_should_quit || (!packet_queue.empty() && plug_count == 0);
278 if (writer_thread_should_quit && packet_queue.empty()) {
283 assert(!packet_queue.empty() && plug_count == 0);
284 vector<QueuedPacket> packets;
285 swap(packets, packet_queue);
288 for (QueuedPacket &qp : packets) {
289 write_packet_or_die(*qp.pkt, qp.unscaled_pts);
290 av_packet_free(&qp.pkt);
296 void Mux::write_header()
298 AVDictionary *options = NULL;
299 vector<pair<string, string>> opts = MUX_OPTS;
300 for (pair<string, string> opt : opts) {
301 av_dict_set(&options, opt.first.c_str(), opt.second.c_str(), 0);
304 int err = avformat_write_header(avctx, &options);
306 char errbuf[AV_ERROR_MAX_STRING_SIZE];
307 av_strerror(err, errbuf, sizeof(errbuf));
308 fprintf(stderr, "avformat_write_header() failed: %s\n", errbuf);
311 for (MuxMetrics *metric : metrics) {
312 metric->metric_written_bytes += avctx->pb->pos;
315 // Make sure the header is written before the constructor exits
316 // (assuming we are in WRITE_FOREGROUND mode).
317 avio_flush(avctx->pb);
321 void MuxMetrics::init(const vector<pair<string, string>> &labels)
323 vector<pair<string, string>> labels_video = labels;
324 labels_video.emplace_back("stream", "video");
325 global_metrics.add("mux_stream_bytes", labels_video, &metric_video_bytes);
327 vector<pair<string, string>> labels_audio = labels;
328 labels_audio.emplace_back("stream", "audio");
329 global_metrics.add("mux_stream_bytes", labels_audio, &metric_audio_bytes);
331 global_metrics.add("mux_written_bytes", labels, &metric_written_bytes);