git.sesse.net Git - nageru/blob - shared/mux.cpp

   1 #include "shared/mux.h"
   2
   3 #include <algorithm>
   4 #include <assert.h>
   5 #include <mutex>
   6 #include <stdint.h>
   7 #include <stdio.h>
   8 #include <stdlib.h>
   9 #include <string.h>
  10 #include <string>
  11 #include <utility>
  12 #include <vector>
  13
  14 extern "C" {
  15 #include <libavformat/avio.h>
  16 #include <libavutil/avutil.h>
  17 #include <libavutil/dict.h>
  18 #include <libavutil/mathematics.h>
  19 #include <libavutil/mem.h>
  20 #include <libavutil/pixfmt.h>
  21 #include <libavutil/rational.h>
  22 }
  23
  24 #include "shared/metrics.h"
  25 #include "shared/shared_defs.h"
  26 #include "shared/timebase.h"
  27
  28 using namespace std;
  29
  30 struct PacketBefore {
  31         PacketBefore(const AVFormatContext *ctx) : ctx(ctx) {}
  32
  33         bool operator() (const Mux::QueuedPacket &a_qp, const Mux::QueuedPacket &b_qp) const {
  34                 const AVPacket *a = a_qp.pkt;
  35                 const AVPacket *b = b_qp.pkt;
  36                 int64_t a_dts = (a->dts == AV_NOPTS_VALUE ? a->pts : a->dts);
  37                 int64_t b_dts = (b->dts == AV_NOPTS_VALUE ? b->pts : b->dts);
  38                 AVRational a_timebase = ctx->streams[a->stream_index]->time_base;
  39                 AVRational b_timebase = ctx->streams[b->stream_index]->time_base;
  40                 if (av_compare_ts(a_dts, a_timebase, b_dts, b_timebase) != 0) {
  41                         return av_compare_ts(a_dts, a_timebase, b_dts, b_timebase) < 0;
  42                 } else {
  43                         return av_compare_ts(a->pts, a_timebase, b->pts, b_timebase) < 0;
  44                 }
  45         }
  46
  47         const AVFormatContext * const ctx;
  48 };
  49
  50 Mux::Mux(AVFormatContext *avctx, int width, int height, Codec video_codec, const string &video_extradata, const AVCodecParameters *audio_codecpar, AVColorSpace color_space, int time_base, function<void(int64_t)> write_callback, WriteStrategy write_strategy, const vector<MuxMetrics *> &metrics, WithSubtitles with_subtitles)
  51         : write_strategy(write_strategy), avctx(avctx), write_callback(write_callback), metrics(metrics)
  52 {
  53         // MPEG-TS ostensibly needs some conversions (e.g. for differing start codes),
  54         // so let FFmpeg insert them as needed in case we are muxing to that.
  55         // Curiously enough, things actually seem to go quite fine without
  56         // (and it also seems FFmpeg's MPEG-TS muxer automatically does stuff like
  57         // repeat PPS/SPS before keyframes for us), but it can't hurt.
  58         avctx->flags |= AVFMT_FLAG_AUTO_BSF;
  59
  60         AVStream *avstream_video = avformat_new_stream(avctx, nullptr);
  61         if (avstream_video == nullptr) {
  62                 fprintf(stderr, "avformat_new_stream() failed\n");
  63                 abort();
  64         }
  65         avstream_video->time_base = AVRational{1, time_base};
  66         avstream_video->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
  67         if (video_codec == CODEC_H264) {
  68                 avstream_video->codecpar->codec_id = AV_CODEC_ID_H264;
  69         } else if (video_codec == CODEC_AV1) {
  70                 avstream_video->codecpar->codec_id = AV_CODEC_ID_AV1;
  71         } else {
  72                 assert(video_codec == CODEC_MJPEG);
  73                 avstream_video->codecpar->codec_id = AV_CODEC_ID_MJPEG;
  74         }
  75         avstream_video->codecpar->width = width;
  76         avstream_video->codecpar->height = height;
  77
  78         // Colorspace details. Closely correspond to settings in EffectChain_finalize,
  79         // as noted in each comment.
  80         // Note that the H.264 stream also contains this information and depending on the
  81         // mux, this might simply get ignored. See sps_rbsp().
  82         // Note that there's no way to change this per-frame as the H.264 stream
  83         // would like to be able to.
  84         avstream_video->codecpar->color_primaries = AVCOL_PRI_BT709;  // RGB colorspace (inout_format.color_space).
  85         avstream_video->codecpar->color_trc = AVCOL_TRC_IEC61966_2_1;  // Gamma curve (inout_format.gamma_curve).
  86         // YUV colorspace (output_ycbcr_format.luma_coefficients).
  87         avstream_video->codecpar->color_space = color_space;
  88         avstream_video->codecpar->color_range = AVCOL_RANGE_MPEG;  // Full vs. limited range (output_ycbcr_format.full_range).
  89         avstream_video->codecpar->chroma_location = AVCHROMA_LOC_LEFT;  // Chroma sample location. See chroma_offset_0[] in Mixer::subsample_chroma().
  90         avstream_video->codecpar->field_order = AV_FIELD_PROGRESSIVE;
  91
  92         if (!video_extradata.empty()) {
  93                 avstream_video->codecpar->extradata = (uint8_t *)av_malloc(video_extradata.size() + AV_INPUT_BUFFER_PADDING_SIZE);
  94                 avstream_video->codecpar->extradata_size = video_extradata.size();
  95                 memcpy(avstream_video->codecpar->extradata, video_extradata.data(), video_extradata.size());
  96         }
  97         streams.push_back(avstream_video);
  98
  99         if (audio_codecpar != nullptr) {
 100                 AVStream *avstream_audio = avformat_new_stream(avctx, nullptr);
 101                 if (avstream_audio == nullptr) {
 102                         fprintf(stderr, "avformat_new_stream() failed\n");
 103                         abort();
 104                 }
 105                 avstream_audio->time_base = AVRational{1, time_base};
 106                 if (avcodec_parameters_copy(avstream_audio->codecpar, audio_codecpar) < 0) {
 107                         fprintf(stderr, "avcodec_parameters_copy() failed\n");
 108                         abort();
 109                 }
 110                 streams.push_back(avstream_audio);
 111         }
 112
 113         if (with_subtitles == WITH_SUBTITLES) {
 114                 AVStream *avstream_subtitles = avformat_new_stream(avctx, nullptr);
 115                 if (avstream_subtitles == nullptr) {
 116                         fprintf(stderr, "avformat_new_stream() failed\n");
 117                         abort();
 118                 }
 119                 avstream_subtitles->time_base = AVRational{1, time_base};
 120                 avstream_subtitles->codecpar->codec_type = AVMEDIA_TYPE_SUBTITLE;
 121                 avstream_subtitles->codecpar->codec_id = AV_CODEC_ID_WEBVTT;
 122                 avstream_subtitles->disposition = AV_DISPOSITION_METADATA;
 123                 streams.push_back(avstream_subtitles);
 124                 subtitle_stream_idx = streams.size() - 1;
 125         }
 126
 127         if (write_strategy == WRITE_BACKGROUND) {
 128                 writer_thread = thread(&Mux::thread_func, this);
 129         } else {
 130                 write_header();
 131         }
 132 }
 133
 134 Mux::~Mux()
 135 {
 136         assert(plug_count == 0);
 137         if (write_strategy == WRITE_BACKGROUND) {
 138                 writer_thread_should_quit = true;
 139                 packet_queue_ready.notify_all();
 140                 writer_thread.join();
 141         }
 142         int64_t old_pos = avctx->pb->pos;
 143         av_write_trailer(avctx);
 144         for (MuxMetrics *metric : metrics) {
 145                 metric->metric_written_bytes += avctx->pb->pos - old_pos;
 146         }
 147
 148         if (!(avctx->oformat->flags & AVFMT_NOFILE) &&
 149             !(avctx->flags & AVFMT_FLAG_CUSTOM_IO)) {
 150                 avio_closep(&avctx->pb);
 151         }
 152         avformat_free_context(avctx);
 153 }
 154
 155 void Mux::add_packet(const AVPacket &pkt, int64_t pts, int64_t dts, AVRational timebase, int stream_index_override)
 156 {
 157         assert(pts >= dts);
 158
 159         AVPacket pkt_copy;
 160         if (av_packet_ref(&pkt_copy, &pkt) < 0) {
 161                 fprintf(stderr, "av_copy_packet() failed\n");
 162                 abort();
 163         }
 164         if (stream_index_override != -1) {
 165                 pkt_copy.stream_index = stream_index_override;
 166         }
 167         assert(size_t(pkt_copy.stream_index) < streams.size());
 168         AVRational time_base = streams[pkt_copy.stream_index]->time_base;
 169         pkt_copy.pts = av_rescale_q(pts, timebase, time_base);
 170         pkt_copy.dts = av_rescale_q(dts, timebase, time_base);
 171         pkt_copy.duration = av_rescale_q(pkt.duration, timebase, time_base);
 172
 173         {
 174                 lock_guard<mutex> lock(mu);
 175                 if (drained) {
 176                         // Just drop the packet on the floor.
 177                 } else if (write_strategy == WriteStrategy::WRITE_BACKGROUND) {
 178                         packet_queue.push_back(QueuedPacket{ av_packet_clone(&pkt_copy), pts });
 179                         if (plug_count == 0)
 180                                 packet_queue_ready.notify_all();
 181                 } else if (plug_count > 0) {
 182                         packet_queue.push_back(QueuedPacket{ av_packet_clone(&pkt_copy), pts });
 183                 } else {
 184                         write_packet_or_die(pkt_copy, pts);
 185                 }
 186         }
 187
 188         av_packet_unref(&pkt_copy);
 189 }
 190
 191 void Mux::write_packet_or_die(const AVPacket &pkt, int64_t unscaled_pts)
 192 {
 193         for (MuxMetrics *metric : metrics) {
 194                 if (pkt.stream_index == 0) {
 195                         metric->metric_video_bytes += pkt.size;
 196                 } else if (pkt.stream_index == 1) {
 197                         metric->metric_audio_bytes += pkt.size;
 198                 } else {
 199                         assert(false);
 200                 }
 201         }
 202         int64_t old_pos = avctx->pb->pos;
 203         int err = av_interleaved_write_frame(avctx, const_cast<AVPacket *>(&pkt));
 204         if (err < 0) {
 205                 char errbuf[AV_ERROR_MAX_STRING_SIZE];
 206                 av_strerror(err, errbuf, sizeof(errbuf));
 207                 fprintf(stderr, "av_interleaved_write_frame() failed: %s\n", errbuf);
 208                 exit(EXIT_FAILURE);
 209         }
 210         avio_flush(avctx->pb);
 211         for (MuxMetrics *metric : metrics) {
 212                 metric->metric_written_bytes += avctx->pb->pos - old_pos;
 213         }
 214
 215         if (pkt.stream_index == 0 && write_callback != nullptr) {
 216                 write_callback(unscaled_pts);
 217         }
 218 }
 219
 220 void Mux::plug()
 221 {
 222         lock_guard<mutex> lock(mu);
 223         assert(!drained);
 224         ++plug_count;
 225 }
 226
 227 void Mux::unplug()
 228 {
 229         lock_guard<mutex> lock(mu);
 230         if (--plug_count > 0) {
 231                 return;
 232         }
 233         assert(plug_count >= 0);
 234
 235         sort(packet_queue.begin(), packet_queue.end(), PacketBefore(avctx));
 236
 237         if (write_strategy == WRITE_BACKGROUND) {
 238                 packet_queue_ready.notify_all();
 239         } else {
 240                 for (QueuedPacket &qp : packet_queue) {
 241                         write_packet_or_die(*qp.pkt, qp.unscaled_pts);
 242                         av_packet_free(&qp.pkt);
 243                 }
 244                 packet_queue.clear();
 245         }
 246 }
 247
 248 void Mux::drain()
 249 {
 250         lock_guard<mutex> lock(mu);
 251         assert(!drained);
 252         assert(plug_count == 0);
 253         for (QueuedPacket &qp : packet_queue) {
 254                 av_packet_free(&qp.pkt);
 255         }
 256         packet_queue.clear();
 257         drained = true;
 258 }
 259
 260 void Mux::undrain()
 261 {
 262         lock_guard<mutex> lock(mu);
 263         assert(drained);
 264         drained = false;
 265 }
 266
 267 void Mux::thread_func()
 268 {
 269         pthread_setname_np(pthread_self(), "Mux");
 270
 271         write_header();
 272
 273         unique_lock<mutex> lock(mu);
 274         for ( ;; ) {
 275                 packet_queue_ready.wait(lock, [this]() {
 276                         return writer_thread_should_quit || (!packet_queue.empty() && plug_count == 0);
 277                 });
 278                 if (writer_thread_should_quit && packet_queue.empty()) {
 279                         // All done.
 280                         break;
 281                 }
 282
 283                 assert(!packet_queue.empty() && plug_count == 0);
 284                 vector<QueuedPacket> packets;
 285                 swap(packets, packet_queue);
 286
 287                 lock.unlock();
 288                 for (QueuedPacket &qp : packets) {
 289                         write_packet_or_die(*qp.pkt, qp.unscaled_pts);
 290                         av_packet_free(&qp.pkt);
 291                 }
 292                 lock.lock();
 293         }
 294 }
 295
 296 void Mux::write_header()
 297 {
 298         AVDictionary *options = NULL;
 299         vector<pair<string, string>> opts = MUX_OPTS;
 300         for (pair<string, string> opt : opts) {
 301                 av_dict_set(&options, opt.first.c_str(), opt.second.c_str(), 0);
 302         }
 303
 304         int err = avformat_write_header(avctx, &options);
 305         if (err < 0) {
 306                 char errbuf[AV_ERROR_MAX_STRING_SIZE];
 307                 av_strerror(err, errbuf, sizeof(errbuf));
 308                 fprintf(stderr, "avformat_write_header() failed: %s\n", errbuf);
 309                 exit(EXIT_FAILURE);
 310         }
 311         for (MuxMetrics *metric : metrics) {
 312                 metric->metric_written_bytes += avctx->pb->pos;
 313         }
 314
 315         // Make sure the header is written before the constructor exits
 316         // (assuming we are in WRITE_FOREGROUND mode).
 317         avio_flush(avctx->pb);
 318
 319 }
 320
 321 void MuxMetrics::init(const vector<pair<string, string>> &labels)
 322 {
 323         vector<pair<string, string>> labels_video = labels;
 324         labels_video.emplace_back("stream", "video");
 325         global_metrics.add("mux_stream_bytes", labels_video, &metric_video_bytes);
 326
 327         vector<pair<string, string>> labels_audio = labels;
 328         labels_audio.emplace_back("stream", "audio");
 329         global_metrics.add("mux_stream_bytes", labels_audio, &metric_audio_bytes);
 330
 331         global_metrics.add("mux_written_bytes", labels, &metric_written_bytes);
 332 }