1 #include "video_encoder.h"
8 #include <sys/socket.h>
14 #include <libavutil/mem.h>
17 #include "audio_encoder.h"
19 #include "av1_encoder.h"
22 #include "shared/ffmpeg_raii.h"
24 #include "shared/httpd.h"
25 #include "shared/mux.h"
26 #include "quicksync_encoder.h"
27 #include "shared/timebase.h"
28 #include "x264_encoder.h"
30 class RefCountedFrame;
33 using namespace movit;
37 string generate_local_dump_filename(int frame)
39 time_t now = time(NULL);
41 localtime_r(&now, &now_tm);
44 strftime(timestamp, sizeof(timestamp), "%F-%H%M%S%z", &now_tm);
46 // Use the frame number to disambiguate between two cuts starting
47 // on the same second.
49 snprintf(filename, sizeof(filename), "%s/%s%s-f%02d%s",
50 global_flags.recording_dir.c_str(),
51 LOCAL_DUMP_PREFIX, timestamp, frame % 100, LOCAL_DUMP_SUFFIX);
57 VideoEncoder::VideoEncoder(ResourcePool *resource_pool, QSurface *surface, const std::string &va_display, int width, int height, HTTPD *httpd, DiskSpaceEstimator *disk_space_estimator)
58 : resource_pool(resource_pool), surface(surface), va_display(va_display), width(width), height(height), httpd(httpd), disk_space_estimator(disk_space_estimator)
60 // TODO: If we're outputting AV1, we can't use MPEG-TS currently.
61 srt_oformat = av_guess_format("mpegts", nullptr, nullptr);
62 assert(srt_oformat != nullptr);
64 oformat = av_guess_format(global_flags.stream_mux_name.c_str(), nullptr, nullptr);
65 assert(oformat != nullptr);
66 if (global_flags.stream_audio_codec_name.empty()) {
67 stream_audio_encoder.reset(new AudioEncoder(AUDIO_OUTPUT_CODEC_NAME, DEFAULT_AUDIO_OUTPUT_BIT_RATE, oformat));
69 stream_audio_encoder.reset(new AudioEncoder(global_flags.stream_audio_codec_name, global_flags.stream_audio_codec_bitrate, oformat));
71 if (global_flags.x264_video_to_http || global_flags.x264_video_to_disk) {
72 x264_encoder.reset(new X264Encoder(oformat, /*use_separate_disk_params=*/false));
74 VideoCodecInterface *http_encoder = x264_encoder.get();
75 VideoCodecInterface *disk_encoder = x264_encoder.get();
77 if (global_flags.av1_video_to_http) {
78 av1_encoder.reset(new AV1Encoder(oformat));
79 http_encoder = av1_encoder.get();
82 if (global_flags.x264_separate_disk_encode) {
83 x264_disk_encoder.reset(new X264Encoder(oformat, /*use_separate_disk_params=*/true));
84 disk_encoder = x264_disk_encoder.get();
87 string filename = generate_local_dump_filename(/*frame=*/0);
88 quicksync_encoder.reset(new QuickSyncEncoder(filename, resource_pool, surface, va_display, width, height, oformat, http_encoder, disk_encoder, disk_space_estimator));
90 open_output_streams();
91 stream_audio_encoder->add_mux(http_mux.get());
92 if (srt_mux != nullptr) {
93 stream_audio_encoder->add_mux(srt_mux.get());
95 quicksync_encoder->set_http_mux(http_mux.get());
96 if (srt_mux != nullptr) {
97 quicksync_encoder->set_srt_mux(srt_mux.get());
99 if (global_flags.x264_video_to_http) {
100 x264_encoder->add_mux(http_mux.get());
101 if (srt_mux != nullptr) {
102 x264_encoder->add_mux(srt_mux.get());
106 if (global_flags.av1_video_to_http) {
107 av1_encoder->add_mux(http_mux.get());
108 if (srt_mux != nullptr) {
109 av1_encoder->add_mux(srt_mux.get());
115 VideoEncoder::~VideoEncoder()
118 quicksync_encoder->shutdown();
119 x264_encoder.reset(nullptr);
120 x264_disk_encoder.reset(nullptr);
121 quicksync_encoder->close_file();
122 quicksync_encoder.reset(nullptr);
123 while (quicksync_encoders_in_shutdown.load() > 0) {
128 void VideoEncoder::do_cut(int frame)
130 string filename = generate_local_dump_filename(frame);
131 printf("Starting new recording: %s\n", filename.c_str());
133 // Do the shutdown of the old encoder in a separate thread, since it can
134 // take some time (it needs to wait for all the frames in the queue to be
135 // done encoding, for one) and we are running on the main mixer thread.
136 // However, since this means both encoders could be sending packets at
137 // the same time, it means pts could come out of order to the stream mux,
138 // and we need to plug it until the shutdown is complete.
140 lock(qs_mu, qs_audio_mu);
141 lock_guard<mutex> lock1(qs_mu, adopt_lock), lock2(qs_audio_mu, adopt_lock);
142 QuickSyncEncoder *old_encoder = quicksync_encoder.release(); // When we go C++14, we can use move capture instead.
143 X264Encoder *old_x264_encoder = nullptr;
144 X264Encoder *old_x264_disk_encoder = nullptr;
145 if (global_flags.x264_video_to_disk) {
146 old_x264_encoder = x264_encoder.release();
148 if (global_flags.x264_separate_disk_encode) {
149 old_x264_disk_encoder = x264_disk_encoder.release();
151 thread([old_encoder, old_x264_encoder, old_x264_disk_encoder, this]{
152 old_encoder->shutdown();
153 delete old_x264_encoder;
154 delete old_x264_disk_encoder;
155 old_encoder->close_file();
158 // We cannot delete the encoder here, as this thread has no OpenGL context.
159 // We'll deal with it in begin_frame().
160 lock_guard<mutex> lock(qs_mu);
161 qs_needing_cleanup.emplace_back(old_encoder);
164 if (global_flags.x264_video_to_disk) {
165 x264_encoder.reset(new X264Encoder(oformat, /*use_separate_disk_params=*/false));
166 assert(global_flags.x264_video_to_http);
167 if (global_flags.x264_video_to_http) {
168 x264_encoder->add_mux(http_mux.get());
170 if (overriding_bitrate != 0) {
171 x264_encoder->change_bitrate(overriding_bitrate);
174 X264Encoder *http_encoder = x264_encoder.get();
175 X264Encoder *disk_encoder = x264_encoder.get();
176 if (global_flags.x264_separate_disk_encode) {
177 x264_disk_encoder.reset(new X264Encoder(oformat, /*use_separate_disk_params=*/true));
178 disk_encoder = x264_disk_encoder.get();
181 quicksync_encoder.reset(new QuickSyncEncoder(filename, resource_pool, surface, va_display, width, height, oformat, http_encoder, disk_encoder, disk_space_estimator));
182 quicksync_encoder->set_http_mux(http_mux.get());
185 void VideoEncoder::change_x264_bitrate(unsigned rate_kbit)
187 overriding_bitrate = rate_kbit;
188 x264_encoder->change_bitrate(rate_kbit);
191 void VideoEncoder::add_audio(int64_t pts, std::vector<float> audio)
193 // Take only qs_audio_mu, since add_audio() is thread safe
194 // (we can only conflict with do_cut(), which takes qs_audio_mu)
195 // and we don't want to contend with begin_frame().
197 lock_guard<mutex> lock(qs_audio_mu);
198 quicksync_encoder->add_audio(pts, audio);
200 stream_audio_encoder->encode_audio(audio, pts + quicksync_encoder->global_delay());
203 bool VideoEncoder::is_zerocopy() const
205 // Explicitly do _not_ take qs_mu; this is called from the mixer,
206 // and qs_mu might be contended. is_zerocopy() is thread safe
207 // and never called in parallel with do_cut() (both happen only
208 // from the mixer thread).
209 return quicksync_encoder->is_zerocopy();
212 bool VideoEncoder::begin_frame(int64_t pts, int64_t duration, movit::YCbCrLumaCoefficients ycbcr_coefficients, const std::vector<RefCountedFrame> &input_frames, GLuint *y_tex, GLuint *cbcr_tex)
214 lock_guard<mutex> lock(qs_mu);
215 qs_needing_cleanup.clear(); // Since we have an OpenGL context here, and are called regularly.
216 return quicksync_encoder->begin_frame(pts, duration, ycbcr_coefficients, input_frames, y_tex, cbcr_tex);
219 RefCountedGLsync VideoEncoder::end_frame()
221 want_srt_metric_update = true;
222 lock_guard<mutex> lock(qs_mu);
223 return quicksync_encoder->end_frame();
226 void VideoEncoder::open_output_streams()
228 for (bool is_srt : {false, true}) {
229 if (is_srt && global_flags.srt_destination_host.empty()) {
233 AVFormatContext *avctx = avformat_alloc_context();
234 avctx->oformat = is_srt ? srt_oformat : oformat;
236 uint8_t *buf = (uint8_t *)av_malloc(MUX_BUFFER_SIZE);
237 avctx->pb = avio_alloc_context(buf, MUX_BUFFER_SIZE, 1, this, nullptr, nullptr, nullptr);
239 avctx->pb->write_packet = &VideoEncoder::write_srt_packet_thunk;
241 avctx->pb->write_data_type = &VideoEncoder::write_packet2_thunk;
242 avctx->pb->ignore_boundary_point = 1;
245 Mux::Codec video_codec;
246 if (global_flags.av1_video_to_http) {
247 video_codec = Mux::CODEC_AV1;
249 video_codec = Mux::CODEC_H264;
252 avctx->flags = AVFMT_FLAG_CUSTOM_IO;
254 string video_extradata;
255 if (global_flags.x264_video_to_http) {
256 video_extradata = x264_encoder->get_global_headers();
258 } else if (global_flags.av1_video_to_http) {
259 video_extradata = av1_encoder->get_global_headers();
263 Mux *mux = new Mux(avctx, width, height, video_codec, video_extradata, stream_audio_encoder->get_codec_parameters().get(),
264 get_color_space(global_flags.ycbcr_rec709_coefficients), COARSE_TIMEBASE,
265 /*write_callback=*/nullptr, is_srt ? Mux::WRITE_BACKGROUND : Mux::WRITE_FOREGROUND, { is_srt ? &srt_mux_metrics : &http_mux_metrics });
268 srt_mux_metrics.init({{ "destination", "srt" }});
269 srt_metrics.init({{ "cardtype", "output" }});
270 global_metrics.add("srt_num_connection_attempts", {{ "cardtype", "output" }}, &metric_srt_num_connection_attempts);
273 http_mux_metrics.init({{ "destination", "http" }});
278 int VideoEncoder::write_packet2_thunk(void *opaque, uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time)
280 VideoEncoder *video_encoder = (VideoEncoder *)opaque;
281 return video_encoder->write_packet2(buf, buf_size, type, time);
284 int VideoEncoder::write_packet2(uint8_t *buf, int buf_size, AVIODataMarkerType type, int64_t time)
286 if (type == AVIO_DATA_MARKER_SYNC_POINT || type == AVIO_DATA_MARKER_BOUNDARY_POINT) {
287 seen_sync_markers = true;
288 } else if (type == AVIO_DATA_MARKER_UNKNOWN && !seen_sync_markers) {
289 // We don't know if this is a keyframe or not (the muxer could
290 // avoid marking it), so we just have to make the best of it.
291 type = AVIO_DATA_MARKER_SYNC_POINT;
294 if (type == AVIO_DATA_MARKER_HEADER) {
295 http_mux_header.append((char *)buf, buf_size);
296 httpd->set_header(HTTPD::StreamID{ HTTPD::MAIN_STREAM, 0 }, http_mux_header);
298 httpd->add_data(HTTPD::StreamID{ HTTPD::MAIN_STREAM, 0 }, (char *)buf, buf_size, type == AVIO_DATA_MARKER_SYNC_POINT, time, AVRational{ AV_TIME_BASE, 1 });
303 int VideoEncoder::write_srt_packet_thunk(void *opaque, uint8_t *buf, int buf_size)
305 VideoEncoder *video_encoder = (VideoEncoder *)opaque;
306 return video_encoder->write_srt_packet(buf, buf_size);
309 static string print_addrinfo(const addrinfo *ai)
311 char hoststr[NI_MAXHOST], portstr[NI_MAXSERV];
312 if (getnameinfo(ai->ai_addr, ai->ai_addrlen, hoststr, sizeof(hoststr), portstr, sizeof(portstr), NI_DGRAM | NI_NUMERICHOST | NI_NUMERICSERV) != 0) {
313 return "<unknown address>"; // Should basically never happen, since we're not doing DNS lookups.
316 if (ai->ai_family == AF_INET6) {
317 return string("[") + hoststr + "]:" + portstr;
319 return string(hoststr) + ":" + portstr;
323 int VideoEncoder::open_srt_socket()
325 int sock = srt_create_socket();
327 fprintf(stderr, "srt_create_socket(): %s\n", srt_getlasterror_str());
331 SRT_TRANSTYPE live = SRTT_LIVE;
332 if (srt_setsockopt(sock, 0, SRTO_TRANSTYPE, &live, sizeof(live)) < 0) {
333 fprintf(stderr, "srt_setsockopt(SRTO_TRANSTYPE): %s\n", srt_getlasterror_str());
338 if (srt_setsockopt(sock, 0, SRTO_LATENCY, &global_flags.srt_output_latency, sizeof(global_flags.srt_output_latency)) < 0) {
339 fprintf(stderr, "srt_setsockopt(SRTO_LATENCY): %s\n", srt_getlasterror_str());
344 if (!global_flags.srt_streamid.empty()) {
345 if (srt_setsockopt(sock, 0, SRTO_STREAMID, global_flags.srt_streamid.data(), global_flags.srt_streamid.size()) < 0) {
346 fprintf(stderr, "srt_setsockopt(SRTO_STREAMID): %s\n", srt_getlasterror_str());
352 if (!global_flags.srt_passphrase.empty()) {
353 if (srt_setsockopt(sock, 0, SRTO_PASSPHRASE, global_flags.srt_passphrase.data(), global_flags.srt_passphrase.size()) < 0) {
354 fprintf(stderr, "srt_setsockopt(SRTO_PASSPHRASE): %s\n", srt_getlasterror_str());
363 int VideoEncoder::connect_to_srt()
365 // We need to specify SOCK_DGRAM as a hint, or we'll get all addresses
366 // three times (for each of TCP, UDP, raw).
368 memset(&hints, 0, sizeof(hints));
369 hints.ai_flags = AI_ADDRCONFIG;
370 hints.ai_socktype = SOCK_DGRAM;
373 int ret = getaddrinfo(global_flags.srt_destination_host.c_str(), global_flags.srt_destination_port.c_str(), &hints, &ai);
375 fprintf(stderr, "getaddrinfo(%s:%s): %s\n", global_flags.srt_destination_host.c_str(), global_flags.srt_destination_port.c_str(), gai_strerror(ret));
379 for (const addrinfo *cur = ai; cur != nullptr; cur = cur->ai_next) {
380 // Seemingly, srt_create_socket() isn't universal; once we try to connect,
381 // it gets locked to either IPv4 or IPv6. So we need to create a new one
382 // for every address we try.
383 int sock = open_srt_socket();
388 ++metric_srt_num_connection_attempts;
389 if (srt_connect(sock, cur->ai_addr, cur->ai_addrlen) < 0) {
390 fprintf(stderr, "srt_connect(%s): %s\n", print_addrinfo(cur).c_str(), srt_getlasterror_str());
394 fprintf(stderr, "Connected to destination SRT endpoint at %s.\n", print_addrinfo(cur).c_str());
399 // Out of candidates, so give up.
404 int VideoEncoder::write_srt_packet(uint8_t *buf, int buf_size)
406 if (want_srt_metric_update.exchange(false) && srt_sock != -1) {
407 srt_metrics.update_srt_stats(srt_sock);
409 while (buf_size > 0 && !should_quit.load()) {
410 if (srt_sock == -1) {
411 srt_sock = connect_to_srt();
412 if (srt_sock == -1) {
416 srt_metrics.update_srt_stats(srt_sock);
418 int to_send = min(buf_size, SRT_LIVE_DEF_PLSIZE);
419 int ret = srt_send(srt_sock, (char *)buf, to_send);
421 fprintf(stderr, "srt_send(): %s\n", srt_getlasterror_str());
423 srt_metrics.metric_srt_uptime_seconds = 0.0 / 0.0;
424 srt_sock = connect_to_srt();