1 #include <movit/effect_util.h>
2 #include <movit/util.h>
3 #include <movit/resource_pool.h> // Must be above the Xlib includes.
11 #include "chroma_subsampler.h"
12 #include "decklink_output.h"
13 #include "decklink_util.h"
15 #include "shared/metrics.h"
16 #include "print_latency.h"
17 #include "shared/timebase.h"
18 #include "v210_converter.h"
20 using namespace movit;
22 using namespace std::chrono;
26 // This class can be deleted during regular use, so make all the metrics static.
27 once_flag decklink_metrics_inited;
28 LatencyHistogram latency_histogram;
29 atomic<int64_t> metric_decklink_output_width_pixels{-1};
30 atomic<int64_t> metric_decklink_output_height_pixels{-1};
31 atomic<int64_t> metric_decklink_output_frame_rate_den{-1};
32 atomic<int64_t> metric_decklink_output_frame_rate_nom{-1};
33 atomic<int64_t> metric_decklink_output_inflight_frames{0};
34 atomic<int64_t> metric_decklink_output_color_mismatch_frames{0};
36 atomic<int64_t> metric_decklink_output_scheduled_frames_dropped{0};
37 atomic<int64_t> metric_decklink_output_scheduled_frames_late{0};
38 atomic<int64_t> metric_decklink_output_scheduled_frames_normal{0};
39 atomic<int64_t> metric_decklink_output_scheduled_frames_preroll{0};
41 atomic<int64_t> metric_decklink_output_completed_frames_completed{0};
42 atomic<int64_t> metric_decklink_output_completed_frames_dropped{0};
43 atomic<int64_t> metric_decklink_output_completed_frames_flushed{0};
44 atomic<int64_t> metric_decklink_output_completed_frames_late{0};
45 atomic<int64_t> metric_decklink_output_completed_frames_unknown{0};
47 atomic<int64_t> metric_decklink_output_scheduled_samples{0};
49 Summary metric_decklink_output_margin_seconds;
53 DeckLinkOutput::DeckLinkOutput(ResourcePool *resource_pool, QSurface *surface, unsigned width, unsigned height, unsigned card_index)
54 : resource_pool(resource_pool), surface(surface), width(width), height(height), card_index(card_index)
56 chroma_subsampler.reset(new ChromaSubsampler(resource_pool));
58 call_once(decklink_metrics_inited, [](){
59 latency_histogram.init("decklink_output");
60 global_metrics.add("decklink_output_width_pixels", &metric_decklink_output_width_pixels, Metrics::TYPE_GAUGE);
61 global_metrics.add("decklink_output_height_pixels", &metric_decklink_output_height_pixels, Metrics::TYPE_GAUGE);
62 global_metrics.add("decklink_output_frame_rate_den", &metric_decklink_output_frame_rate_den, Metrics::TYPE_GAUGE);
63 global_metrics.add("decklink_output_frame_rate_nom", &metric_decklink_output_frame_rate_nom, Metrics::TYPE_GAUGE);
64 global_metrics.add("decklink_output_inflight_frames", &metric_decklink_output_inflight_frames, Metrics::TYPE_GAUGE);
65 global_metrics.add("decklink_output_color_mismatch_frames", &metric_decklink_output_color_mismatch_frames);
67 global_metrics.add("decklink_output_scheduled_frames", {{ "status", "dropped" }}, &metric_decklink_output_scheduled_frames_dropped);
68 global_metrics.add("decklink_output_scheduled_frames", {{ "status", "late" }}, &metric_decklink_output_scheduled_frames_late);
69 global_metrics.add("decklink_output_scheduled_frames", {{ "status", "normal" }}, &metric_decklink_output_scheduled_frames_normal);
70 global_metrics.add("decklink_output_scheduled_frames", {{ "status", "preroll" }}, &metric_decklink_output_scheduled_frames_preroll);
72 global_metrics.add("decklink_output_completed_frames", {{ "status", "completed" }}, &metric_decklink_output_completed_frames_completed);
73 global_metrics.add("decklink_output_completed_frames", {{ "status", "dropped" }}, &metric_decklink_output_completed_frames_dropped);
74 global_metrics.add("decklink_output_completed_frames", {{ "status", "flushed" }}, &metric_decklink_output_completed_frames_flushed);
75 global_metrics.add("decklink_output_completed_frames", {{ "status", "late" }}, &metric_decklink_output_completed_frames_late);
76 global_metrics.add("decklink_output_completed_frames", {{ "status", "unknown" }}, &metric_decklink_output_completed_frames_unknown);
78 global_metrics.add("decklink_output_scheduled_samples", &metric_decklink_output_scheduled_samples);
79 vector<double> quantiles{0.01, 0.1, 0.25, 0.5, 0.75, 0.9, 0.99};
80 metric_decklink_output_margin_seconds.init(quantiles, 60.0);
81 global_metrics.add("decklink_output_margin_seconds", &metric_decklink_output_margin_seconds);
85 bool DeckLinkOutput::set_device(IDeckLink *decklink)
87 if (decklink->QueryInterface(IID_IDeckLinkInput, (void**)&input) != S_OK) {
90 if (decklink->QueryInterface(IID_IDeckLinkOutput, (void**)&output) != S_OK) {
91 fprintf(stderr, "Warning: Card %u has no outputs\n", card_index);
95 IDeckLinkDisplayModeIterator *mode_it;
96 if (output->GetDisplayModeIterator(&mode_it) != S_OK) {
97 fprintf(stderr, "Warning: Failed to enumerate output display modes for card %u\n", card_index);
103 for (const auto &it : summarize_video_modes(mode_it, card_index)) {
104 if (it.second.width != width || it.second.height != height) {
108 // We could support interlaced modes, but let's stay out of it for now,
109 // since we don't have interlaced stream output.
110 if (it.second.interlaced) {
114 video_modes.insert(it);
119 // HDMI or SDI generally mean “both HDMI and SDI at the same time” on DeckLink cards
120 // that support both; pick_default_video_connection() will generally pick one of those
121 // if they exist. (--prefer-hdmi-input would also affect the selection despite the name
122 // of the option, but since either generally means both, it's inconsequential.)
123 // We're not very likely to need analog outputs, so we don't need a way to change
125 video_connection = pick_default_video_connection(decklink, BMDDeckLinkVideoOutputConnections, card_index);
129 void DeckLinkOutput::start_output(uint32_t mode, int64_t base_pts, bool is_master_card)
132 assert(!playback_initiated);
134 if (video_modes.empty()) {
135 fprintf(stderr, "ERROR: No matching output modes for %dx%d found\n", width, height);
139 should_quit.unquit();
140 playback_initiated = true;
141 playback_started = false;
142 if (is_master_card) {
143 this->base_pts = base_pts;
145 this->next_output_pts = 0;
148 IDeckLinkConfiguration *config = nullptr;
149 if (output->QueryInterface(IID_IDeckLinkConfiguration, (void**)&config) != S_OK) {
150 fprintf(stderr, "Failed to get configuration interface for output card\n");
153 if (config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true) != S_OK) {
154 fprintf(stderr, "Failed to set low latency output\n");
157 if (config->SetInt(bmdDeckLinkConfigVideoOutputConnection, video_connection) != S_OK) {
158 fprintf(stderr, "Failed to set video output connection for card %u\n", card_index);
161 if (config->SetFlag(bmdDeckLinkConfigOutput1080pAsPsF, true) != S_OK) {
162 fprintf(stderr, "Failed to set PsF flag for card\n");
165 if (config->SetFlag(bmdDeckLinkConfigSMPTELevelAOutput, true) != S_OK) {
166 // This affects at least some no-name SDI->HDMI converters.
167 // Warn, but don't die.
168 fprintf(stderr, "WARNING: Failed to enable SMTPE Level A; resolutions like 1080p60 might have issues.\n");
171 BMDDisplayModeSupport support;
172 IDeckLinkDisplayMode *display_mode;
173 BMDPixelFormat pixel_format = global_flags.ten_bit_output ? bmdFormat10BitYUV : bmdFormat8BitYUV;
174 if (output->DoesSupportVideoMode(mode, pixel_format, bmdVideoOutputFlagDefault,
175 &support, &display_mode) != S_OK) {
176 fprintf(stderr, "Couldn't ask for format support\n");
180 if (support == bmdDisplayModeNotSupported) {
181 fprintf(stderr, "Requested display mode not supported\n");
185 current_mode_flags = display_mode->GetFlags();
187 BMDTimeValue time_value;
188 BMDTimeScale time_scale;
189 if (display_mode->GetFrameRate(&time_value, &time_scale) != S_OK) {
190 fprintf(stderr, "Couldn't get frame rate\n");
194 metric_decklink_output_width_pixels = width;
195 metric_decklink_output_height_pixels = height;
196 metric_decklink_output_frame_rate_nom = time_value;
197 metric_decklink_output_frame_rate_den = time_scale;
199 frame_duration = time_value * TIMEBASE / time_scale;
201 display_mode->Release();
203 if (input != nullptr) {
204 if (input->DisableVideoInput() != S_OK) {
205 fprintf(stderr, "Warning: Failed to disable video input for card %d\n", card_index);
207 if (input->DisableAudioInput() != S_OK) {
208 fprintf(stderr, "Warning: Failed to disable audio input for card %d\n", card_index);
212 HRESULT result = output->EnableVideoOutput(mode, bmdVideoOutputFlagDefault);
213 if (result != S_OK) {
214 fprintf(stderr, "Couldn't enable output with error 0x%x\n", result);
217 if (output->SetScheduledFrameCompletionCallback(this) != S_OK) {
218 fprintf(stderr, "Couldn't set callback\n");
221 assert(OUTPUT_FREQUENCY == 48000);
222 if (output->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2, bmdAudioOutputStreamTimestamped) != S_OK) {
223 fprintf(stderr, "Couldn't enable audio output\n");
226 if (is_master_card) {
227 if (output->BeginAudioPreroll() != S_OK) {
228 fprintf(stderr, "Couldn't begin audio preroll\n");
232 if (output->StartScheduledPlayback(/*base_pts=*/0, TIMEBASE, 1.0) != S_OK) {
233 fprintf(stderr, "Could not start playback\n");
236 playback_started = true;
239 present_thread = thread([this]{
240 QOpenGLContext *context = create_context(this->surface);
241 eglBindAPI(EGL_OPENGL_API);
242 if (!make_current(context, this->surface)) {
243 printf("display=%p surface=%p context=%p curr=%p err=%d\n", eglGetCurrentDisplay(), this->surface, context, eglGetCurrentContext(),
247 present_thread_func();
248 delete_context(context);
252 void DeckLinkOutput::end_output()
254 if (!playback_initiated) {
259 frame_queues_changed.notify_all();
260 present_thread.join();
261 playback_initiated = false;
263 output->StopScheduledPlayback(0, nullptr, 0);
264 output->DisableVideoOutput();
265 output->DisableAudioOutput();
267 // Wait until all frames are accounted for, and free them.
269 unique_lock<mutex> lock(frame_queue_mutex);
270 while (!(frame_freelist.empty() && scheduled_frames.empty())) {
271 frame_queues_changed.wait(lock, [this]{ return !frame_freelist.empty(); });
272 frame_freelist.pop();
276 if (input != nullptr) {
280 if (output != nullptr) {
286 void DeckLinkOutput::send_frame(GLuint y_tex, GLuint cbcr_tex, YCbCrLumaCoefficients output_ycbcr_coefficients, const vector<RefCountedFrame> &input_frames, int64_t pts, int64_t duration)
288 assert(!should_quit.should_quit());
290 input_jitter_history.frame_arrived(steady_clock::now(), duration, /*dropped_frames=*/0, true);
292 if ((current_mode_flags & bmdDisplayModeColorspaceRec601) && output_ycbcr_coefficients == YCBCR_REC_709) {
293 if (!last_frame_had_mode_mismatch) {
294 fprintf(stderr, "WARNING: Chosen output mode expects Rec. 601 Y'CbCr coefficients.\n");
295 fprintf(stderr, " Consider --output-ycbcr-coefficients=rec601 (or =auto).\n");
297 last_frame_had_mode_mismatch = true;
298 ++metric_decklink_output_color_mismatch_frames;
299 } else if ((current_mode_flags & bmdDisplayModeColorspaceRec709) && output_ycbcr_coefficients == YCBCR_REC_601) {
300 if (!last_frame_had_mode_mismatch) {
301 fprintf(stderr, "WARNING: Chosen output mode expects Rec. 709 Y'CbCr coefficients.\n");
302 fprintf(stderr, " Consider --output-ycbcr-coefficients=rec709 (or =auto).\n");
304 last_frame_had_mode_mismatch = true;
305 ++metric_decklink_output_color_mismatch_frames;
307 last_frame_had_mode_mismatch = false;
310 unique_ptr<Frame> frame = get_frame();
311 if (global_flags.ten_bit_output) {
312 chroma_subsampler->create_v210(y_tex, cbcr_tex, width, height, frame->uyvy_tex);
314 chroma_subsampler->create_uyvy(y_tex, cbcr_tex, width, height, frame->uyvy_tex);
317 // Download the UYVY texture to the PBO.
318 glPixelStorei(GL_PACK_ROW_LENGTH, 0);
321 glBindBuffer(GL_PIXEL_PACK_BUFFER, frame->pbo);
324 if (global_flags.ten_bit_output) {
325 glBindTexture(GL_TEXTURE_2D, frame->uyvy_tex);
327 glGetTexImage(GL_TEXTURE_2D, 0, GL_RGBA, GL_UNSIGNED_INT_2_10_10_10_REV, BUFFER_OFFSET(0));
330 glBindTexture(GL_TEXTURE_2D, frame->uyvy_tex);
332 glGetTexImage(GL_TEXTURE_2D, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, BUFFER_OFFSET(0));
336 glBindTexture(GL_TEXTURE_2D, 0);
338 glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);
341 glMemoryBarrier(GL_TEXTURE_UPDATE_BARRIER_BIT | GL_CLIENT_MAPPED_BUFFER_BARRIER_BIT);
344 frame->fence = RefCountedGLsync(GL_SYNC_GPU_COMMANDS_COMPLETE, /*flags=*/0);
346 glFlush(); // Make the DeckLink thread see the fence as soon as possible.
349 frame->input_frames = input_frames;
350 frame->received_ts = find_received_timestamp(input_frames);
352 frame->duration = duration;
355 lock_guard<mutex> lock(frame_queue_mutex);
356 pending_video_frames.push(move(frame));
358 frame_queues_changed.notify_all();
361 void DeckLinkOutput::send_audio(int64_t pts, const std::vector<float> &samples)
363 return; // FIXME we may need to map pts or something?
365 unique_ptr<int32_t[]> int_samples(new int32_t[samples.size()]);
366 for (size_t i = 0; i < samples.size(); ++i) {
367 int_samples[i] = lrintf(samples[i] * 2147483648.0f);
370 uint32_t frames_written;
371 HRESULT result = output->ScheduleAudioSamples(int_samples.get(), samples.size() / 2,
372 pts, TIMEBASE, &frames_written);
373 if (result != S_OK) {
374 fprintf(stderr, "ScheduleAudioSamples(pts=%" PRId64 ") failed (result=0x%08x)\n", pts, result);
376 if (frames_written != samples.size() / 2) {
377 fprintf(stderr, "ScheduleAudioSamples() returned short write (%u/%zu)\n", frames_written, samples.size() / 2);
380 metric_decklink_output_scheduled_samples += samples.size() / 2;
383 void DeckLinkOutput::wait_for_frame(int64_t pts, int *dropped_frames, int64_t *frame_duration, bool *is_preroll, steady_clock::time_point *frame_timestamp)
385 assert(!should_quit.should_quit());
388 *frame_duration = this->frame_duration;
390 const BMDTimeValue buffer = lrint(*frame_duration * global_flags.output_buffer_frames);
391 const BMDTimeValue max_overshoot = lrint(*frame_duration * global_flags.output_slop_frames);
392 BMDTimeValue target_time = pts - buffer;
394 // While prerolling, we send out frames as quickly as we can.
395 if (target_time < base_pts) {
397 ++metric_decklink_output_scheduled_frames_preroll;
401 *is_preroll = !playback_started;
403 if (!playback_started) {
404 if (output->EndAudioPreroll() != S_OK) {
405 fprintf(stderr, "Could not end audio preroll\n");
408 if (output->StartScheduledPlayback(base_pts, TIMEBASE, 1.0) != S_OK) {
409 fprintf(stderr, "Could not start playback\n");
412 playback_started = true;
415 BMDTimeValue stream_frame_time;
416 double playback_speed;
417 output->GetScheduledStreamTime(TIMEBASE, &stream_frame_time, &playback_speed);
419 *frame_timestamp = steady_clock::now() +
420 nanoseconds((target_time - stream_frame_time) * 1000000000 / TIMEBASE);
422 metric_decklink_output_margin_seconds.count_event(
423 (target_time - stream_frame_time) / double(TIMEBASE));
425 // If we're ahead of time, wait for the frame to (approximately) start.
426 if (stream_frame_time < target_time) {
427 should_quit.sleep_until(*frame_timestamp);
428 ++metric_decklink_output_scheduled_frames_normal;
432 // If we overshot the previous frame by just a little,
433 // fire off one immediately.
434 if (stream_frame_time < target_time + max_overshoot) {
435 fprintf(stderr, "Warning: Frame was %ld ms late (but not skipping it due to --output-slop-frames).\n",
436 lrint((stream_frame_time - target_time) * 1000.0 / TIMEBASE));
437 ++metric_decklink_output_scheduled_frames_late;
441 // Oops, we missed by more than one frame. Return immediately,
442 // but drop so that we catch up.
443 *dropped_frames = (stream_frame_time - target_time + *frame_duration - 1) / *frame_duration;
444 const int64_t ns_per_frame = this->frame_duration * 1000000000 / TIMEBASE;
445 *frame_timestamp += nanoseconds(*dropped_frames * ns_per_frame);
446 fprintf(stderr, "Dropped %d output frames; skipping.\n", *dropped_frames);
447 metric_decklink_output_scheduled_frames_dropped += *dropped_frames;
448 ++metric_decklink_output_scheduled_frames_normal;
451 uint32_t DeckLinkOutput::pick_video_mode(uint32_t mode) const
453 if (video_modes.count(mode)) {
457 // Prioritize 59.94 > 60 > 29.97. If none of those are found, then pick the highest one.
458 for (const pair<int, int> &desired : vector<pair<int, int>>{ { 50, 1 }, { 60000, 1001 }, { 60, 1 }, { 30000, 1001 } }) {
459 for (const auto &it : video_modes) {
460 if (it.second.frame_rate_num * desired.second == desired.first * it.second.frame_rate_den) {
466 uint32_t best_mode = 0;
467 double best_fps = 0.0;
468 for (const auto &it : video_modes) {
469 double fps = double(it.second.frame_rate_num) / it.second.frame_rate_den;
470 if (fps > best_fps) {
471 best_mode = it.first;
478 YCbCrLumaCoefficients DeckLinkOutput::preferred_ycbcr_coefficients() const
480 if (current_mode_flags & bmdDisplayModeColorspaceRec601) {
481 return YCBCR_REC_601;
483 // Don't bother checking bmdDisplayModeColorspaceRec709;
484 // if none is set, 709 is a good default anyway.
485 return YCBCR_REC_709;
489 HRESULT DeckLinkOutput::ScheduledFrameCompleted(/* in */ IDeckLinkVideoFrame *completedFrame, /* in */ BMDOutputFrameCompletionResult result)
491 Frame *frame = static_cast<Frame *>(completedFrame);
493 BMDTimeValue stream_frame_time, played_at_time;
494 BMDTimeValue hardwareTime, timeInFrame, ticksPerFrame;
495 double playback_speed;
496 output->GetFrameCompletionReferenceTimestamp(frame, TIMEBASE, &played_at_time);
497 output->GetScheduledStreamTime(TIMEBASE, &stream_frame_time, &playback_speed);
498 output->GetHardwareReferenceClock(TIMEBASE, &hardwareTime, &timeInFrame, &ticksPerFrame);
500 steady_clock::time_point now = steady_clock::now();
501 int frame_delay = (stream_frame_time - frame->pts) / frame_duration - 1;
502 map<int, string> status = {
503 { bmdOutputFrameCompleted, "played" },
504 { bmdOutputFrameDisplayedLate, "DELAYED" },
505 { bmdOutputFrameDropped, "DROPPED" },
506 { bmdOutputFrameFlushed, "FLUSHED" }
509 if ((result == bmdOutputFrameCompleted || result == bmdOutputFrameDisplayedLate) && false) {
510 fprintf(stderr, "now=%ld / %.2f: frame with pts=%ld (%ld ago, %d delay) / %.2f was %s at time %ld (%ld ago)\n",
511 stream_frame_time, PTSToTime(stream_frame_time),
512 frame->pts, stream_frame_time - frame->pts, frame_delay, PTSToTime(frame->pts),
513 status[result].c_str(),
514 played_at_time, hardwareTime - played_at_time);
515 } else if (result == bmdOutputFrameDisplayedLate) {
516 fprintf(stderr, "now=%ld / %.2f: frame with pts=%ld (%ld ago, %d delay) / %.2f was %s to %.2f\n",
517 stream_frame_time, PTSToTime(stream_frame_time),
518 frame->pts, stream_frame_time - frame->pts, frame_delay, PTSToTime(frame->pts),
519 status[result].c_str(), PTSToTime(frame->pts) + frame_delay);
521 fprintf(stderr, "now=%ld / %.2f: frame with pts=%ld (%ld ago, %d delay) / %.2f was %s\n",
522 stream_frame_time, PTSToTime(stream_frame_time),
523 frame->pts, stream_frame_time - frame->pts, frame_delay, PTSToTime(frame->pts),
524 status[result].c_str());
526 if (frame_delay < 0) {
527 fprintf(stderr, "ERROR: Frame went backwards in time (scheduled to start at pts=%ld, ended at or before pts=%ld), something is strange.\n",
528 frame->pts, stream_frame_time);
533 case bmdOutputFrameCompleted:
534 ++metric_decklink_output_completed_frames_completed;
535 if (frame_delay != 0) {
536 fprintf(stderr, "ERROR: Frame was reportedly completed without delay, but was delayed nevertheless.\n");
537 // Our callback _might_ be delayed 1+ frame for other reasons,
538 // so ignore this. It's a pity GetFrameCompletionReferenceTimestamp()
539 // cannot give us a timestamp on the same time scale as
540 // GetScheduledStreamTime(); it would be more robust.
544 case bmdOutputFrameDisplayedLate:
545 // fprintf(stderr, "Output frame displayed late (pts=%" PRId64 ")\n", frame->pts);
546 //fprintf(stderr, "Consider increasing --output-buffer-frames if this persists.\n");
547 ++metric_decklink_output_completed_frames_late;
549 case bmdOutputFrameDropped:
550 // fprintf(stderr, "Output frame was dropped (pts=%" PRId64 ")\n", frame->pts);
551 //fprintf(stderr, "Consider increasing --output-buffer-frames if this persists.\n");
552 ++metric_decklink_output_completed_frames_dropped;
554 case bmdOutputFrameFlushed:
555 // fprintf(stderr, "Output frame was flushed (pts=%" PRId64 ")\n", frame->pts);
556 ++metric_decklink_output_completed_frames_flushed;
559 fprintf(stderr, "Output frame completed with unknown status %d\n", result);
560 ++metric_decklink_output_completed_frames_unknown;
564 static int frameno = 0;
565 print_latency("DeckLink output latency (frame received → output on HDMI):", frame->received_ts, false, &frameno, &latency_histogram);
568 lock_guard<mutex> lock(frame_queue_mutex);
569 frame_freelist.push(unique_ptr<Frame>(frame));
571 // Dropped frames can come out-of-order, so we can't just look at the front;
572 // we need to go and find it in the list.
573 auto it = find(scheduled_frames.begin(), scheduled_frames.end(), frame);
574 assert(it != scheduled_frames.end());
575 scheduled_frames.erase(it);
576 --metric_decklink_output_inflight_frames;
578 if (frame_delay > 0 && result == bmdOutputFrameDisplayedLate) {
579 // All frames that were queued earlier will be delayed,
580 // so update so that we don't double-count the delay.
581 int64_t prev_frame_pts = frame->pts + frame_delay * frame_duration;
582 for (Frame *other_frame : scheduled_frames) {
583 int64_t old_pts = other_frame->pts;
584 other_frame->pts = std::max(other_frame->pts, prev_frame_pts + frame_duration);
585 fprintf(stderr, " - moving frame from pts=%ld (%.3f) to pts=%ld (%.3f)\n",
586 old_pts, PTSToTime(old_pts), other_frame->pts, PTSToTime(other_frame->pts));
587 prev_frame_pts = other_frame->pts;
590 int64_t earliest_next_frame = (stream_frame_time + frame_duration - 1) / frame_duration * frame_duration;
591 earliest_next_frame = std::max(earliest_next_frame, prev_frame_pts + frame_duration);
592 if (next_output_pts < earliest_next_frame) {
593 // In effect, duplicate a frame. FIXME write something about this
594 // FIXME is this really right now? but perhaps we're forced
595 // and it messes up the queue length calculation temporarily, we need something else there
596 //fprintf(stderr, "Duplicating frame %d times due to starvation!\n", frame_delay);
598 // FIXME this causes us to believe in output jitter?
599 fprintf(stderr, " - moving output pointer from pts=%ld (%.3f) to pts=%ld (%.3f)\n",
600 next_output_pts, PTSToTime(next_output_pts), earliest_next_frame, PTSToTime(earliest_next_frame));
601 next_output_pts = earliest_next_frame;
606 if (result == bmdOutputFrameCompleted || result == bmdOutputFrameDisplayedLate) {
607 //output_jitter_history.frame_arrived(now, frame_duration, /*dropped_frames=*/frame_delay, true);
608 // TODO: backdate now, and possibly also get_expected_next_frame()?
609 queue_length_policy.update_policy(
611 input_jitter_history.get_expected_next_frame(),
612 frame->duration, frame_duration,
613 input_jitter_history.estimate_max_jitter(),
614 // output_jitter_history.estimate_max_jitter(), true);
616 num_safe_frames = queue_length_policy.get_safe_queue_length();
618 fprintf(stderr, "%zu frames in flight, safe queue length = %u (starv=%d)\n", scheduled_frames.size(), queue_length_policy.get_safe_queue_length(), frame_delay);
624 HRESULT DeckLinkOutput::ScheduledPlaybackHasStopped()
626 printf("playback stopped!\n");
630 unique_ptr<DeckLinkOutput::Frame> DeckLinkOutput::get_frame()
632 lock_guard<mutex> lock(frame_queue_mutex);
634 if (!frame_freelist.empty()) {
635 unique_ptr<Frame> frame = move(frame_freelist.front());
636 frame_freelist.pop();
640 unique_ptr<Frame> frame(new Frame);
643 if (global_flags.ten_bit_output) {
644 stride = v210Converter::get_v210_stride(width);
645 GLint v210_width = stride / sizeof(uint32_t);
646 frame->uyvy_tex = resource_pool->create_2d_texture(GL_RGB10_A2, v210_width, height);
648 // We need valid texture state, or NVIDIA won't allow us to write to the texture.
649 glBindTexture(GL_TEXTURE_2D, frame->uyvy_tex);
651 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
655 frame->uyvy_tex = resource_pool->create_2d_texture(GL_RGBA8, width / 2, height);
658 glGenBuffers(1, &frame->pbo);
660 glBindBuffer(GL_PIXEL_PACK_BUFFER, frame->pbo);
662 glBufferStorage(GL_PIXEL_PACK_BUFFER, stride * height, nullptr, GL_MAP_READ_BIT | GL_MAP_PERSISTENT_BIT);
664 frame->uyvy_ptr = (uint8_t *)glMapBufferRange(GL_PIXEL_PACK_BUFFER, 0, stride * height, GL_MAP_READ_BIT | GL_MAP_PERSISTENT_BIT);
666 frame->uyvy_ptr_local.reset(new uint8_t[stride * height]);
667 frame->resource_pool = resource_pool;
672 void DeckLinkOutput::present_thread_func()
674 pthread_setname_np(pthread_self(), "DeckLinkOutput");
676 unique_ptr<Frame> frame;
678 unique_lock<mutex> lock(frame_queue_mutex);
679 frame_queues_changed.wait(lock, [this]{
680 return should_quit.should_quit() || !pending_video_frames.empty();
682 if (should_quit.should_quit()) {
685 frame = move(pending_video_frames.front());
686 pending_video_frames.pop();
688 if (scheduled_frames.size() > num_safe_frames) { // FIXME check off-by-one here
689 fprintf(stderr, "Dropping frame to keep latency down!\n");
694 // Overwrite the pts given by the client; it doesn't own our clock.
695 // TODO: Write something about what semi-unsynchronized really means.
696 frame->pts = next_output_pts;
697 next_output_pts += frame_duration;
701 int err = glClientWaitSync(frame->fence.get(), /*flags=*/0, 0);
702 if (err == GL_TIMEOUT_EXPIRED) {
703 // NVIDIA likes to busy-wait; yield instead.
704 this_thread::sleep_for(milliseconds(1));
710 frame->fence.reset();
712 if (global_flags.ten_bit_output) {
713 memcpy(frame->uyvy_ptr_local.get(), frame->uyvy_ptr, v210Converter::get_v210_stride(width) * height);
715 memcpy(frame->uyvy_ptr_local.get(), frame->uyvy_ptr, width * height * 2);
718 // Release any input frames we needed to render this frame.
719 frame->input_frames.clear();
721 BMDTimeValue pts = frame->pts;
722 BMDTimeValue duration = frame->duration;
723 HRESULT res = output->ScheduleVideoFrame(frame.get(), pts, duration, TIMEBASE);
724 lock_guard<mutex> lock(frame_queue_mutex);
726 scheduled_frames.push_back(frame.release()); // Owned by the driver now.
727 ++metric_decklink_output_inflight_frames;
729 fprintf(stderr, "Could not schedule video frame! (error=0x%08x)\n", res);
731 frame_freelist.push(move(frame));
736 double DeckLinkOutput::PTSToTime(int64_t pts)
738 return double(pts) / frame_duration;
741 HRESULT STDMETHODCALLTYPE DeckLinkOutput::QueryInterface(REFIID, LPVOID *)
743 return E_NOINTERFACE;
746 ULONG STDMETHODCALLTYPE DeckLinkOutput::AddRef()
748 return refcount.fetch_add(1) + 1;
751 ULONG STDMETHODCALLTYPE DeckLinkOutput::Release()
753 int new_ref = refcount.fetch_sub(1) - 1;
759 DeckLinkOutput::Frame::~Frame()
761 glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo);
763 glUnmapBuffer(GL_PIXEL_PACK_BUFFER);
765 glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);
767 glDeleteBuffers(1, &pbo);
769 resource_pool->release_2d_texture(uyvy_tex);
773 HRESULT STDMETHODCALLTYPE DeckLinkOutput::Frame::QueryInterface(REFIID, LPVOID *)
775 return E_NOINTERFACE;
778 ULONG STDMETHODCALLTYPE DeckLinkOutput::Frame::AddRef()
780 return refcount.fetch_add(1) + 1;
783 ULONG STDMETHODCALLTYPE DeckLinkOutput::Frame::Release()
785 int new_ref = refcount.fetch_sub(1) - 1;
791 long DeckLinkOutput::Frame::GetWidth()
793 return global_flags.width;
796 long DeckLinkOutput::Frame::GetHeight()
798 return global_flags.height;
801 long DeckLinkOutput::Frame::GetRowBytes()
803 if (global_flags.ten_bit_output) {
804 return v210Converter::get_v210_stride(global_flags.width);
806 return global_flags.width * 2;
810 BMDPixelFormat DeckLinkOutput::Frame::GetPixelFormat()
812 if (global_flags.ten_bit_output) {
813 return bmdFormat10BitYUV;
815 return bmdFormat8BitYUV;
819 BMDFrameFlags DeckLinkOutput::Frame::GetFlags()
821 return bmdFrameFlagDefault;
824 HRESULT DeckLinkOutput::Frame::GetBytes(/* out */ void **buffer)
826 *buffer = uyvy_ptr_local.get();
830 HRESULT DeckLinkOutput::Frame::GetTimecode(/* in */ BMDTimecodeFormat format, /* out */ IDeckLinkTimecode **timecode)
832 fprintf(stderr, "STUB: GetTimecode()\n");
836 HRESULT DeckLinkOutput::Frame::GetAncillaryData(/* out */ IDeckLinkVideoFrameAncillary **ancillary)
838 fprintf(stderr, "STUB: GetAncillaryData()\n");