1 #include <movit/effect_util.h>
2 #include <movit/util.h>
3 #include <movit/resource_pool.h> // Must be above the Xlib includes.
8 #include "chroma_subsampler.h"
9 #include "decklink_output.h"
10 #include "decklink_util.h"
12 #include "print_latency.h"
13 #include "resource_pool.h"
16 using namespace movit;
18 using namespace std::chrono;
20 DeckLinkOutput::DeckLinkOutput(ResourcePool *resource_pool, QSurface *surface, unsigned width, unsigned height, unsigned card_index)
21 : resource_pool(resource_pool), surface(surface), width(width), height(height), card_index(card_index)
23 chroma_subsampler.reset(new ChromaSubsampler(resource_pool));
26 void DeckLinkOutput::set_device(IDeckLink *decklink)
28 if (decklink->QueryInterface(IID_IDeckLinkOutput, (void**)&output) != S_OK) {
29 fprintf(stderr, "Card %u has no outputs\n", card_index);
33 IDeckLinkDisplayModeIterator *mode_it;
34 if (output->GetDisplayModeIterator(&mode_it) != S_OK) {
35 fprintf(stderr, "Failed to enumerate output display modes for card %u\n", card_index);
41 for (const auto &it : summarize_video_modes(mode_it, card_index)) {
42 if (it.second.width != width || it.second.height != height) {
46 // We could support interlaced modes, but let's stay out of it for now,
47 // since we don't have interlaced stream output.
48 if (it.second.interlaced) {
52 video_modes.insert(it);
57 // HDMI or SDI generally mean “both HDMI and SDI at the same time” on DeckLink cards
58 // that support both; pick_default_video_connection() will generally pick one of those
59 // if they exist. We're not very likely to need analog outputs, so we don't need a way
60 // to change beyond that.
61 video_connection = pick_default_video_connection(decklink, BMDDeckLinkVideoOutputConnections, card_index);
64 void DeckLinkOutput::start_output(uint32_t mode, int64_t base_pts)
67 assert(!playback_initiated);
70 playback_initiated = true;
71 playback_started = false;
72 this->base_pts = base_pts;
74 IDeckLinkConfiguration *config = nullptr;
75 if (output->QueryInterface(IID_IDeckLinkConfiguration, (void**)&config) != S_OK) {
76 fprintf(stderr, "Failed to get configuration interface for output card\n");
79 if (config->SetFlag(bmdDeckLinkConfigLowLatencyVideoOutput, true) != S_OK) {
80 fprintf(stderr, "Failed to set low latency output\n");
83 if (config->SetInt(bmdDeckLinkConfigVideoOutputConnection, video_connection) != S_OK) {
84 fprintf(stderr, "Failed to set video output connection for card %u\n", card_index);
87 if (config->SetFlag(bmdDeckLinkConfigUse1080pNotPsF, true) != S_OK) {
88 fprintf(stderr, "Failed to set PsF flag for card\n");
91 if (config->SetFlag(bmdDeckLinkConfigSMPTELevelAOutput, true) != S_OK) {
92 // This affects at least some no-name SDI->HDMI converters.
93 // Warn, but don't die.
94 fprintf(stderr, "WARNING: Failed to enable SMTPE Level A; resolutions like 1080p60 might have issues.\n");
97 BMDDisplayModeSupport support;
98 IDeckLinkDisplayMode *display_mode;
99 if (output->DoesSupportVideoMode(mode, bmdFormat8BitYUV, bmdVideoOutputFlagDefault,
100 &support, &display_mode) != S_OK) {
101 fprintf(stderr, "Couldn't ask for format support\n");
105 if (support == bmdDisplayModeNotSupported) {
106 fprintf(stderr, "Requested display mode not supported\n");
110 current_mode_flags = display_mode->GetFlags();
112 BMDTimeValue time_value;
113 BMDTimeScale time_scale;
114 if (display_mode->GetFrameRate(&time_value, &time_scale) != S_OK) {
115 fprintf(stderr, "Couldn't get frame rate\n");
119 frame_duration = time_value * TIMEBASE / time_scale;
121 display_mode->Release();
123 HRESULT result = output->EnableVideoOutput(mode, bmdVideoOutputFlagDefault);
124 if (result != S_OK) {
125 fprintf(stderr, "Couldn't enable output with error 0x%x\n", result);
128 if (output->SetScheduledFrameCompletionCallback(this) != S_OK) {
129 fprintf(stderr, "Couldn't set callback\n");
132 assert(OUTPUT_FREQUENCY == 48000);
133 if (output->EnableAudioOutput(bmdAudioSampleRate48kHz, bmdAudioSampleType32bitInteger, 2, bmdAudioOutputStreamTimestamped) != S_OK) {
134 fprintf(stderr, "Couldn't enable audio output\n");
137 if (output->BeginAudioPreroll() != S_OK) {
138 fprintf(stderr, "Couldn't begin audio preroll\n");
142 present_thread = thread([this]{
143 QOpenGLContext *context = create_context(this->surface);
144 eglBindAPI(EGL_OPENGL_API);
145 if (!make_current(context, this->surface)) {
146 printf("display=%p surface=%p context=%p curr=%p err=%d\n", eglGetCurrentDisplay(), this->surface, context, eglGetCurrentContext(),
150 present_thread_func();
151 delete_context(context);
155 void DeckLinkOutput::end_output()
157 if (!playback_initiated) {
162 frame_queues_changed.notify_all();
163 present_thread.join();
164 playback_initiated = false;
166 output->StopScheduledPlayback(0, nullptr, 0);
167 output->DisableVideoOutput();
168 output->DisableAudioOutput();
170 // Wait until all frames are accounted for, and free them.
172 unique_lock<mutex> lock(frame_queue_mutex);
173 while (!(frame_freelist.empty() && num_frames_in_flight == 0)) {
174 frame_queues_changed.wait(lock, [this]{ return !frame_freelist.empty(); });
175 frame_freelist.pop();
180 void DeckLinkOutput::send_frame(GLuint y_tex, GLuint cbcr_tex, YCbCrLumaCoefficients output_ycbcr_coefficients, const vector<RefCountedFrame> &input_frames, int64_t pts, int64_t duration)
182 assert(!should_quit);
184 if ((current_mode_flags & bmdDisplayModeColorspaceRec601) && output_ycbcr_coefficients == YCBCR_REC_709) {
185 if (!last_frame_had_mode_mismatch) {
186 fprintf(stderr, "WARNING: Chosen output mode expects Rec. 601 Y'CbCr coefficients.\n");
187 fprintf(stderr, " Consider --output-ycbcr-coefficients=rec601 (or =auto).\n");
189 last_frame_had_mode_mismatch = true;
190 } else if ((current_mode_flags & bmdDisplayModeColorspaceRec709) && output_ycbcr_coefficients == YCBCR_REC_601) {
191 if (!last_frame_had_mode_mismatch) {
192 fprintf(stderr, "WARNING: Chosen output mode expects Rec. 709 Y'CbCr coefficients.\n");
193 fprintf(stderr, " Consider --output-ycbcr-coefficients=rec709 (or =auto).\n");
195 last_frame_had_mode_mismatch = true;
197 last_frame_had_mode_mismatch = false;
200 unique_ptr<Frame> frame = move(get_frame());
201 chroma_subsampler->create_uyvy(y_tex, cbcr_tex, width, height, frame->uyvy_tex);
203 // Download the UYVY texture to the PBO.
204 glPixelStorei(GL_PACK_ROW_LENGTH, 0);
207 glBindBuffer(GL_PIXEL_PACK_BUFFER, frame->pbo);
210 glBindTexture(GL_TEXTURE_2D, frame->uyvy_tex);
212 glGetTexImage(GL_TEXTURE_2D, 0, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, BUFFER_OFFSET(0));
215 glBindTexture(GL_TEXTURE_2D, 0);
217 glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);
220 glMemoryBarrier(GL_TEXTURE_UPDATE_BARRIER_BIT | GL_CLIENT_MAPPED_BUFFER_BARRIER_BIT);
223 frame->fence = RefCountedGLsync(GL_SYNC_GPU_COMMANDS_COMPLETE, /*flags=*/0);
225 glFlush(); // Make the DeckLink thread see the fence as soon as possible.
228 frame->input_frames = input_frames;
229 frame->received_ts = find_received_timestamp(input_frames);
231 frame->duration = duration;
234 unique_lock<mutex> lock(frame_queue_mutex);
235 pending_video_frames.push(move(frame));
237 frame_queues_changed.notify_all();
240 void DeckLinkOutput::send_audio(int64_t pts, const std::vector<float> &samples)
242 unique_ptr<int32_t[]> int_samples(new int32_t[samples.size()]);
243 for (size_t i = 0; i < samples.size(); ++i) {
244 int_samples[i] = lrintf(samples[i] * 2147483648.0f);
247 uint32_t frames_written;
248 HRESULT result = output->ScheduleAudioSamples(int_samples.get(), samples.size() / 2,
249 pts, TIMEBASE, &frames_written);
250 if (result != S_OK) {
251 fprintf(stderr, "ScheduleAudioSamples(pts=%ld) failed (result=0x%08x)\n", pts, result);
253 if (frames_written != samples.size() / 2) {
254 fprintf(stderr, "ScheduleAudioSamples() returned short write (%u/%ld)\n", frames_written, samples.size() / 2);
259 void DeckLinkOutput::wait_for_frame(int64_t pts, int *dropped_frames, int64_t *frame_duration, bool *is_preroll, steady_clock::time_point *frame_timestamp)
261 assert(!should_quit);
264 *frame_duration = this->frame_duration;
266 const BMDTimeValue buffer = lrint(*frame_duration * global_flags.output_buffer_frames);
267 const BMDTimeValue max_overshoot = lrint(*frame_duration * global_flags.output_slop_frames);
268 BMDTimeValue target_time = pts - buffer;
270 // While prerolling, we send out frames as quickly as we can.
271 if (target_time < base_pts) {
276 *is_preroll = !playback_started;
278 if (!playback_started) {
279 if (output->EndAudioPreroll() != S_OK) {
280 fprintf(stderr, "Could not end audio preroll\n");
283 if (output->StartScheduledPlayback(base_pts, TIMEBASE, 1.0) != S_OK) {
284 fprintf(stderr, "Could not start playback\n");
287 playback_started = true;
290 BMDTimeValue stream_frame_time;
291 double playback_speed;
292 output->GetScheduledStreamTime(TIMEBASE, &stream_frame_time, &playback_speed);
294 *frame_timestamp = steady_clock::now() +
295 nanoseconds((target_time - stream_frame_time) * 1000000000 / TIMEBASE);
297 // If we're ahead of time, wait for the frame to (approximately) start.
298 if (stream_frame_time < target_time) {
299 this_thread::sleep_until(*frame_timestamp);
303 // If we overshot the previous frame by just a little,
304 // fire off one immediately.
305 if (stream_frame_time < target_time + max_overshoot) {
306 fprintf(stderr, "Warning: Frame was %ld ms late (but not skipping it due to --output-slop-frames).\n",
307 lrint((stream_frame_time - target_time) * 1000.0 / TIMEBASE));
311 // Oops, we missed by more than one frame. Return immediately,
312 // but drop so that we catch up.
313 *dropped_frames = (stream_frame_time - target_time + *frame_duration - 1) / *frame_duration;
314 const int64_t ns_per_frame = this->frame_duration * 1000000000 / TIMEBASE;
315 *frame_timestamp += nanoseconds(*dropped_frames * ns_per_frame);
316 fprintf(stderr, "Dropped %d output frames; skipping.\n", *dropped_frames);
319 uint32_t DeckLinkOutput::pick_video_mode(uint32_t mode) const
321 if (video_modes.count(mode)) {
325 // Prioritize 59.94 > 60 > 29.97. If none of those are found, then pick the highest one.
326 for (const pair<int, int> &desired : vector<pair<int, int>>{ { 60000, 1001 }, { 60, 0 }, { 30000, 1001 } }) {
327 for (const auto &it : video_modes) {
328 if (it.second.frame_rate_num * desired.second == desired.first * it.second.frame_rate_den) {
334 uint32_t best_mode = 0;
335 double best_fps = 0.0;
336 for (const auto &it : video_modes) {
337 double fps = double(it.second.frame_rate_num) / it.second.frame_rate_den;
338 if (fps > best_fps) {
339 best_mode = it.first;
346 YCbCrLumaCoefficients DeckLinkOutput::preferred_ycbcr_coefficients() const
348 if (current_mode_flags & bmdDisplayModeColorspaceRec601) {
349 return YCBCR_REC_601;
351 // Don't bother checking bmdDisplayModeColorspaceRec709;
352 // if none is set, 709 is a good default anyway.
353 return YCBCR_REC_709;
357 HRESULT DeckLinkOutput::ScheduledFrameCompleted(/* in */ IDeckLinkVideoFrame *completedFrame, /* in */ BMDOutputFrameCompletionResult result)
359 Frame *frame = static_cast<Frame *>(completedFrame);
361 case bmdOutputFrameCompleted:
363 case bmdOutputFrameDisplayedLate:
364 fprintf(stderr, "Output frame displayed late (pts=%ld)\n", frame->pts);
365 fprintf(stderr, "Consider increasing --output-buffer-frames if this persists.\n");
367 case bmdOutputFrameDropped:
368 fprintf(stderr, "Output frame was dropped (pts=%ld)\n", frame->pts);
369 fprintf(stderr, "Consider increasing --output-buffer-frames if this persists.\n");
371 case bmdOutputFrameFlushed:
372 fprintf(stderr, "Output frame was flushed (pts=%ld)\n", frame->pts);
375 fprintf(stderr, "Output frame completed with unknown status %d\n", result);
379 static int frameno = 0;
380 print_latency("DeckLink output latency (frame received → output on HDMI):", frame->received_ts, false, &frameno);
383 lock_guard<mutex> lock(frame_queue_mutex);
384 frame_freelist.push(unique_ptr<Frame>(frame));
385 --num_frames_in_flight;
391 HRESULT DeckLinkOutput::ScheduledPlaybackHasStopped()
393 printf("playback stopped!\n");
397 unique_ptr<DeckLinkOutput::Frame> DeckLinkOutput::get_frame()
399 lock_guard<mutex> lock(frame_queue_mutex);
401 if (!frame_freelist.empty()) {
402 unique_ptr<Frame> frame = move(frame_freelist.front());
403 frame_freelist.pop();
407 unique_ptr<Frame> frame(new Frame);
409 frame->uyvy_tex = resource_pool->create_2d_texture(GL_RGBA8, width / 2, height);
411 glGenBuffers(1, &frame->pbo);
413 glBindBuffer(GL_PIXEL_PACK_BUFFER, frame->pbo);
415 glBufferStorage(GL_PIXEL_PACK_BUFFER, width * height * 2, NULL, GL_MAP_READ_BIT | GL_MAP_PERSISTENT_BIT);
417 frame->uyvy_ptr = (uint8_t *)glMapBufferRange(GL_PIXEL_PACK_BUFFER, 0, width * height * 2, GL_MAP_READ_BIT | GL_MAP_PERSISTENT_BIT);
419 frame->uyvy_ptr_local.reset(new uint8_t[width * height * 2]);
420 frame->resource_pool = resource_pool;
425 void DeckLinkOutput::present_thread_func()
427 pthread_setname_np(pthread_self(), "DeckLinkOutput");
429 unique_ptr<Frame> frame;
431 unique_lock<mutex> lock(frame_queue_mutex);
432 frame_queues_changed.wait(lock, [this]{
433 return should_quit || !pending_video_frames.empty();
438 frame = move(pending_video_frames.front());
439 pending_video_frames.pop();
440 ++num_frames_in_flight;
443 glWaitSync(frame->fence.get(), /*flags=*/0, GL_TIMEOUT_IGNORED);
445 frame->fence.reset();
447 memcpy(frame->uyvy_ptr_local.get(), frame->uyvy_ptr, width * height * 2);
449 // Release any input frames we needed to render this frame.
450 frame->input_frames.clear();
452 BMDTimeValue pts = frame->pts;
453 BMDTimeValue duration = frame->duration;
454 HRESULT res = output->ScheduleVideoFrame(frame.get(), pts, duration, TIMEBASE);
456 frame.release(); // Owned by the driver now.
458 fprintf(stderr, "Could not schedule video frame! (error=0x%08x)\n", res);
460 lock_guard<mutex> lock(frame_queue_mutex);
461 frame_freelist.push(move(frame));
462 --num_frames_in_flight;
467 HRESULT STDMETHODCALLTYPE DeckLinkOutput::QueryInterface(REFIID, LPVOID *)
469 return E_NOINTERFACE;
472 ULONG STDMETHODCALLTYPE DeckLinkOutput::AddRef()
474 return refcount.fetch_add(1) + 1;
477 ULONG STDMETHODCALLTYPE DeckLinkOutput::Release()
479 int new_ref = refcount.fetch_sub(1) - 1;
485 DeckLinkOutput::Frame::~Frame()
487 glBindBuffer(GL_PIXEL_PACK_BUFFER, pbo);
489 glUnmapBuffer(GL_PIXEL_PACK_BUFFER);
491 glBindBuffer(GL_PIXEL_PACK_BUFFER, 0);
493 glDeleteBuffers(1, &pbo);
495 resource_pool->release_2d_texture(uyvy_tex);
499 HRESULT STDMETHODCALLTYPE DeckLinkOutput::Frame::QueryInterface(REFIID, LPVOID *)
501 return E_NOINTERFACE;
504 ULONG STDMETHODCALLTYPE DeckLinkOutput::Frame::AddRef()
506 return refcount.fetch_add(1) + 1;
509 ULONG STDMETHODCALLTYPE DeckLinkOutput::Frame::Release()
511 int new_ref = refcount.fetch_sub(1) - 1;
517 long DeckLinkOutput::Frame::GetWidth()
519 return global_flags.width;
522 long DeckLinkOutput::Frame::GetHeight()
524 return global_flags.height;
527 long DeckLinkOutput::Frame::GetRowBytes()
529 return global_flags.width * 2;
532 BMDPixelFormat DeckLinkOutput::Frame::GetPixelFormat()
534 return bmdFormat8BitYUV;
537 BMDFrameFlags DeckLinkOutput::Frame::GetFlags()
539 return bmdFrameFlagDefault;
542 HRESULT DeckLinkOutput::Frame::GetBytes(/* out */ void **buffer)
544 *buffer = uyvy_ptr_local.get();
548 HRESULT DeckLinkOutput::Frame::GetTimecode(/* in */ BMDTimecodeFormat format, /* out */ IDeckLinkTimecode **timecode)
550 fprintf(stderr, "STUB: GetTimecode()\n");
554 HRESULT DeckLinkOutput::Frame::GetAncillaryData(/* out */ IDeckLinkVideoFrameAncillary **ancillary)
556 fprintf(stderr, "STUB: GetAncillaryData()\n");