8 #include <condition_variable>
23 // An interface for frame allocators; if you do not specify one
24 // (using set_video_frame_allocator), a default one that pre-allocates
25 // a freelist of eight frames using new[] will be used. Specifying
26 // your own can be useful if you have special demands for where you want the
27 // frame to end up and don't want to spend the extra copy to get it there, for
28 // instance GPU memory.
29 class FrameAllocator {
32 uint8_t *data = nullptr;
33 uint8_t *data2 = nullptr; // Only if interleaved == true.
34 uint8_t *data_copy = nullptr; // Will get a non-interleaved copy if not nullptr.
35 size_t len = 0; // Number of bytes we actually have.
36 size_t size = 0; // Number of bytes we have room for.
38 void *userdata = nullptr;
39 FrameAllocator *owner = nullptr;
41 // If set to true, every other byte will go to data and to data2.
42 // If so, <len> and <size> are still about the number of total bytes
43 // so if size == 1024, there's 512 bytes in data and 512 in data2.
45 // This doesn't really make any sense if you asked for the
46 // 10BitYCbCr pixel format.
47 bool interleaved = false;
49 // At what point this frame was received. Note that this marks the
50 // _end_ of the frame being received, not the beginning.
51 // Thus, if you want to measure latency, you'll also need to include
52 // the time the frame actually took to transfer (usually 1/fps,
53 // ie., the frames are typically transferred in real time).
54 std::chrono::steady_clock::time_point received_timestamp =
55 std::chrono::steady_clock::time_point::min();
58 virtual ~FrameAllocator();
60 // Request a video frame. Note that this is called from the
61 // USB thread, which runs with realtime priority and is
62 // very sensitive to delays. Thus, you should not do anything
63 // here that might sleep, including calling malloc().
64 // (Taking a mutex is borderline.)
66 // The Frame object will be given to the frame callback,
67 // which is responsible for releasing the video frame back
68 // once it is usable for new frames (ie., it will no longer
69 // be read from). You can use the "userdata" pointer for
70 // whatever you want to identify this frame if you need to.
72 // Returning a Frame with data==nullptr is allowed;
73 // if so, the frame in progress will be dropped.
74 virtual Frame alloc_frame() = 0;
76 // Similar to alloc_frame(), with two additional restrictions:
78 // - The width, height and stride given must be correct
79 // (can not be changed after the call).
80 // - create_frame(), unlike alloc_frame(), is allowed to sleep
81 // (so bmusb will never call it, but in Nageru, other producers
84 // These two restrictions are relevant for Nageru, since it means that
85 // it can make frame_copy point directly into a VA-API buffer to avoid
87 virtual Frame create_frame(size_t width, size_t height, size_t stride)
92 virtual void release_frame(Frame frame) = 0;
95 // Audio is more important than video, and also much cheaper.
96 // By having many more audio frames available, hopefully if something
97 // starts to drop, we'll have CPU load go down (from not having to
98 // process as much video) before we have to drop audio.
99 #define NUM_QUEUED_VIDEO_FRAMES 16
100 #define NUM_QUEUED_AUDIO_FRAMES 64
102 class MallocFrameAllocator : public FrameAllocator {
104 MallocFrameAllocator(size_t frame_size, size_t num_queued_frames);
105 Frame alloc_frame() override;
106 void release_frame(Frame frame) override;
111 std::mutex freelist_mutex;
112 std::stack<std::unique_ptr<uint8_t[]>> freelist; // All of size <frame_size>.
115 // Represents an input mode you can tune a card to.
118 bool autodetect = false; // If true, all the remaining fields are irrelevant.
119 unsigned width = 0, height = 0;
120 unsigned frame_rate_num = 0, frame_rate_den = 0;
121 bool interlaced = false;
124 // Represents the format of an actual frame coming in.
125 // Note: Frame rate is _frame_ rate, not field rate. So 1080i60 gets 30/1, _not_ 60/1.
126 // "second_field_start" is only valid for interlaced modes. If it is 1,
127 // the two fields are actually stored interlaced (ie., every other line).
128 // If not, each field is stored consecutively, and it signifies how many lines
129 // from the very top of the frame there are before the second field
130 // starts (so it will always be >= height/2 + extra_lines_top).
132 uint16_t id = 0; // For debugging/logging only.
133 unsigned width = 0, height = 0, second_field_start = 0;
134 unsigned extra_lines_top = 0, extra_lines_bottom = 0;
135 unsigned frame_rate_nom = 0, frame_rate_den = 0;
136 unsigned stride = 0; // In bytes, assuming no interleaving.
137 bool interlaced = false;
138 bool has_signal = false;
139 bool is_connected = true; // If false, then has_signal makes no sense.
143 uint16_t id = 0; // For debugging/logging only.
144 unsigned bits_per_sample = 0;
145 unsigned num_channels = 0;
146 unsigned sample_rate = 48000;
150 // 8-bit 4:2:2 in the standard Cb Y Cr Y order (UYVY).
151 // This is the default.
152 PixelFormat_8BitYCbCr,
154 // 10-bit 4:2:2 in v210 order. Six pixels (six Y', three Cb,
155 // three Cr) are packed into four 32-bit little-endian ints
156 // in the following pattern (see e.g. the DeckLink documentation
166 // If you read in RGB order and ignore the unused top bits,
167 // this is essentially Cb Y Cr Y order, just like UYVY is.
169 // Note that unlike true v210, there is no guarantee about
170 // 128-byte line alignment (or lack thereof); you should check
171 // the stride member of VideoFormat.
172 PixelFormat_10BitYCbCr,
174 // 8-bit 4:4:4:4 BGRA (in that order). bmusb itself doesn't
175 // produce this, but it is useful to represent e.g. synthetic inputs.
176 PixelFormat_8BitBGRA,
178 // 8-bit 4:2:0, 4:2:2, 4:4:4 or really anything else, planar
179 // (ie., first all Y', then all Cb, then all Cr). bmusb doesn't
180 // produce this, nor does it specify a mechanism to describe
181 // the precise details of the format.
182 PixelFormat_8BitYCbCrPlanar,
184 // These exist only so that the type is guaranteed wide enough
185 // to contain values up to 127. CaptureInterface instances
186 // are free to use them as they see fit for private uses.
187 PixelFormat_Unused100 = 100,
188 PixelFormat_Unused127 = 127
191 typedef std::function<void(uint16_t timecode,
192 FrameAllocator::Frame video_frame, size_t video_offset, VideoFormat video_format,
193 FrameAllocator::Frame audio_frame, size_t audio_offset, AudioFormat audio_format)>
196 typedef std::function<void(libusb_device *dev)> card_connected_callback_t;
197 typedef std::function<void()> card_disconnected_callback_t;
199 class CaptureInterface {
201 virtual ~CaptureInterface() {}
203 virtual std::map<uint32_t, VideoMode> get_available_video_modes() const = 0;
204 virtual uint32_t get_current_video_mode() const = 0;
205 virtual void set_video_mode(uint32_t video_mode_id) = 0;
207 // TODO: Add a way to query this based on mode?
208 virtual std::set<PixelFormat> get_available_pixel_formats() const = 0;
209 virtual void set_pixel_format(PixelFormat pixel_format) = 0;
210 virtual PixelFormat get_current_pixel_format() const = 0;
212 virtual std::map<uint32_t, std::string> get_available_video_inputs() const = 0;
213 virtual void set_video_input(uint32_t video_input_id) = 0;
214 virtual uint32_t get_current_video_input() const = 0;
216 virtual std::map<uint32_t, std::string> get_available_audio_inputs() const = 0;
217 virtual void set_audio_input(uint32_t audio_input_id) = 0;
218 virtual uint32_t get_current_audio_input() const = 0;
220 // Does not take ownership.
221 virtual void set_video_frame_allocator(FrameAllocator *allocator) = 0;
223 virtual FrameAllocator *get_video_frame_allocator() = 0;
225 // Does not take ownership.
226 virtual void set_audio_frame_allocator(FrameAllocator *allocator) = 0;
228 virtual FrameAllocator *get_audio_frame_allocator() = 0;
230 virtual void set_frame_callback(frame_callback_t callback) = 0;
232 // Needs to be run before configure_card().
233 virtual void set_dequeue_thread_callbacks(std::function<void()> init, std::function<void()> cleanup) = 0;
235 // Only valid after configure_card().
236 virtual std::string get_description() const = 0;
238 virtual void configure_card() = 0;
240 virtual void start_bm_capture() = 0;
242 virtual void stop_dequeue_thread() = 0;
244 // If a card is disconnected, it cannot come back; you should call stop_dequeue_thread()
246 virtual bool get_disconnected() const = 0;
249 // The actual capturing class, representing capture from a single card.
250 class BMUSBCapture : public CaptureInterface {
252 BMUSBCapture(int card_index, libusb_device *dev = nullptr)
253 : card_index(card_index), dev(dev)
259 // Note: Cards could be unplugged and replugged between this call and
260 // actually opening the card (in configure_card()).
261 static unsigned num_cards();
263 std::set<PixelFormat> get_available_pixel_formats() const override
265 return std::set<PixelFormat>{ PixelFormat_8BitYCbCr, PixelFormat_10BitYCbCr };
268 void set_pixel_format(PixelFormat pixel_format) override;
270 PixelFormat get_current_pixel_format() const
272 return current_pixel_format;
275 std::map<uint32_t, VideoMode> get_available_video_modes() const override;
276 uint32_t get_current_video_mode() const override;
277 void set_video_mode(uint32_t video_mode_id) override;
279 virtual std::map<uint32_t, std::string> get_available_video_inputs() const override;
280 virtual void set_video_input(uint32_t video_input_id) override;
281 virtual uint32_t get_current_video_input() const override { return current_video_input; }
283 virtual std::map<uint32_t, std::string> get_available_audio_inputs() const override;
284 virtual void set_audio_input(uint32_t audio_input_id) override;
285 virtual uint32_t get_current_audio_input() const override { return current_audio_input; }
287 // Does not take ownership.
288 void set_video_frame_allocator(FrameAllocator *allocator) override
290 video_frame_allocator = allocator;
291 if (owned_video_frame_allocator.get() != allocator) {
292 owned_video_frame_allocator.reset();
296 FrameAllocator *get_video_frame_allocator() override
298 return video_frame_allocator;
301 // Does not take ownership.
302 void set_audio_frame_allocator(FrameAllocator *allocator) override
304 audio_frame_allocator = allocator;
305 if (owned_audio_frame_allocator.get() != allocator) {
306 owned_audio_frame_allocator.reset();
310 FrameAllocator *get_audio_frame_allocator() override
312 return audio_frame_allocator;
315 void set_frame_callback(frame_callback_t callback) override
317 frame_callback = callback;
320 // Needs to be run before configure_card().
321 void set_dequeue_thread_callbacks(std::function<void()> init, std::function<void()> cleanup) override
323 dequeue_init_callback = init;
324 dequeue_cleanup_callback = cleanup;
325 has_dequeue_callbacks = true;
328 // Only valid after configure_card().
329 std::string get_description() const override {
333 void configure_card() override;
334 void start_bm_capture() override;
335 void stop_dequeue_thread() override;
336 bool get_disconnected() const override { return disconnected; }
338 // TODO: It's rather messy to have these outside the interface.
339 static void start_bm_thread();
340 static void stop_bm_thread();
342 // Hotplug event (for devices being inserted between start_bm_thread()
343 // and stop_bm_thread()); entirely optional, but must be set before
344 // start_bm_capture(). Note that your callback should do as little work
345 // as possible, since the callback comes from the main USB handling
346 // thread, which is very time-sensitive.
348 // The callback function transfers ownership. If you don't want to hold
349 // on to the device given to you in the callback, you need to call
350 // libusb_unref_device().
351 static void set_card_connected_callback(card_connected_callback_t callback,
352 bool hotplug_existing_devices_arg = false)
354 card_connected_callback = callback;
355 hotplug_existing_devices = hotplug_existing_devices_arg;
358 // Similar to set_card_connected_callback(), with the same caveats.
359 // (Note that this is set per-card and not global, as it is logically
360 // connected to an existing BMUSBCapture object.)
361 void set_card_disconnected_callback(card_disconnected_callback_t callback)
363 card_disconnected_callback = callback;
370 FrameAllocator::Frame frame;
373 void start_new_audio_block(const uint8_t *start);
374 void start_new_frame(const uint8_t *start);
376 void queue_frame(uint16_t format, uint16_t timecode, FrameAllocator::Frame frame, std::deque<QueuedFrame> *q);
377 void dequeue_thread_func();
379 static void usb_thread_func();
380 static void cb_xfr(struct libusb_transfer *xfr);
381 static int cb_hotplug(libusb_context *ctx, libusb_device *dev, libusb_hotplug_event event, void *user_data);
383 void update_capture_mode();
385 std::string description;
387 FrameAllocator::Frame current_video_frame;
388 FrameAllocator::Frame current_audio_frame;
390 std::mutex queue_lock;
391 std::condition_variable queues_not_empty;
392 std::deque<QueuedFrame> pending_video_frames;
393 std::deque<QueuedFrame> pending_audio_frames;
395 FrameAllocator *video_frame_allocator = nullptr;
396 FrameAllocator *audio_frame_allocator = nullptr;
397 std::unique_ptr<FrameAllocator> owned_video_frame_allocator;
398 std::unique_ptr<FrameAllocator> owned_audio_frame_allocator;
399 frame_callback_t frame_callback = nullptr;
400 static card_connected_callback_t card_connected_callback;
401 static bool hotplug_existing_devices;
402 card_disconnected_callback_t card_disconnected_callback = nullptr;
404 std::thread dequeue_thread;
405 std::atomic<bool> dequeue_thread_should_quit;
406 bool has_dequeue_callbacks = false;
407 std::function<void()> dequeue_init_callback = nullptr;
408 std::function<void()> dequeue_cleanup_callback = nullptr;
410 int current_register = 0;
412 static constexpr int NUM_BMUSB_REGISTERS = 60;
413 uint8_t register_file[NUM_BMUSB_REGISTERS];
415 // If <dev> is nullptr, will choose device number <card_index> from the list
416 // of available devices on the system. <dev> is not used after configure_card()
417 // (it will be unref-ed).
419 libusb_device *dev = nullptr;
421 std::vector<libusb_transfer *> iso_xfrs;
422 int assumed_frame_width = 1280;
424 libusb_device_handle *devh = nullptr;
425 uint32_t current_video_input = 0x00000000; // HDMI/SDI.
426 uint32_t current_audio_input = 0x00000000; // Embedded.
427 PixelFormat current_pixel_format = PixelFormat_8BitYCbCr;
429 bool disconnected = false;