1 #include "jpeg_frame_view.h"
8 #include <condition_variable>
14 #include <QMouseEvent>
17 #include <movit/resource_pool.h>
18 #include <movit/init.h>
19 #include <movit/util.h>
22 #include "post_to_main_thread.h"
23 #include "vaapi_jpeg_decoder.h"
24 #include "video_stream.h"
25 #include "ycbcr_converter.h"
27 using namespace movit;
30 // Just an arbitrary order for std::map.
31 struct JPEGIDLexicalOrder
33 bool operator() (const JPEGID &a, const JPEGID &b) const
35 if (a.stream_idx != b.stream_idx)
36 return a.stream_idx < b.stream_idx;
39 return a.interpolated < b.interpolated;
44 shared_ptr<Frame> frame;
48 thread JPEGFrameView::jpeg_decoder_thread;
50 map<JPEGID, LRUFrame, JPEGIDLexicalOrder> cache; // Under cache_mu.
51 condition_variable any_pending_decodes, cache_updated;
52 deque<pair<JPEGID, JPEGFrameView *>> pending_decodes; // Under cache_mu.
53 atomic<size_t> event_counter{0};
54 extern QGLWidget *global_share_widget;
55 extern atomic<bool> should_quit;
57 shared_ptr<Frame> decode_jpeg(const string &filename)
59 shared_ptr<Frame> frame;
60 if (vaapi_jpeg_decoding_usable) {
61 frame = decode_jpeg_vaapi(filename);
62 if (frame != nullptr) {
65 fprintf(stderr, "VA-API hardware decoding failed; falling back to software.\n");
68 frame.reset(new Frame);
70 jpeg_decompress_struct dinfo;
72 dinfo.err = jpeg_std_error(&jerr);
73 jpeg_create_decompress(&dinfo);
75 FILE *fp = fopen(filename.c_str(), "rb");
77 perror(filename.c_str());
80 jpeg_stdio_src(&dinfo, fp);
82 jpeg_read_header(&dinfo, true);
84 if (dinfo.num_components != 3) {
85 fprintf(stderr, "Not a color JPEG. (%d components, Y=%dx%d, Cb=%dx%d, Cr=%dx%d)\n",
87 dinfo.comp_info[0].h_samp_factor, dinfo.comp_info[0].v_samp_factor,
88 dinfo.comp_info[1].h_samp_factor, dinfo.comp_info[1].v_samp_factor,
89 dinfo.comp_info[2].h_samp_factor, dinfo.comp_info[2].v_samp_factor);
92 if (dinfo.comp_info[0].h_samp_factor != dinfo.max_h_samp_factor ||
93 dinfo.comp_info[0].v_samp_factor != dinfo.max_v_samp_factor || // Y' must not be subsampled.
94 dinfo.comp_info[1].h_samp_factor != dinfo.comp_info[2].h_samp_factor ||
95 dinfo.comp_info[1].v_samp_factor != dinfo.comp_info[2].v_samp_factor || // Cb and Cr must be identically subsampled.
96 (dinfo.max_h_samp_factor % dinfo.comp_info[1].h_samp_factor) != 0 ||
97 (dinfo.max_v_samp_factor % dinfo.comp_info[1].v_samp_factor) != 0) { // No 2:3 subsampling or other weirdness.
98 fprintf(stderr, "Unsupported subsampling scheme. (Y=%dx%d, Cb=%dx%d, Cr=%dx%d)\n",
99 dinfo.comp_info[0].h_samp_factor, dinfo.comp_info[0].v_samp_factor,
100 dinfo.comp_info[1].h_samp_factor, dinfo.comp_info[1].v_samp_factor,
101 dinfo.comp_info[2].h_samp_factor, dinfo.comp_info[2].v_samp_factor);
104 dinfo.raw_data_out = true;
106 jpeg_start_decompress(&dinfo);
108 frame->width = dinfo.output_width;
109 frame->height = dinfo.output_height;
110 frame->chroma_subsampling_x = dinfo.max_h_samp_factor / dinfo.comp_info[1].h_samp_factor;
111 frame->chroma_subsampling_y = dinfo.max_v_samp_factor / dinfo.comp_info[1].v_samp_factor;
113 unsigned h_mcu_size = DCTSIZE * dinfo.max_h_samp_factor;
114 unsigned v_mcu_size = DCTSIZE * dinfo.max_v_samp_factor;
115 unsigned mcu_width_blocks = (dinfo.output_width + h_mcu_size - 1) / h_mcu_size;
116 unsigned mcu_height_blocks = (dinfo.output_height + v_mcu_size - 1) / v_mcu_size;
118 unsigned luma_width_blocks = mcu_width_blocks * dinfo.comp_info[0].h_samp_factor;
119 unsigned chroma_width_blocks = mcu_width_blocks * dinfo.comp_info[1].h_samp_factor;
120 unsigned luma_height_blocks = mcu_height_blocks * dinfo.comp_info[0].v_samp_factor;
121 unsigned chroma_height_blocks = mcu_height_blocks * dinfo.comp_info[1].v_samp_factor;
123 // TODO: Decode into a PBO.
124 frame->y.reset(new uint8_t[luma_width_blocks * luma_height_blocks * DCTSIZE2]);
125 frame->cb.reset(new uint8_t[chroma_width_blocks * chroma_height_blocks * DCTSIZE2]);
126 frame->cr.reset(new uint8_t[chroma_width_blocks * chroma_height_blocks * DCTSIZE2]);
127 frame->pitch_y = luma_width_blocks * DCTSIZE;
128 frame->pitch_chroma = chroma_width_blocks * DCTSIZE;
130 JSAMPROW yptr[v_mcu_size], cbptr[v_mcu_size], crptr[v_mcu_size];
131 JSAMPARRAY data[3] = { yptr, cbptr, crptr };
132 for (unsigned y = 0; y < mcu_height_blocks; ++y) {
133 // NOTE: The last elements of cbptr/crptr will be unused for vertically subsampled chroma.
134 for (unsigned yy = 0; yy < v_mcu_size; ++yy) {
135 yptr[yy] = frame->y.get() + (y * DCTSIZE * dinfo.max_v_samp_factor + yy) * frame->pitch_y;
136 cbptr[yy] = frame->cb.get() + (y * DCTSIZE * dinfo.comp_info[1].v_samp_factor + yy) * frame->pitch_chroma;
137 crptr[yy] = frame->cr.get() + (y * DCTSIZE * dinfo.comp_info[1].v_samp_factor + yy) * frame->pitch_chroma;
140 jpeg_read_raw_data(&dinfo, data, v_mcu_size);
143 (void) jpeg_finish_decompress(&dinfo);
144 jpeg_destroy_decompress(&dinfo);
152 // Assumes cache_mu is held.
153 vector<size_t> lru_timestamps;
154 for (const auto &key_and_value : cache) {
155 lru_timestamps.push_back(key_and_value.second.last_used);
158 size_t cutoff_point = CACHE_SIZE / 10; // Prune away the 10% oldest ones.
159 nth_element(lru_timestamps.begin(), lru_timestamps.begin() + cutoff_point, lru_timestamps.end());
160 size_t must_be_used_after = lru_timestamps[cutoff_point];
161 for (auto it = cache.begin(); it != cache.end(); ) {
162 if (it->second.last_used < must_be_used_after) {
163 it = cache.erase(it);
170 shared_ptr<Frame> decode_jpeg_with_cache(JPEGID id, CacheMissBehavior cache_miss_behavior, bool *did_decode)
174 unique_lock<mutex> lock(cache_mu);
175 auto it = cache.find(id);
176 if (it != cache.end()) {
177 it->second.last_used = event_counter++;
178 return it->second.frame;
182 if (cache_miss_behavior == RETURN_NULLPTR_IF_NOT_IN_CACHE) {
186 assert(!id.interpolated);
188 shared_ptr<Frame> frame = decode_jpeg(filename_for_frame(id.stream_idx, id.pts));
190 unique_lock<mutex> lock(cache_mu);
191 cache[id] = LRUFrame{ frame, event_counter++ };
193 if (cache.size() > CACHE_SIZE) {
199 void jpeg_decoder_thread_func()
201 size_t num_decoded = 0, num_dropped = 0;
203 pthread_setname_np(pthread_self(), "JPEGDecoder");
204 while (!should_quit.load()) {
207 CacheMissBehavior cache_miss_behavior = DECODE_IF_NOT_IN_CACHE;
209 unique_lock<mutex> lock(cache_mu); // TODO: Perhaps under another lock?
210 any_pending_decodes.wait(lock, [] {
211 return !pending_decodes.empty() || should_quit.load();
213 if (should_quit.load()) break;
214 id = pending_decodes.front().first;
215 dest = pending_decodes.front().second;
216 pending_decodes.pop_front();
218 size_t num_pending = 0;
219 for (const pair<JPEGID, JPEGFrameView *> &decode : pending_decodes) {
220 if (decode.second == dest) {
224 if (num_pending > 3) {
225 cache_miss_behavior = RETURN_NULLPTR_IF_NOT_IN_CACHE;
230 shared_ptr<Frame> frame;
231 if (id.interpolated) {
232 // Interpolated frames are never decoded by us,
233 // put directly into the cache from VideoStream.
234 unique_lock<mutex> lock(cache_mu);
235 cache_updated.wait(lock, [id] {
236 return cache.count(id) != 0 || should_quit.load();
238 if (should_quit.load()) break;
239 found_in_cache = true; // Don't count it as a decode.
241 auto it = cache.find(id);
242 assert(it != cache.end());
244 it->second.last_used = event_counter++;
245 frame = it->second.frame;
246 if (frame == nullptr) {
247 // We inserted a nullptr as signal that the frame was never
248 // interpolated and that we should stop waiting.
249 // But don't let it linger in the cache anymore.
253 frame = decode_jpeg_with_cache(id, cache_miss_behavior, &found_in_cache);
256 if (frame == nullptr) {
257 assert(id.interpolated || cache_miss_behavior == RETURN_NULLPTR_IF_NOT_IN_CACHE);
262 if (!found_in_cache) {
264 if (num_decoded % 1000 == 0) {
265 fprintf(stderr, "Decoded %zu images, dropped %zu (%.2f%% dropped)\n",
266 num_decoded, num_dropped, (100.0 * num_dropped) / (num_decoded + num_dropped));
270 // TODO: Could we get jitter between non-interpolated and interpolated frames here?
271 dest->setDecodedFrame(frame);
275 void JPEGFrameView::shutdown()
277 any_pending_decodes.notify_all();
278 jpeg_decoder_thread.join();
281 JPEGFrameView::JPEGFrameView(QWidget *parent)
282 : QGLWidget(parent, global_share_widget) {
285 void JPEGFrameView::setFrame(unsigned stream_idx, int64_t pts, bool interpolated)
287 current_stream_idx = stream_idx;
289 unique_lock<mutex> lock(cache_mu);
290 pending_decodes.emplace_back(JPEGID{ stream_idx, pts, interpolated }, this);
291 any_pending_decodes.notify_all();
294 void JPEGFrameView::insert_interpolated_frame(unsigned stream_idx, int64_t pts, shared_ptr<Frame> frame)
296 JPEGID id{ stream_idx, pts, true };
298 // We rely on the frame not being evicted from the cache before
299 // jpeg_decoder_thread() sees it and can display it (otherwise,
300 // that thread would hang). With a default cache of 1000 elements,
301 // that would sound like a reasonable assumption.
302 unique_lock<mutex> lock(cache_mu);
303 cache[id] = LRUFrame{ std::move(frame), event_counter++ };
304 cache_updated.notify_all();
307 ResourcePool *resource_pool = nullptr;
309 void JPEGFrameView::initializeGL()
312 glDisable(GL_DEPTH_TEST);
315 static once_flag once;
317 resource_pool = new ResourcePool;
318 jpeg_decoder_thread = std::thread(jpeg_decoder_thread_func);
321 ycbcr_converter.reset(new YCbCrConverter(YCbCrConverter::OUTPUT_TO_RGBA, resource_pool));
323 ImageFormat inout_format;
324 inout_format.color_space = COLORSPACE_sRGB;
325 inout_format.gamma_curve = GAMMA_sRGB;
327 overlay_chain.reset(new EffectChain(overlay_base_width, overlay_base_height, resource_pool));
328 overlay_input = (movit::FlatInput *)overlay_chain->add_input(new FlatInput(inout_format, FORMAT_GRAYSCALE, GL_UNSIGNED_BYTE, overlay_base_width, overlay_base_height));
330 overlay_chain->add_output(inout_format, OUTPUT_ALPHA_FORMAT_POSTMULTIPLIED);
331 overlay_chain->finalize();
334 void JPEGFrameView::resizeGL(int width, int height)
337 glViewport(0, 0, width, height);
340 // Save these, as width() and height() will lie with DPI scaling.
345 void JPEGFrameView::paintGL()
347 glViewport(0, 0, gl_width, gl_height);
348 if (current_frame == nullptr) {
349 glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
350 glClear(GL_COLOR_BUFFER_BIT);
355 current_chain->render_to_screen();
357 if (overlay_image != nullptr) {
358 if (overlay_input_needs_refresh) {
359 overlay_input->set_width(overlay_width);
360 overlay_input->set_height(overlay_height);
361 overlay_input->set_pixel_data(overlay_image->bits());
363 glViewport(gl_width - overlay_width, 0, overlay_width, overlay_height);
364 overlay_chain->render_to_screen();
368 void JPEGFrameView::setDecodedFrame(std::shared_ptr<Frame> frame)
370 post_to_main_thread([this, frame] {
371 current_frame = frame;
372 current_chain = ycbcr_converter->prepare_chain_for_conversion(frame);
377 void JPEGFrameView::mousePressEvent(QMouseEvent *event)
379 if (event->type() == QEvent::MouseButtonPress && event->button() == Qt::LeftButton) {
384 void JPEGFrameView::set_overlay(const string &text)
387 overlay_image.reset();
391 float dpr = QGuiApplication::primaryScreen()->devicePixelRatio();
392 overlay_width = lrint(overlay_base_width * dpr);
393 overlay_height = lrint(overlay_base_height * dpr);
395 overlay_image.reset(new QImage(overlay_width, overlay_height, QImage::Format_Grayscale8));
396 overlay_image->setDevicePixelRatio(dpr);
397 overlay_image->fill(0);
398 QPainter painter(overlay_image.get());
400 painter.setPen(Qt::white);
401 QFont font = painter.font();
402 font.setPointSize(12);
403 painter.setFont(font);
405 painter.drawText(QRectF(0, 0, overlay_base_width, overlay_base_height), Qt::AlignCenter, QString::fromStdString(text));
407 // Don't refresh immediately; we might not have an OpenGL context here.
408 overlay_input_needs_refresh = true;