git.sesse.net Git - pkanalytics/blob - video_widget.cpp

   1 #define GL_GLEXT_PROTOTYPES
   2
   3 #include "video_widget.h"
   4
   5 #include <assert.h>
   6 #include <pthread.h>
   7 #include <stdint.h>
   8 #include <stdio.h>
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <sys/stat.h>
  12 #include <unistd.h>
  13
  14 extern "C" {
  15 #include <libavcodec/avcodec.h>
  16 #include <libavformat/avformat.h>
  17 #include <libavutil/avutil.h>
  18 #include <libavutil/error.h>
  19 #include <libavutil/frame.h>
  20 #include <libavutil/imgutils.h>
  21 #include <libavutil/mem.h>
  22 #include <libavutil/pixfmt.h>
  23 #include <libavutil/opt.h>
  24 #include <libswscale/swscale.h>
  25 }
  26
  27 #include <chrono>
  28 #include <cstdint>
  29 #include <utility>
  30 #include <vector>
  31 #include <unordered_set>
  32
  33 #include "post_to_main_thread.h"
  34
  35 #include <QOpenGLFunctions>
  36 #include <QWheelEvent>
  37 #include <QMouseEvent>
  38
  39 using namespace std;
  40 using namespace std::chrono;
  41
  42 namespace {
  43
  44 bool is_full_range(const AVPixFmtDescriptor *desc)
  45 {
  46         // This is horrible, but there's no better way that I know of.
  47         return (strchr(desc->name, 'j') != nullptr);
  48 }
  49
  50 AVPixelFormat decide_dst_format(AVPixelFormat src_format)
  51 {
  52         // If this is a non-Y'CbCr format, just convert to 4:4:4 Y'CbCr
  53         // and be done with it. It's too strange to spend a lot of time on.
  54         // (Let's hope there's no alpha.)
  55         const AVPixFmtDescriptor *src_desc = av_pix_fmt_desc_get(src_format);
  56         if (src_desc == nullptr ||
  57             src_desc->nb_components != 3 ||
  58             (src_desc->flags & AV_PIX_FMT_FLAG_RGB)) {
  59                 return AV_PIX_FMT_YUV444P;
  60         }
  61
  62         // The best for us would be Cb and Cr together if possible,
  63         // but FFmpeg doesn't support that except in the special case of
  64         // NV12, so we need to go to planar even for the case of NV12.
  65         // Thus, look for the closest (but no worse) 8-bit planar Y'CbCr format
  66         // that matches in color range. (This will also include the case of
  67         // the source format already being acceptable.)
  68         bool src_full_range = is_full_range(src_desc);
  69         const char *best_format = "yuv444p";
  70         unsigned best_score = numeric_limits<unsigned>::max();
  71         for (const AVPixFmtDescriptor *desc = av_pix_fmt_desc_next(nullptr);
  72              desc;
  73              desc = av_pix_fmt_desc_next(desc)) {
  74                 // Find planar Y'CbCr formats only.
  75                 if (desc->nb_components != 3) continue;
  76                 if (desc->flags & AV_PIX_FMT_FLAG_RGB) continue;
  77                 if (!(desc->flags & AV_PIX_FMT_FLAG_PLANAR)) continue;
  78                 if (desc->comp[0].plane != 0 ||
  79                     desc->comp[1].plane != 1 ||
  80                     desc->comp[2].plane != 2) continue;
  81
  82                 // 8-bit formats only.
  83                 if (desc->flags & AV_PIX_FMT_FLAG_BE) continue;
  84                 if (desc->comp[0].depth != 8) continue;
  85
  86                 // Same or better chroma resolution only.
  87                 int chroma_w_diff = src_desc->log2_chroma_w - desc->log2_chroma_w;
  88                 int chroma_h_diff = src_desc->log2_chroma_h - desc->log2_chroma_h;
  89                 if (chroma_w_diff < 0 || chroma_h_diff < 0)
  90                         continue;
  91
  92                 // Matching full/limited range only.
  93                 if (is_full_range(desc) != src_full_range)
  94                         continue;
  95
  96                 // Pick something with as little excess chroma resolution as possible.
  97                 unsigned score = (1 << (chroma_w_diff)) << chroma_h_diff;
  98                 if (score < best_score) {
  99                         best_score = score;
 100                         best_format = desc->name;
 101                 }
 102         }
 103         return av_get_pix_fmt(best_format);
 104 }
 105
 106 }  // namespace
 107
 108 bool VideoWidget::process_queued_commands(AVFormatContext *format_ctx, AVCodecContext *video_codec_ctx, int video_stream_index, bool *seeked)
 109 {
 110         // Process any queued commands from other threads.
 111         vector<QueuedCommand> commands;
 112         {
 113                 lock_guard<mutex> lock(queue_mu);
 114                 swap(commands, command_queue);
 115         }
 116
 117         for (const QueuedCommand &cmd : commands) {
 118                 switch (cmd.command) {
 119                 case QueuedCommand::PAUSE:
 120                         paused = true;
 121                         break;
 122                 case QueuedCommand::RESUME:
 123                         paused = false;
 124                         pts_origin = last_pts;
 125                         start = next_frame_start = steady_clock::now();
 126                         break;
 127                 case QueuedCommand::SEEK:
 128                 case QueuedCommand::SEEK_ABSOLUTE:
 129                         // Dealt with below.
 130                         break;
 131                 }
 132         }
 133
 134         // Combine all seeks into one big one. (There are edge cases where this is probably
 135         // subtly wrong, but we'll live with it.)
 136         int64_t base_pts = last_pts;
 137         int64_t relative_seek_ms = 0;
 138         int64_t relative_seek_frames = 0;
 139         for (const QueuedCommand &cmd : commands) {
 140                 if (cmd.command == QueuedCommand::SEEK) {
 141                         relative_seek_ms += cmd.relative_seek_ms;
 142                         relative_seek_frames += cmd.relative_seek_frames;
 143                 } else if (cmd.command == QueuedCommand::SEEK_ABSOLUTE) {
 144                         base_pts = av_rescale_q(cmd.seek_ms, AVRational{ 1, 1000 }, video_timebase);
 145                         relative_seek_ms = 0;
 146                         relative_seek_frames = 0;
 147                 }
 148         }
 149         int64_t relative_seek_pts = av_rescale_q(relative_seek_ms, AVRational{ 1, 1000 }, video_timebase);
 150         if (relative_seek_ms != 0 && relative_seek_pts == 0) {
 151                 // Just to be sure rounding errors don't move us into nothingness.
 152                 relative_seek_pts = (relative_seek_ms > 0) ? 1 : -1;
 153         }
 154         int64_t goal_pts = base_pts + relative_seek_pts;
 155         if (goal_pts != last_pts || relative_seek_frames < 0) {
 156                 avcodec_flush_buffers(video_codec_ctx);
 157                 queued_frames.clear();
 158
 159                 // Seek to the last keyframe before this point.
 160                 int64_t seek_pts = goal_pts;
 161                 if (relative_seek_frames < 0) {
 162                         // If we're frame-skipping backwards, add 100 ms of slop for each frame
 163                         // so we're fairly certain we are able to see the ones we want.
 164                         seek_pts -= av_rescale_q(-relative_seek_frames, AVRational{ 1, 10 }, video_timebase);
 165                 }
 166                 av_seek_frame(format_ctx, video_stream_index, seek_pts, AVSEEK_FLAG_BACKWARD);
 167
 168                 // Decode frames until EOF, or until we see something past our seek point.
 169                 std::deque<AVFrameWithDeleter> queue;
 170                 for ( ;; ) {
 171                         bool error = false;
 172                         AVFrameWithDeleter frame = decode_frame(format_ctx, video_codec_ctx,
 173                                 pathname, video_stream_index, &error);
 174                         if (frame == nullptr || error) {
 175                                 break;
 176                         }
 177
 178                         int64_t frame_pts = frame->pts;
 179                         if (relative_seek_frames < 0) {
 180                                 // Buffer this frame; don't display it unless we know it's the Nth-latest.
 181                                 queue.push_back(std::move(frame));
 182                                 if (queue.size() > uint64_t(-relative_seek_frames) + 1) {
 183                                         queue.pop_front();
 184                                 }
 185                         }
 186                         if (frame_pts >= goal_pts) {
 187                                 if (relative_seek_frames > 0) {
 188                                         --relative_seek_frames;
 189                                 } else {
 190                                         if (relative_seek_frames < 0) {
 191                                                 // Hope we have the right amount.
 192                                                 // The rest will remain in the queue for when we play forward again.
 193                                                 frame = std::move(queue.front());
 194                                                 queue.pop_front();
 195                                                 queued_frames = std::move(queue);
 196                                         }
 197                                         {
 198                                                 lock_guard lock(current_frame_mu);
 199                                                 current_frame.reset(new Frame(make_video_frame(frame.get())));
 200                                         }
 201                                         update();
 202                                         store_pts(frame->pts);
 203                                         break;
 204                                 }
 205                         }
 206                 }
 207
 208                 // NOTE: We keep pause status as-is.
 209
 210                 pts_origin = last_pts;
 211                 start = next_frame_start = last_frame = steady_clock::now();
 212                 if (seeked) {
 213                         *seeked = true;
 214                 }
 215         } else if (relative_seek_frames > 0) {
 216                 // The base PTS is fine, we only need to skip a few frames forwards.
 217                 while (relative_seek_frames > 1) {
 218                         // Eat a frame (ignore errors).
 219                         bool error;
 220                         decode_frame(format_ctx, video_codec_ctx, pathname, video_stream_index, &error);
 221                         --relative_seek_frames;
 222                 }
 223
 224                 // Display the last one.
 225                 bool error;
 226                 AVFrameWithDeleter frame = decode_frame(format_ctx, video_codec_ctx,
 227                         pathname, video_stream_index, &error);
 228                 if (frame == nullptr || error) {
 229                         return true;
 230                 }
 231                 {
 232                         lock_guard lock(current_frame_mu);
 233                         current_frame.reset(new Frame(make_video_frame(frame.get())));
 234                 }
 235                 update();
 236                 store_pts(frame->pts);
 237         }
 238         return false;
 239 }
 240
 241 VideoWidget::VideoWidget(QWidget *parent)
 242         : QOpenGLWidget(parent) {}
 243
 244 GLuint compile_shader(const string &shader_src, GLenum type)
 245 {
 246         GLuint obj = glCreateShader(type);
 247         const GLchar* source[] = { shader_src.data() };
 248         const GLint length[] = { (GLint)shader_src.size() };
 249         glShaderSource(obj, 1, source, length);
 250         glCompileShader(obj);
 251
 252         GLchar info_log[4096];
 253         GLsizei log_length = sizeof(info_log) - 1;
 254         glGetShaderInfoLog(obj, log_length, &log_length, info_log);
 255         info_log[log_length] = 0;
 256         if (strlen(info_log) > 0) {
 257                 fprintf(stderr, "Shader compile log: %s\n", info_log);
 258         }
 259
 260         GLint status;
 261         glGetShaderiv(obj, GL_COMPILE_STATUS, &status);
 262         if (status == GL_FALSE) {
 263                 // Add some line numbers to easier identify compile errors.
 264                 string src_with_lines = "/*   1 */ ";
 265                 size_t lineno = 1;
 266                 for (char ch : shader_src) {
 267                         src_with_lines.push_back(ch);
 268                         if (ch == '\n') {
 269                                 char buf[32];
 270                                 snprintf(buf, sizeof(buf), "/* %3zu */ ", ++lineno);
 271                                 src_with_lines += buf;
 272                         }
 273                 }
 274
 275                 fprintf(stderr, "Failed to compile shader:\n%s\n", src_with_lines.c_str());
 276                 exit(1);
 277         }
 278
 279         return obj;
 280 }
 281
 282 void VideoWidget::initializeGL()
 283 {
 284         glDisable(GL_BLEND);
 285         glDisable(GL_DEPTH_TEST);
 286         glDepthMask(GL_FALSE);
 287         glCreateTextures(GL_TEXTURE_2D, 3, tex);
 288
 289         ycbcr_vertex_shader = compile_shader(R"(
 290 #version 460 core
 291
 292 layout(location = 0) in vec2 position;
 293 layout(location = 1) in vec2 texcoord;
 294 out vec2 tc;
 295
 296 void main()
 297 {
 298         // The result of glOrtho(0.0, 1.0, 0.0, 1.0, 0.0, 1.0) is:
 299         //
 300         //   2.000  0.000  0.000 -1.000
 301         //   0.000  2.000  0.000 -1.000
 302         //   0.000  0.000 -2.000 -1.000
 303         //   0.000  0.000  0.000  1.000
 304         gl_Position = vec4(2.0 * position.x - 1.0, 2.0 * position.y - 1.0, -1.0, 1.0);
 305         tc = texcoord;
 306         tc.y = 1.0f - tc.y;
 307 }
 308 )", GL_VERTEX_SHADER);
 309         ycbcr_fragment_shader = compile_shader(R"(
 310 #version 460 core
 311
 312 layout(location = 0) uniform sampler2D tex_y;
 313 layout(location = 1) uniform sampler2D tex_cb;
 314 layout(location = 2) uniform sampler2D tex_cr;
 315 layout(location = 3) uniform vec2 cbcr_offset;
 316
 317 in vec2 tc;
 318 out vec4 FragColor;
 319
 320 // Computed statically by Movit, for limited-range BT.709.
 321 // (We don't check whether the input could be BT.601 or BT.2020 currently, or full-range)
 322 const mat3 inv_ycbcr_matrix = mat3(
 323         1.16438f, 1.16438f, 1.16438f,
 324         0.0f, -0.21325f, 2.11240f,
 325         1.79274f, -0.53291f, 0.0f
 326 );
 327
 328 void main()
 329 {
 330         if (tc.x < 0.0 || tc.x > 1.0 || tc.y < 0.0 || tc.y > 1.0) {
 331                 FragColor.rgba = vec4(0.0f, 0.0f, 0.0f, 1.0f);
 332                 return;
 333         }
 334
 335         vec3 ycbcr;
 336         ycbcr.r = texture(tex_y, tc).r;
 337         ycbcr.g = texture(tex_cb, tc + cbcr_offset).r;
 338         ycbcr.b = texture(tex_cr, tc + cbcr_offset).r;
 339         ycbcr -= vec3(16.0f / 255.0f, 128.0f / 255.0f, 128.0f / 255.0f);
 340         FragColor.rgb = inv_ycbcr_matrix * ycbcr;
 341         FragColor.a = 1.0f;
 342 }
 343 )", GL_FRAGMENT_SHADER);
 344         ycbcr_program = glCreateProgram();
 345         glAttachShader(ycbcr_program, ycbcr_vertex_shader);
 346         glAttachShader(ycbcr_program, ycbcr_fragment_shader);
 347         glLinkProgram(ycbcr_program);
 348
 349         GLint success;
 350         glGetProgramiv(ycbcr_program, GL_LINK_STATUS, &success);
 351         if (success == GL_FALSE) {
 352                 GLchar error_log[1024] = {0};
 353                 glGetProgramInfoLog(ycbcr_program, 1024, nullptr, error_log);
 354                 fprintf(stderr, "Error linking program: %s\n", error_log);
 355                 exit(1);
 356         }
 357
 358         glCreateSamplers(1, &bilinear_sampler);
 359         glSamplerParameteri(bilinear_sampler, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_NEAREST);
 360         glSamplerParameteri(bilinear_sampler, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
 361         glSamplerParameteri(bilinear_sampler, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
 362         glSamplerParameteri(bilinear_sampler, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
 363 }
 364
 365 void VideoWidget::resizeGL(int w, int h)
 366 {
 367         glViewport(0, 0, w, h);
 368         display_aspect = double(w) / h;
 369 }
 370
 371 int num_levels(GLuint width, GLuint height)
 372 {
 373         int levels = 1;
 374         while (width > 1 || height > 1) {
 375                 width = max(width / 2, 1u);
 376                 height = max(height / 2, 1u);
 377                 ++levels;
 378         }
 379         return levels;
 380 }
 381
 382 void VideoWidget::paintGL()
 383 {
 384         std::shared_ptr<Frame> frame;
 385         {
 386                 lock_guard lock(current_frame_mu);
 387                 frame = current_frame;
 388         }
 389         if (frame == nullptr) {
 390                 glClear(GL_COLOR_BUFFER_BIT);
 391                 return;
 392         }
 393
 394         glUseProgram(ycbcr_program);
 395         if (frame->width != last_width || frame->height != last_height) {
 396                 glTextureStorage2D(tex[0], num_levels(frame->width, frame->height), GL_R8, frame->width, frame->height);
 397         }
 398         if (frame->chroma_width != last_chroma_width || frame->chroma_height != last_chroma_height) {
 399                 for (GLuint num : { tex[1], tex[2] }) {
 400                         glTextureStorage2D(num, num_levels(frame->chroma_width, frame->chroma_height), GL_R8, frame->chroma_width, frame->chroma_height);
 401                 }
 402         }
 403
 404         glTextureSubImage2D(tex[0], 0, 0, 0, frame->width, frame->height, GL_RED, GL_UNSIGNED_BYTE, frame->data.get());
 405         glGenerateTextureMipmap(tex[0]);
 406
 407         glTextureSubImage2D(tex[1], 0, 0, 0, frame->chroma_width, frame->chroma_height, GL_RED, GL_UNSIGNED_BYTE, frame->data.get() + frame->width * frame->height);
 408         glGenerateTextureMipmap(tex[1]);
 409
 410         glTextureSubImage2D(tex[2], 0, 0, 0, frame->chroma_width, frame->chroma_height, GL_RED, GL_UNSIGNED_BYTE, frame->data.get() + frame->width * frame->height + frame->chroma_width * frame->chroma_height);
 411         glGenerateTextureMipmap(tex[2]);
 412
 413         glBindTextureUnit(0, tex[0]);
 414         glBindTextureUnit(1, tex[1]);
 415         glBindTextureUnit(2, tex[2]);
 416         glBindSampler(0, bilinear_sampler);
 417         glBindSampler(1, bilinear_sampler);
 418         glBindSampler(2, bilinear_sampler);
 419         glProgramUniform1i(ycbcr_program, 0, 0);
 420         glProgramUniform1i(ycbcr_program, 1, 1);
 421         glProgramUniform1i(ycbcr_program, 2, 2);
 422         glProgramUniform2f(ycbcr_program, 3, cbcr_offset[0], -cbcr_offset[1]);
 423
 424         float tx1 = 0.0f;
 425         float tx2 = 1.0f;
 426         float ty1 = 0.0f;
 427         float ty2 = 1.0f;
 428
 429         double video_aspect = double(frame->width) / frame->height;
 430         if (display_aspect > video_aspect) {
 431                 double extra_width = frame->height * display_aspect - frame->width;
 432                 tx1 = -0.5 * extra_width / frame->width;
 433                 tx2 = 1.0 + 0.5 * extra_width / frame->width;
 434         } else if (display_aspect < video_aspect) {
 435                 double extra_height = frame->width / display_aspect - frame->height;
 436                 ty1 = -0.5 * extra_height / frame->height;
 437                 ty2 = 1.0 + 0.5 * extra_height / frame->height;
 438         }
 439
 440         glBegin(GL_QUADS);
 441
 442         // (0,0)
 443         glVertexAttrib2f(1, tx1, ty1);
 444         glVertex2f(zoom_matrix[2 * 3 + 0], zoom_matrix[2 * 3 + 1]);
 445
 446         // (0,1)
 447         glVertexAttrib2f(1, tx1, ty2);
 448         glVertex2f(zoom_matrix[1 * 3 + 0] + zoom_matrix[2 * 3 + 0], zoom_matrix[1 * 3 + 1] + zoom_matrix[2 * 3 + 1]);
 449
 450         // (1,1)
 451         glVertexAttrib2f(1, tx2, ty2);
 452         glVertex2f(zoom_matrix[0 * 3 + 0] + zoom_matrix[1 * 3 + 0] + zoom_matrix[2 * 3 + 0],
 453                    zoom_matrix[1 * 3 + 0] + zoom_matrix[1 * 3 + 1] + zoom_matrix[2 * 3 + 1]);
 454
 455         // (1,0)
 456         glVertexAttrib2f(1, tx2, ty1);
 457         glVertex2f(zoom_matrix[0 * 3 + 0] + zoom_matrix[2 * 3 + 0],
 458                    zoom_matrix[1 * 3 + 0] + zoom_matrix[2 * 3 + 1]);
 459
 460         glEnd();
 461 }
 462
 463 void matmul3x3(const double a[9], const double b[9], double res[9])
 464 {
 465         for (int i = 0; i < 3; ++i) {
 466                 for (int j = 0; j < 3; ++j) {
 467                         double sum = 0.0;
 468                         for (int k = 0; k < 3; ++k) {
 469                                 sum += a[i * 3 + k] * b[k * 3 + j];
 470                         }
 471                         res[i * 3 + j] = sum;
 472                 }
 473         }
 474 }
 475
 476 void VideoWidget::wheelEvent(QWheelEvent *event)
 477 {
 478         int delta = event->angleDelta().y();
 479         if (delta == 0) {
 480                 return;
 481         }
 482         double x = event->position().x() / width();
 483         double y = 1.0 - event->position().y() / height();
 484         double zoom = delta > 0 ? pow(1.005, delta) : pow(1/1.005, -delta);
 485
 486         const double inv_translation_matrix[9] = {
 487                 1.0, 0.0, 0.0,
 488                 0.0, 1.0, 0.0,
 489                 -x, -y, 1.0
 490         };
 491         const double scale_matrix[9] = {
 492                 zoom, 0.0, 0.0,
 493                 0.0, zoom, 0.0,
 494                 0.0, 0.0, 1.0
 495         };
 496         const double translation_matrix[9] = {
 497                 1.0, 0.0, 0.0,
 498                 0.0, 1.0, 0.0,
 499                 x, y, 1.0
 500         };
 501         double tmp1[9], tmp2[9];
 502         matmul3x3(zoom_matrix, inv_translation_matrix, tmp1);
 503         matmul3x3(tmp1, scale_matrix, tmp2);
 504         matmul3x3(tmp2, translation_matrix, zoom_matrix);
 505
 506         fixup_zoom_matrix();
 507         update();
 508 }
 509
 510 void VideoWidget::mousePressEvent(QMouseEvent *e)
 511 {
 512         if (e->button() == Qt::BackButton) {
 513                 emit mouse_back_clicked();
 514         } else if (e->button() == Qt::ForwardButton) {
 515                 emit mouse_forward_clicked();
 516         } else if (e->button() == Qt::LeftButton) {
 517                 dragging = true;
 518                 last_drag_x = e->position().x();
 519                 last_drag_y = e->position().y();
 520         }
 521 }
 522
 523 void VideoWidget::mouseReleaseEvent(QMouseEvent *e)
 524 {
 525         if (e->button() == Qt::LeftButton) {
 526                 dragging = false;
 527         }
 528 }
 529
 530 void VideoWidget::mouseMoveEvent(QMouseEvent *e)
 531 {
 532         if (!dragging) {
 533                 return;
 534         }
 535         float dx = (e->position().x() - last_drag_x) / width();
 536         float dy = (e->position().y() - last_drag_y) / height();
 537
 538         //zoom_matrix[6] += dx * zoom_matrix[0];
 539         //zoom_matrix[7] += dy * zoom_matrix[4];
 540         zoom_matrix[6] += dx;
 541         zoom_matrix[7] -= dy;
 542         fixup_zoom_matrix();
 543
 544         last_drag_x = e->position().x();
 545         last_drag_y = e->position().y();
 546
 547         update();
 548 }
 549
 550 // Normalize the matrix so that we never get skew or similar,
 551 // and also never can zoom or pan too far out.
 552 void VideoWidget::fixup_zoom_matrix()
 553 {
 554         // Correct for any numerical errors (we know the matrix must be orthogonal
 555         // and have zero rotation).
 556         zoom_matrix[4] = zoom_matrix[0];
 557         zoom_matrix[1] = zoom_matrix[2] = zoom_matrix[3] = zoom_matrix[5] = 0.0;
 558         zoom_matrix[8] = 1.0;
 559
 560         // We can't zoom further out than 1:1. (Perhaps it would be nice to
 561         // reuse the last zoom-in point to do this, but the center will have to do
 562         // for now.)
 563         if (zoom_matrix[0] < 1.0) {
 564                 const double zoom = 1.0 / zoom_matrix[0];
 565                 const double inv_translation_matrix[9] = {
 566                         1.0, 0.0, 0.0,
 567                         0.0, 1.0, 0.0,
 568                         -0.5, -0.5, 1.0
 569                 };
 570                 const double scale_matrix[9] = {
 571                         zoom, 0.0, 0.0,
 572                         0.0, zoom, 0.0,
 573                         0.0, 0.0, 1.0
 574                 };
 575                 const double translation_matrix[9] = {
 576                         1.0, 0.0, 0.0,
 577                         0.0, 1.0, 0.0,
 578                         0.5, 0.5, 1.0
 579                 };
 580                 double tmp1[9], tmp2[9];
 581                 matmul3x3(zoom_matrix, inv_translation_matrix, tmp1);
 582                 matmul3x3(tmp1, scale_matrix, tmp2);
 583                 matmul3x3(tmp2, translation_matrix, zoom_matrix);
 584         }
 585
 586         // Looking at the points we'll draw with glVertex2f(), make sure none of them are
 587         // inside the square (which would generally mean we've panned ourselves out-of-bounds).
 588         // We simply adjust the translation, which is possible because we fixed scaling above.
 589         zoom_matrix[6] = min(zoom_matrix[6], 0.0);  // Left side (x=0).
 590         zoom_matrix[7] = min(zoom_matrix[7], 0.0);  // Bottom side (y=0).
 591         zoom_matrix[6] = std::max(zoom_matrix[6], 1.0 - zoom_matrix[0]);  // Right side (x=1).
 592         zoom_matrix[7] = std::max(zoom_matrix[7], 1.0 - zoom_matrix[4]);  // Top side (y=1).
 593 }
 594
 595 void VideoWidget::open(const string &filename)
 596 {
 597         stop();
 598         internal_rewind();
 599         pathname = filename;
 600         play();
 601 }
 602
 603 void VideoWidget::play()
 604 {
 605         if (running) {
 606                 std::lock_guard<std::mutex> lock(queue_mu);
 607                 command_queue.push_back(QueuedCommand { QueuedCommand::RESUME });
 608                 producer_thread_should_quit.wakeup();
 609                 return;
 610         }
 611         running = true;
 612         producer_thread_should_quit.unquit();
 613         producer_thread = std::thread(&VideoWidget::producer_thread_func, this);
 614 }
 615
 616 void VideoWidget::pause()
 617 {
 618         if (!running) {
 619                 return;
 620         }
 621         std::lock_guard<std::mutex> lock(queue_mu);
 622         command_queue.push_back(QueuedCommand { QueuedCommand::PAUSE });
 623         producer_thread_should_quit.wakeup();
 624 }
 625
 626 void VideoWidget::seek(int64_t relative_seek_ms)
 627 {
 628         if (!running) {
 629                 return;
 630         }
 631         std::lock_guard<std::mutex> lock(queue_mu);
 632         command_queue.push_back(QueuedCommand { QueuedCommand::SEEK, relative_seek_ms, 0, 0 });
 633         producer_thread_should_quit.wakeup();
 634 }
 635
 636 void VideoWidget::seek_frames(int64_t relative_seek_frames)
 637 {
 638         if (!running) {
 639                 return;
 640         }
 641         std::lock_guard<std::mutex> lock(queue_mu);
 642         command_queue.push_back(QueuedCommand { QueuedCommand::SEEK, 0, relative_seek_frames, 0 });
 643         producer_thread_should_quit.wakeup();
 644 }
 645
 646 void VideoWidget::seek_absolute(int64_t position_ms)
 647 {
 648         if (!running) {
 649                 return;
 650         }
 651         std::lock_guard<std::mutex> lock(queue_mu);
 652         command_queue.push_back(QueuedCommand { QueuedCommand::SEEK_ABSOLUTE, 0, 0, position_ms });
 653         producer_thread_should_quit.wakeup();
 654 }
 655
 656 void VideoWidget::stop()
 657 {
 658         if (!running) {
 659                 return;
 660         }
 661         running = false;
 662         producer_thread_should_quit.quit();
 663         producer_thread.join();
 664 }
 665
 666 void VideoWidget::producer_thread_func()
 667 {
 668         if (!producer_thread_should_quit.should_quit()) {
 669                 if (!play_video(pathname)) {
 670                         // TODO: Send the error back to the UI somehow.
 671                 }
 672         }
 673 }
 674
 675 void VideoWidget::internal_rewind()
 676 {
 677         pts_origin = last_pts = 0;
 678         last_position = 0;
 679         start = next_frame_start = steady_clock::now();
 680 }
 681
 682 template<AVHWDeviceType type>
 683 AVPixelFormat get_hw_format(AVCodecContext *ctx, const AVPixelFormat *fmt)
 684 {
 685         bool found_config_of_right_type = false;
 686         for (int i = 0;; ++i) {  // Termination condition inside loop.
 687                 const AVCodecHWConfig *config = avcodec_get_hw_config(ctx->codec, i);
 688                 if (config == nullptr) {  // End of list.
 689                         break;
 690                 }
 691                 if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) ||
 692                     config->device_type != type) {
 693                         // Not interesting for us.
 694                         continue;
 695                 }
 696
 697                 // We have a config of the right type, but does it actually support
 698                 // the pixel format we want? (Seemingly, FFmpeg's way of signaling errors
 699                 // is to just replace the pixel format with a software-decoded one,
 700                 // such as yuv420p.)
 701                 found_config_of_right_type = true;
 702                 for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
 703                         if (config->pix_fmt == *fmt_ptr) {
 704                                 fprintf(stderr, "Initialized '%s' hardware decoding for codec '%s'.\n",
 705                                         av_hwdevice_get_type_name(type), ctx->codec->name);
 706                                 if (ctx->profile == FF_PROFILE_H264_BASELINE) {
 707                                         fprintf(stderr, "WARNING: Stream claims to be H.264 Baseline, which is generally poorly supported in hardware decoders.\n");
 708                                         fprintf(stderr, "         Consider encoding it as Constrained Baseline, Main or High instead.\n");
 709                                         fprintf(stderr, "         Decoding might fail and fall back to software.\n");
 710                                 }
 711                                 return config->pix_fmt;
 712                         }
 713                 }
 714                 fprintf(stderr, "Decoder '%s' supports only these pixel formats:", ctx->codec->name);
 715                 unordered_set<AVPixelFormat> seen;
 716                 for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
 717                         if (!seen.count(*fmt_ptr)) {
 718                                 fprintf(stderr, " %s", av_get_pix_fmt_name(*fmt_ptr));
 719                                 seen.insert(*fmt_ptr);
 720                         }
 721                 }
 722                 fprintf(stderr, " (wanted %s for hardware acceleration)\n", av_get_pix_fmt_name(config->pix_fmt));
 723
 724         }
 725
 726         if (!found_config_of_right_type) {
 727                 fprintf(stderr, "Decoder '%s' does not support device type '%s'.\n", ctx->codec->name, av_hwdevice_get_type_name(type));
 728         }
 729
 730         // We found no VA-API formats, so take the first software format.
 731         for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
 732                 if ((av_pix_fmt_desc_get(*fmt_ptr)->flags & AV_PIX_FMT_FLAG_HWACCEL) == 0) {
 733                         fprintf(stderr, "Falling back to software format %s.\n", av_get_pix_fmt_name(*fmt_ptr));
 734                         return *fmt_ptr;
 735                 }
 736         }
 737
 738         // Fallback: Just return anything. (Should never really happen.)
 739         return fmt[0];
 740 }
 741
 742 AVFrameWithDeleter VideoWidget::decode_frame(AVFormatContext *format_ctx, AVCodecContext *video_codec_ctx,
 743         const std::string &pathname, int video_stream_index,
 744         bool *error)
 745 {
 746         *error = false;
 747
 748         if (!queued_frames.empty()) {
 749                 AVFrameWithDeleter frame = std::move(queued_frames.front());
 750                 queued_frames.pop_front();
 751                 return frame;
 752         }
 753
 754         // Read packets until we have a frame or there are none left.
 755         bool frame_finished = false;
 756         AVFrameWithDeleter video_avframe = av_frame_alloc_unique();
 757         bool eof = false;
 758         do {
 759                 AVPacket *pkt = av_packet_alloc();
 760                 unique_ptr<AVPacket, decltype(av_packet_unref)*> pkt_cleanup(
 761                         pkt, av_packet_unref);
 762                 pkt->data = nullptr;
 763                 pkt->size = 0;
 764                 if (av_read_frame(format_ctx, pkt) == 0) {
 765                         if (pkt->stream_index == video_stream_index) {
 766                                 if (avcodec_send_packet(video_codec_ctx, pkt) < 0) {
 767                                         fprintf(stderr, "%s: Cannot send packet to video codec.\n", pathname.c_str());
 768                                         *error = true;
 769                                         return AVFrameWithDeleter(nullptr);
 770                                 }
 771                         }
 772                 } else {
 773                         eof = true;  // Or error, but ignore that for the time being.
 774                 }
 775
 776                 // Decode video, if we have a frame.
 777                 int err = avcodec_receive_frame(video_codec_ctx, video_avframe.get());
 778                 if (err == 0) {
 779                         frame_finished = true;
 780                         break;
 781                 } else if (err != AVERROR(EAGAIN)) {
 782                         fprintf(stderr, "%s: Cannot receive frame from video codec.\n", pathname.c_str());
 783                         *error = true;
 784                         return AVFrameWithDeleter(nullptr);
 785                 }
 786         } while (!eof);
 787
 788         if (frame_finished)
 789                 return video_avframe;
 790         else
 791                 return AVFrameWithDeleter(nullptr);
 792 }
 793
 794 int find_stream_index(AVFormatContext *ctx, AVMediaType media_type)
 795 {
 796         for (unsigned i = 0; i < ctx->nb_streams; ++i) {
 797                 if (ctx->streams[i]->codecpar->codec_type == media_type) {
 798                         return i;
 799                 }
 800         }
 801         return -1;
 802 }
 803
 804 steady_clock::time_point compute_frame_start(int64_t frame_pts, int64_t pts_origin, const AVRational &video_timebase, const steady_clock::time_point &origin, double rate)
 805 {
 806         const duration<double> pts((frame_pts - pts_origin) * double(video_timebase.num) / double(video_timebase.den));
 807         return origin + duration_cast<steady_clock::duration>(pts / rate);
 808 }
 809
 810 bool VideoWidget::play_video(const string &pathname)
 811 {
 812         queued_frames.clear();
 813         AVFormatContextWithCloser format_ctx = avformat_open_input_unique(pathname.c_str(), /*fmt=*/nullptr,
 814                 /*options=*/nullptr);
 815         if (format_ctx == nullptr) {
 816                 fprintf(stderr, "%s: Error opening file\n", pathname.c_str());
 817                 return false;
 818         }
 819
 820         if (avformat_find_stream_info(format_ctx.get(), nullptr) < 0) {
 821                 fprintf(stderr, "%s: Error finding stream info\n", pathname.c_str());
 822                 return false;
 823         }
 824
 825         int video_stream_index = find_stream_index(format_ctx.get(), AVMEDIA_TYPE_VIDEO);
 826         if (video_stream_index == -1) {
 827                 fprintf(stderr, "%s: No video stream found\n", pathname.c_str());
 828                 return false;
 829         }
 830
 831         // Open video decoder.
 832         const AVCodecParameters *video_codecpar = format_ctx->streams[video_stream_index]->codecpar;
 833         const AVCodec *video_codec = avcodec_find_decoder(video_codecpar->codec_id);
 834
 835         video_timebase = format_ctx->streams[video_stream_index]->time_base;
 836         AVCodecContextWithDeleter video_codec_ctx = avcodec_alloc_context3_unique(nullptr);
 837         if (avcodec_parameters_to_context(video_codec_ctx.get(), video_codecpar) < 0) {
 838                 fprintf(stderr, "%s: Cannot fill video codec parameters\n", pathname.c_str());
 839                 return false;
 840         }
 841         if (video_codec == nullptr) {
 842                 fprintf(stderr, "%s: Cannot find video decoder\n", pathname.c_str());
 843                 return false;
 844         }
 845
 846         // Seemingly, it's not too easy to make something that just initializes
 847         // “whatever goes”, so we don't get CUDA or VULKAN or whatever here
 848         // without enumerating through several different types.
 849         // VA-API and VDPAU will do for now. We prioritize VDPAU for the
 850         // simple reason that there's a VA-API-via-VDPAU emulation for NVidia
 851         // cards that seems to work, but just hangs when trying to transfer the frame.
 852         //
 853         // Note that we don't actually check codec support beforehand,
 854         // so if you have a low-end VDPAU device but a high-end VA-API device,
 855         // you lose out on the extra codec support from the latter.
 856         AVBufferRef *hw_device_ctx = nullptr;
 857         if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VDPAU, nullptr, nullptr, 0) >= 0) {
 858                 video_codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
 859                 video_codec_ctx->get_format = get_hw_format<AV_HWDEVICE_TYPE_VDPAU>;
 860         } else if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI, nullptr, nullptr, 0) >= 0) {
 861                 video_codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
 862                 video_codec_ctx->get_format = get_hw_format<AV_HWDEVICE_TYPE_VAAPI>;
 863         } else {
 864                 fprintf(stderr, "Failed to initialize VA-API or VDPAU for FFmpeg acceleration. Decoding video in software.\n");
 865         }
 866
 867         if (avcodec_open2(video_codec_ctx.get(), video_codec, nullptr) < 0) {
 868                 fprintf(stderr, "%s: Cannot open video decoder\n", pathname.c_str());
 869                 return false;
 870         }
 871         unique_ptr<AVCodecContext, decltype(avcodec_close)*> video_codec_ctx_cleanup(
 872                 video_codec_ctx.get(), avcodec_close);
 873
 874         internal_rewind();
 875
 876         // Main loop.
 877         int consecutive_errors = 0;
 878         double rate = 1.0;
 879         while (!producer_thread_should_quit.should_quit()) {
 880                 if (process_queued_commands(format_ctx.get(), video_codec_ctx.get(), video_stream_index, /*seeked=*/nullptr)) {
 881                         return true;
 882                 }
 883                 if (paused) {
 884                         producer_thread_should_quit.sleep_for(hours(1));
 885                         continue;
 886                 }
 887
 888                 bool error;
 889                 AVFrameWithDeleter frame = decode_frame(format_ctx.get(), video_codec_ctx.get(),
 890                         pathname, video_stream_index, &error);
 891                 if (error) {
 892                         if (++consecutive_errors >= 100) {
 893                                 fprintf(stderr, "More than 100 consecutive video frames, aborting playback.\n");
 894                                 return false;
 895                         } else {
 896                                 continue;
 897                         }
 898                 } else {
 899                         consecutive_errors = 0;
 900                 }
 901                 if (frame == nullptr) {
 902                         // EOF.
 903                         return false;
 904                 }
 905
 906                 // Sleep until it's time to present this frame.
 907                 for ( ;; ) {
 908                         if (last_pts == 0 && pts_origin == 0) {
 909                                 pts_origin = frame->pts;
 910                         }
 911                         steady_clock::time_point now = steady_clock::now();
 912                         next_frame_start = compute_frame_start(frame->pts, pts_origin, video_timebase, start, rate);
 913
 914                         if (duration<double>(now - next_frame_start).count() >= 0.1) {
 915                                 // If we don't have enough CPU to keep up, or if we have a live stream
 916                                 // where the initial origin was somehow wrong, we could be behind indefinitely.
 917                                 fprintf(stderr, "%s: Playback %.0f ms behind, resetting time scale\n",
 918                                         pathname.c_str(),
 919                                         1e3 * duration<double>(now - next_frame_start).count());
 920                                 pts_origin = frame->pts;
 921                                 start = next_frame_start = now;
 922                         }
 923                         bool finished_wakeup;
 924                         finished_wakeup = producer_thread_should_quit.sleep_until(next_frame_start);
 925                         if (finished_wakeup) {
 926                                 {
 927                                         lock_guard lock(current_frame_mu);
 928                                         current_frame.reset(new Frame(make_video_frame(frame.get())));
 929                                 }
 930                                 last_frame = steady_clock::now();
 931                                 update();
 932                                 break;
 933                         } else {
 934                                 if (producer_thread_should_quit.should_quit()) break;
 935
 936                                 bool seeked = false;
 937                                 if (process_queued_commands(format_ctx.get(), video_codec_ctx.get(), video_stream_index, &seeked)) {
 938                                         return true;
 939                                 }
 940
 941                                 if (paused) {
 942                                         // Just paused, so present the frame immediately and then go into deep sleep.
 943                                         {
 944                                                 lock_guard lock(current_frame_mu);
 945                                                 current_frame.reset(new Frame(make_video_frame(frame.get())));
 946                                         }
 947                                         last_frame = steady_clock::now();
 948                                         update();
 949                                         break;
 950                                 }
 951
 952                                 // If we just seeked, drop this frame on the floor and be done.
 953                                 if (seeked) {
 954                                         break;
 955                                 }
 956                         }
 957                 }
 958                 store_pts(frame->pts);
 959         }
 960         return true;
 961 }
 962
 963 void VideoWidget::store_pts(int64_t pts)
 964 {
 965         last_pts = pts;
 966         last_position = lrint(pts * double(video_timebase.num) / double(video_timebase.den) * 1000);
 967         post_to_main_thread([this, last_position{last_position.load()}] {
 968                 emit position_changed(last_position);
 969         });
 970 }
 971
 972 // Taken from Movit (see the comment there for explanation)
 973 float compute_chroma_offset(float pos, unsigned subsampling_factor, unsigned resolution)
 974 {
 975         float local_chroma_pos = (0.5 + pos * (subsampling_factor - 1)) / subsampling_factor;
 976         if (fabs(local_chroma_pos - 0.5) < 1e-10) {
 977                 // x + (-0) can be optimized away freely, as opposed to x + 0.
 978                 return -0.0;
 979         } else {
 980                 return (0.5 - local_chroma_pos) / resolution;
 981         }
 982 }
 983
 984 VideoWidget::Frame VideoWidget::make_video_frame(const AVFrame *frame)
 985 {
 986         Frame video_frame;
 987         AVFrameWithDeleter sw_frame;
 988
 989         if (frame->format == AV_PIX_FMT_VAAPI ||
 990             frame->format == AV_PIX_FMT_VDPAU) {
 991                 // Get the frame down to the CPU. (TODO: See if we can keep it
 992                 // on the GPU all the way, since it will be going up again later.
 993                 // However, this only works if the OpenGL GPU is the same one.)
 994                 sw_frame = av_frame_alloc_unique();
 995                 int err = av_hwframe_transfer_data(sw_frame.get(), frame, 0);
 996                 if (err != 0) {
 997                         fprintf(stderr, "%s: Cannot transfer hardware video frame to software.\n", pathname.c_str());
 998                 } else {
 999                         sw_frame->pts = frame->pts;
1000                         sw_frame->pkt_duration = frame->pkt_duration;
1001                         frame = sw_frame.get();
1002                 }
1003         }
1004
1005         if (sws_ctx == nullptr ||
1006             sws_last_width != frame->width ||
1007             sws_last_height != frame->height ||
1008             sws_last_src_format != frame->format) {
1009                 sws_dst_format = decide_dst_format(AVPixelFormat(frame->format));
1010                 sws_ctx.reset(
1011                         sws_getContext(frame->width, frame->height, AVPixelFormat(frame->format),
1012                                 frame->width, frame->height, sws_dst_format,
1013                                 SWS_BICUBIC, nullptr, nullptr, nullptr));
1014                 sws_last_width = frame->width;
1015                 sws_last_height = frame->height;
1016                 sws_last_src_format = frame->format;
1017         }
1018         if (sws_ctx == nullptr) {
1019                 fprintf(stderr, "Could not create scaler context\n");
1020                 abort();
1021         }
1022
1023         uint8_t *pic_data[4] = { nullptr, nullptr, nullptr, nullptr };
1024         int linesizes[4] = { 0, 0, 0, 0 };
1025         const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(sws_dst_format);
1026
1027         video_frame.width = frame->width;
1028         video_frame.height = frame->height;
1029         video_frame.chroma_width = AV_CEIL_RSHIFT(int(frame->width), desc->log2_chroma_w);
1030         video_frame.chroma_height = AV_CEIL_RSHIFT(int(frame->height), desc->log2_chroma_h);
1031
1032         // We always assume left chroma placement for now.
1033         cbcr_offset[0] = compute_chroma_offset(0.0f, 1 << desc->log2_chroma_w, video_frame.chroma_width);
1034         cbcr_offset[1] = compute_chroma_offset(0.5f, 1 << desc->log2_chroma_h, video_frame.chroma_height);
1035
1036         size_t len = frame->width * frame->height + 2 * video_frame.chroma_width * video_frame.chroma_height;
1037         video_frame.data.reset(new uint8_t[len]);
1038
1039         pic_data[0] = video_frame.data.get();
1040         linesizes[0] = frame->width;
1041
1042         pic_data[1] = pic_data[0] + frame->width * frame->height;
1043         linesizes[1] = video_frame.chroma_width;
1044
1045         pic_data[2] = pic_data[1] + video_frame.chroma_width * video_frame.chroma_height;
1046         linesizes[2] = video_frame.chroma_width;
1047
1048         sws_scale(sws_ctx.get(), frame->data, frame->linesize, 0, frame->height, pic_data, linesizes);
1049
1050         return video_frame;
1051 }
1052