#include <QMainWindow>
#include <QApplication>
#include <QGridLayout>
-#include <QVideoWidget>
#include <QShortcut>
#include <QInputDialog>
#include <QTimer>
#include "players.h"
#include "formations.h"
#include "json.h"
+#include "video_widget.h"
using namespace std;
FormationsModel *offensive_formations, FormationsModel *defensive_formations)
: events(events), players(players), offensive_formations(offensive_formations), defensive_formations(defensive_formations)
{
- video = new QMediaPlayer;
- //video->setSource(QUrl::fromLocalFile("/home/sesse/dev/stats/ultimate.mkv"));
- video->setSource(QUrl::fromLocalFile("/home/sesse/dev/stats/ultimate-prores.mkv"));
- video->play();
-
ui = new Ui::MainWindow;
ui->setupUi(this);
+ ui->video->open("/home/sesse/dev/stats/ultimate-prores.mkv");
+ ui->video->play();
+
ui->event_view->setModel(events);
ui->event_view->setColumnWidth(1, 150);
ui->event_view->setColumnWidth(2, 150);
connect(ui->event_view->selectionModel(), &QItemSelectionModel::currentRowChanged,
[this, events](const QModelIndex ¤t, const QModelIndex &previous) {
- int64_t t = events->get_time(current.row());
- if (t != video->position()) {
- video->setPosition(events->get_time(current.row()));
+ uint64_t t = events->get_time(current.row());
+ if (t != ui->video->get_position()) {
+ ui->video->seek_absolute(events->get_time(current.row()));
} else {
// Selection could have changed, so we still need to update.
// (Just calling setPosition() would not give us the signal
auto formation_changed = [this](const QModelIndex ¤t, const QModelIndex &previous) {
QTimer::singleShot(1, [=]{ // The selection is wrong until the callback actually returns.
- update_action_buttons(video->position());
+ update_action_buttons(ui->video->get_position());
});
};
ui->offensive_formation_view->setModel(offensive_formations);
formation_double_clicked(false, index.row());
});
- connect(video, &QMediaPlayer::positionChanged, [this](uint64_t pos) {
+ connect(ui->video, &VideoWidget::position_changed, [this](uint64_t pos) {
position_changed(pos);
});
- video->setVideoOutput(ui->video);
// It's not really clear whether PgUp should be forwards or backwards,
// but mpv does at least up = forwards, so that's probably standard.
QShortcut *pgdown = new QShortcut(QKeySequence(Qt::Key_PageDown), this);
- connect(pgdown, &QShortcut::activated, [this]() { seek(-120000); });
+ connect(pgdown, &QShortcut::activated, [this]() { ui->video->seek(-120000); });
QShortcut *pgup = new QShortcut(QKeySequence(Qt::Key_PageUp), this);
- connect(pgup, &QShortcut::activated, [this]() { seek(120000); });
-
- // Ugh. Used when Qt messes up and hangs the video.
- QShortcut *f5 = new QShortcut(QKeySequence(Qt::Key_F5), this);
- connect(f5, &QShortcut::activated, [this]() {
- QVideoWidget *nvw = new QVideoWidget(ui->video->parentWidget());
- nvw->setObjectName("video");
- nvw->setMinimumSize(QSize(320, 240));
- video->setVideoOutput(nvw);
- ui->main_grid->replaceWidget(ui->video, nvw);
- ui->video = nvw;
- });
+ connect(pgup, &QShortcut::activated, [this]() { ui->video->seek(120000); });
- connect(ui->minus10s, &QPushButton::clicked, [this]() { seek(-10000); });
- connect(ui->plus10s, &QPushButton::clicked, [this]() { seek(10000); });
+ connect(ui->minus10s, &QPushButton::clicked, [this]() { ui->video->seek(-10000); });
+ connect(ui->plus10s, &QPushButton::clicked, [this]() { ui->video->seek(10000); });
- connect(ui->minus2s, &QPushButton::clicked, [this]() { seek(-2000); });
- connect(ui->plus2s, &QPushButton::clicked, [this]() { seek(2000); });
+ connect(ui->minus2s, &QPushButton::clicked, [this]() { ui->video->seek(-2000); });
+ connect(ui->plus2s, &QPushButton::clicked, [this]() { ui->video->seek(2000); });
- // TODO: Would be nice to actually have a frame...
- connect(ui->minus1f, &QPushButton::clicked, [this]() { seek(-20); });
- connect(ui->plus1f, &QPushButton::clicked, [this]() { seek(20); });
+ connect(ui->minus1f, &QPushButton::clicked, [this]() { ui->video->seek_frames(-1); });
+ connect(ui->plus1f, &QPushButton::clicked, [this]() { ui->video->seek_frames(1); });
connect(ui->play_pause, &QPushButton::clicked, [this]() {
if (playing) {
- video->pause();
+ ui->video->pause();
ui->play_pause->setText("Play (space)");
} else {
- video->setPlaybackRate(1.0);
- video->play();
+ ui->video->play();
ui->play_pause->setText("Pause (space)");
}
playing = !playing;
connect(ui->offense_label, &ClickableLabel::clicked, [this]() { insert_noplayer_event("set_offense"); });
connect(ui->catch_, &QPushButton::clicked, [this]() { set_current_event_type("catch"); });
connect(ui->throwaway, &QPushButton::clicked, [this, events]() {
- EventsModel::Status s = events->get_status_at(video->position());
+ EventsModel::Status s = events->get_status_at(ui->video->get_position());
if (s.attack_state == EventsModel::Status::DEFENSE && s.pull_state == EventsModel::Status::PULL_IN_AIR) {
insert_noplayer_event("pull_oob");
} else {
connect(ui->offensive_soft_plus, &QPushButton::clicked, [this]() { set_current_event_type("offensive_soft_plus"); });
connect(ui->offensive_soft_minus, &QPushButton::clicked, [this]() { set_current_event_type("offensive_soft_minus"); });
connect(ui->pull, &QPushButton::clicked, [this, events]() {
- EventsModel::Status s = events->get_status_at(video->position());
+ EventsModel::Status s = events->get_status_at(ui->video->get_position());
if (s.pull_state == EventsModel::Status::SHOULD_PULL) {
set_current_event_type("pull");
} else if (s.pull_state == EventsModel::Status::PULL_IN_AIR) {
connect(ui->their_throwaway, &QPushButton::clicked, [this]() { insert_noplayer_event("their_throwaway"); });
connect(ui->their_goal, &QPushButton::clicked, [this]() { insert_noplayer_event("their_goal"); });
connect(ui->their_pull, &QPushButton::clicked, [this, events]() {
- EventsModel::Status s = events->get_status_at(video->position());
+ EventsModel::Status s = events->get_status_at(ui->video->get_position());
if (s.pull_state == EventsModel::Status::SHOULD_PULL) {
insert_noplayer_event("their_pull");
}
// Misc. events
connect(ui->substitution, &QPushButton::clicked, [this]() { make_substitution(); });
connect(ui->stoppage, &QPushButton::clicked, [this, events]() {
- EventsModel::Status s = events->get_status_at(video->position());
+ EventsModel::Status s = events->get_status_at(ui->video->get_position());
if (s.stoppage) {
insert_noplayer_event("restart");
} else {
void MainWindow::position_changed(uint64_t pos)
{
ui->timestamp->setText(QString::fromUtf8(format_timestamp(pos)));
- if (buffered_seek) {
- video->setPosition(*buffered_seek);
- buffered_seek.reset();
- }
if (!playing) {
- video->pause(); // We only played to get a picture.
+ ui->video->pause(); // We only played to get a picture.
}
if (playing) {
- QModelIndex row = events->get_last_event_qt(video->position());
+ QModelIndex row = events->get_last_event_qt(ui->video->get_position());
ui->event_view->scrollTo(row, QAbstractItemView::PositionAtCenter);
}
update_ui_from_time(pos);
}
-void MainWindow::seek(int64_t delta_ms)
-{
- int64_t current_pos = buffered_seek ? *buffered_seek : video->position();
- uint64_t pos = max<int64_t>(current_pos + delta_ms, 0);
- buffered_seek = pos;
- if (!playing) {
- video->setPlaybackRate(0.01);
- video->play(); // Or Qt won't show the seek.
- }
-}
-
void MainWindow::insert_player_event(int button_id)
{
- uint64_t t = video->position();
+ uint64_t t = ui->video->get_position();
vector<int> team = events->sort_team(events->get_team_at(t));
if (unsigned(button_id) >= team.size()) {
return;
void MainWindow::insert_noplayer_event(const string &type)
{
- uint64_t t = video->position();
+ uint64_t t = ui->video->get_position();
ui->event_view->selectionModel()->blockSignals(true);
ui->event_view->selectRow(events->insert_event(t, nullopt, nullopt, type));
}
int row = select->selectedRows().front().row(); // Should only be one, due to our selection behavior.
events->set_event_type(row, type);
- update_ui_from_time(video->position());
+ update_ui_from_time(ui->video->get_position());
}
// Formation buttons either modify the existing formation (if we've selected
string expected_type = offense ? "formation_offense" : "formation_defense";
if (events->get_event_type(row) == expected_type) {
events->set_event_formation(row, formation_id);
- update_ui_from_time(video->position());
+ update_ui_from_time(ui->video->get_position());
return;
}
}
// Insert a new formation event instead (same as double-click on the selected one).
- events->set_formation_at(video->position(), offense, formation_id);
- update_ui_from_time(video->position());
+ events->set_formation_at(ui->video->get_position(), offense, formation_id);
+ update_ui_from_time(ui->video->get_position());
}
void MainWindow::delete_current_event()
ui->event_view->selectionModel()->blockSignals(true);
events->delete_event(row);
ui->event_view->selectionModel()->blockSignals(false);
- update_ui_from_time(video->position());
+ update_ui_from_time(ui->video->get_position());
}
void MainWindow::make_substitution()
for (QModelIndex row : select->selectedRows()) {
new_team.insert(players->get_player_id(row.row()));
}
- events->set_team_at(video->position(), new_team);
+ events->set_team_at(ui->video->get_position(), new_team);
}
void MainWindow::update_ui_from_time(uint64_t t)
view->selectionModel()->select(formations->index(formations->get_row_from_id(id), 0), QItemSelectionModel::ClearAndSelect);
events->inserted_new_formation(id, new_formation_str.toStdString());
} else {
- events->set_formation_at(video->position(), offense, id);
+ events->set_formation_at(ui->video->get_position(), offense, id);
}
- update_ui_from_time(video->position());
+ update_ui_from_time(ui->video->get_position());
}
sqlite3 *open_db(const char *filename)
--- /dev/null
+#define GL_GLEXT_PROTOTYPES
+
+#include "video_widget.h"
+
+#include <assert.h>
+#include <pthread.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+#include <libavutil/avutil.h>
+#include <libavutil/error.h>
+#include <libavutil/frame.h>
+#include <libavutil/imgutils.h>
+#include <libavutil/mem.h>
+#include <libavutil/pixfmt.h>
+#include <libavutil/opt.h>
+#include <libswscale/swscale.h>
+}
+
+#include <chrono>
+#include <cstdint>
+#include <utility>
+#include <vector>
+#include <unordered_set>
+
+#include <QOpenGLFunctions>
+
+using namespace std;
+using namespace std::chrono;
+
+namespace {
+
+bool is_full_range(const AVPixFmtDescriptor *desc)
+{
+ // This is horrible, but there's no better way that I know of.
+ return (strchr(desc->name, 'j') != nullptr);
+}
+
+AVPixelFormat decide_dst_format(AVPixelFormat src_format)
+{
+ // If this is a non-Y'CbCr format, just convert to 4:4:4 Y'CbCr
+ // and be done with it. It's too strange to spend a lot of time on.
+ // (Let's hope there's no alpha.)
+ const AVPixFmtDescriptor *src_desc = av_pix_fmt_desc_get(src_format);
+ if (src_desc == nullptr ||
+ src_desc->nb_components != 3 ||
+ (src_desc->flags & AV_PIX_FMT_FLAG_RGB)) {
+ return AV_PIX_FMT_YUV444P;
+ }
+
+ // The best for us would be Cb and Cr together if possible,
+ // but FFmpeg doesn't support that except in the special case of
+ // NV12, so we need to go to planar even for the case of NV12.
+ // Thus, look for the closest (but no worse) 8-bit planar Y'CbCr format
+ // that matches in color range. (This will also include the case of
+ // the source format already being acceptable.)
+ bool src_full_range = is_full_range(src_desc);
+ const char *best_format = "yuv444p";
+ unsigned best_score = numeric_limits<unsigned>::max();
+ for (const AVPixFmtDescriptor *desc = av_pix_fmt_desc_next(nullptr);
+ desc;
+ desc = av_pix_fmt_desc_next(desc)) {
+ // Find planar Y'CbCr formats only.
+ if (desc->nb_components != 3) continue;
+ if (desc->flags & AV_PIX_FMT_FLAG_RGB) continue;
+ if (!(desc->flags & AV_PIX_FMT_FLAG_PLANAR)) continue;
+ if (desc->comp[0].plane != 0 ||
+ desc->comp[1].plane != 1 ||
+ desc->comp[2].plane != 2) continue;
+
+ // 8-bit formats only.
+ if (desc->flags & AV_PIX_FMT_FLAG_BE) continue;
+ if (desc->comp[0].depth != 8) continue;
+
+ // Same or better chroma resolution only.
+ int chroma_w_diff = src_desc->log2_chroma_w - desc->log2_chroma_w;
+ int chroma_h_diff = src_desc->log2_chroma_h - desc->log2_chroma_h;
+ if (chroma_w_diff < 0 || chroma_h_diff < 0)
+ continue;
+
+ // Matching full/limited range only.
+ if (is_full_range(desc) != src_full_range)
+ continue;
+
+ // Pick something with as little excess chroma resolution as possible.
+ unsigned score = (1 << (chroma_w_diff)) << chroma_h_diff;
+ if (score < best_score) {
+ best_score = score;
+ best_format = desc->name;
+ }
+ }
+ return av_get_pix_fmt(best_format);
+}
+
+} // namespace
+
+bool VideoWidget::process_queued_commands(AVFormatContext *format_ctx, AVCodecContext *video_codec_ctx, int video_stream_index, bool *seeked)
+{
+ // Process any queued commands from other threads.
+ vector<QueuedCommand> commands;
+ {
+ lock_guard<mutex> lock(queue_mu);
+ swap(commands, command_queue);
+ }
+
+ for (const QueuedCommand &cmd : commands) {
+ switch (cmd.command) {
+ case QueuedCommand::PAUSE:
+ paused = true;
+ break;
+ case QueuedCommand::RESUME:
+ paused = false;
+ pts_origin = last_pts;
+ start = next_frame_start = steady_clock::now();
+ break;
+ case QueuedCommand::SEEK:
+ case QueuedCommand::SEEK_ABSOLUTE:
+ // Dealt with below.
+ break;
+ }
+ }
+
+ // Combine all seeks into one big one. (There are edge cases where this is probably
+ // subtly wrong, but we'll live with it.)
+ int64_t base_pts = last_pts;
+ int64_t relative_seek_ms = 0;
+ int64_t relative_seek_frames = 0;
+ for (const QueuedCommand &cmd : commands) {
+ if (cmd.command == QueuedCommand::SEEK) {
+ relative_seek_ms += cmd.relative_seek_ms;
+ relative_seek_frames += cmd.relative_seek_frames;
+ } else if (cmd.command == QueuedCommand::SEEK_ABSOLUTE) {
+ base_pts = cmd.seek_ms;
+ relative_seek_ms = 0;
+ relative_seek_frames = 0;
+ }
+ }
+ int64_t relative_seek_pts = av_rescale_q(relative_seek_ms, AVRational{ 1, 1000 }, video_timebase);
+ if (relative_seek_ms != 0 && relative_seek_pts == 0) {
+ // Just to be sure rounding errors don't move us into nothingness.
+ relative_seek_pts = (relative_seek_ms > 0) ? 1 : -1;
+ }
+ int64_t goal_pts = base_pts + relative_seek_pts;
+ if (goal_pts != last_pts || relative_seek_frames < 0) {
+ avcodec_flush_buffers(video_codec_ctx);
+ queued_frames.clear();
+
+ // Seek to the last keyframe before this point.
+ int64_t seek_pts = goal_pts;
+ if (relative_seek_frames < 0) {
+ // If we're frame-skipping backwards, add 100 ms of slop for each frame
+ // so we're fairly certain we are able to see the ones we want.
+ seek_pts -= av_rescale_q(-relative_seek_frames, AVRational{ 1, 10 }, video_timebase);
+ }
+ av_seek_frame(format_ctx, video_stream_index, seek_pts, AVSEEK_FLAG_BACKWARD);
+
+ // Decode frames until EOF, or until we see something past our seek point.
+ std::deque<AVFrameWithDeleter> queue;
+ for ( ;; ) {
+ bool error = false;
+ AVFrameWithDeleter frame = decode_frame(format_ctx, video_codec_ctx,
+ pathname, video_stream_index, &error);
+ if (frame == nullptr || error) {
+ break;
+ }
+
+ int64_t frame_pts = frame->pts;
+ if (relative_seek_frames < 0) {
+ // Buffer this frame; don't display it unless we know it's the Nth-latest.
+ queue.push_back(std::move(frame));
+ if (queue.size() > uint64_t(-relative_seek_frames) + 1) {
+ queue.pop_front();
+ }
+ }
+ if (frame_pts >= goal_pts) {
+ if (relative_seek_frames > 0) {
+ --relative_seek_frames;
+ } else {
+ if (relative_seek_frames < 0) {
+ // Hope we have the right amount.
+ // The rest will remain in the queue for when we play forward again.
+ frame = std::move(queue.front());
+ queue.pop_front();
+ queued_frames = std::move(queue);
+ }
+ current_frame.reset(new Frame(make_video_frame(frame.get())));
+ update();
+ store_pts(frame->pts);
+ break;
+ }
+ }
+ }
+
+ // NOTE: We keep pause status as-is.
+
+ pts_origin = last_pts;
+ start = next_frame_start = last_frame = steady_clock::now();
+ if (seeked) {
+ *seeked = true;
+ }
+ } else if (relative_seek_frames > 0) {
+ // The base PTS is fine, we only need to skip a few frames forwards.
+ while (relative_seek_frames > 1) {
+ // Eat a frame (ignore errors).
+ bool error;
+ decode_frame(format_ctx, video_codec_ctx, pathname, video_stream_index, &error);
+ --relative_seek_frames;
+ }
+
+ // Display the last one.
+ bool error;
+ AVFrameWithDeleter frame = decode_frame(format_ctx, video_codec_ctx,
+ pathname, video_stream_index, &error);
+ if (frame == nullptr || error) {
+ return true;
+ }
+ current_frame.reset(new Frame(make_video_frame(frame.get())));
+ update();
+ store_pts(frame->pts);
+ }
+ return false;
+}
+
+VideoWidget::VideoWidget(QWidget *parent)
+ : QOpenGLWidget(parent) {}
+
+GLuint compile_shader(const string &shader_src, GLenum type)
+{
+ GLuint obj = glCreateShader(type);
+ const GLchar* source[] = { shader_src.data() };
+ const GLint length[] = { (GLint)shader_src.size() };
+ glShaderSource(obj, 1, source, length);
+ glCompileShader(obj);
+
+ GLchar info_log[4096];
+ GLsizei log_length = sizeof(info_log) - 1;
+ glGetShaderInfoLog(obj, log_length, &log_length, info_log);
+ info_log[log_length] = 0;
+ if (strlen(info_log) > 0) {
+ fprintf(stderr, "Shader compile log: %s\n", info_log);
+ }
+
+ GLint status;
+ glGetShaderiv(obj, GL_COMPILE_STATUS, &status);
+ if (status == GL_FALSE) {
+ // Add some line numbers to easier identify compile errors.
+ string src_with_lines = "/* 1 */ ";
+ size_t lineno = 1;
+ for (char ch : shader_src) {
+ src_with_lines.push_back(ch);
+ if (ch == '\n') {
+ char buf[32];
+ snprintf(buf, sizeof(buf), "/* %3zu */ ", ++lineno);
+ src_with_lines += buf;
+ }
+ }
+
+ fprintf(stderr, "Failed to compile shader:\n%s\n", src_with_lines.c_str());
+ exit(1);
+ }
+
+ return obj;
+}
+
+void VideoWidget::initializeGL()
+{
+ glDisable(GL_BLEND);
+ glDisable(GL_DEPTH_TEST);
+ glDepthMask(GL_FALSE);
+ glCreateTextures(GL_TEXTURE_2D, 3, tex);
+
+ ycbcr_vertex_shader = compile_shader(R"(
+#version 460 core
+
+layout(location = 0) in vec2 position;
+layout(location = 1) in vec2 texcoord;
+out vec2 tc;
+
+void main()
+{
+ // The result of glOrtho(0.0, 1.0, 0.0, 1.0, 0.0, 1.0) is:
+ //
+ // 2.000 0.000 0.000 -1.000
+ // 0.000 2.000 0.000 -1.000
+ // 0.000 0.000 -2.000 -1.000
+ // 0.000 0.000 0.000 1.000
+ gl_Position = vec4(2.0 * position.x - 1.0, 2.0 * position.y - 1.0, -1.0, 1.0);
+ tc = texcoord;
+ tc.y = 1.0f - tc.y;
+}
+)", GL_VERTEX_SHADER);
+ ycbcr_fragment_shader = compile_shader(R"(
+#version 460 core
+
+layout(location = 0) uniform sampler2D tex_y;
+layout(location = 1) uniform sampler2D tex_cb;
+layout(location = 2) uniform sampler2D tex_cr;
+layout(location = 3) uniform vec2 cbcr_offset;
+
+in vec2 tc;
+out vec4 FragColor;
+
+// Computed statically by Movit, for limited-range BT.709.
+// (We don't check whether the input could be BT.601 or BT.2020 currently, or full-range)
+const mat3 inv_ycbcr_matrix = mat3(
+ 1.16438f, 1.16438f, 1.16438f,
+ 0.0f, -0.21325f, 2.11240f,
+ 1.79274f, -0.53291f, 0.0f
+);
+
+void main()
+{
+ if (tc.x < 0.0 || tc.x > 1.0 || tc.y < 0.0 || tc.y > 1.0) {
+ FragColor.rgba = vec4(0.0f, 0.0f, 0.0f, 1.0f);
+ return;
+ }
+
+ vec3 ycbcr;
+ ycbcr.r = texture(tex_y, tc).r;
+ ycbcr.g = texture(tex_cb, tc + cbcr_offset).r;
+ ycbcr.b = texture(tex_cr, tc + cbcr_offset).r;
+ ycbcr -= vec3(16.0f / 255.0f, 128.0f / 255.0f, 128.0f / 255.0f);
+ FragColor.rgb = inv_ycbcr_matrix * ycbcr;
+ FragColor.a = 1.0f;
+}
+)", GL_FRAGMENT_SHADER);
+ ycbcr_program = glCreateProgram();
+ glAttachShader(ycbcr_program, ycbcr_vertex_shader);
+ glAttachShader(ycbcr_program, ycbcr_fragment_shader);
+ glLinkProgram(ycbcr_program);
+
+ GLint success;
+ glGetProgramiv(ycbcr_program, GL_LINK_STATUS, &success);
+ if (success == GL_FALSE) {
+ GLchar error_log[1024] = {0};
+ glGetProgramInfoLog(ycbcr_program, 1024, nullptr, error_log);
+ fprintf(stderr, "Error linking program: %s\n", error_log);
+ exit(1);
+ }
+
+ glCreateSamplers(1, &bilinear_sampler);
+ glSamplerParameteri(bilinear_sampler, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_NEAREST);
+ glSamplerParameteri(bilinear_sampler, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+ glSamplerParameteri(bilinear_sampler, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+ glSamplerParameteri(bilinear_sampler, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+}
+
+void VideoWidget::resizeGL(int w, int h)
+{
+ glViewport(0, 0, w, h);
+ display_aspect = double(w) / h;
+}
+
+int num_levels(GLuint width, GLuint height)
+{
+ int levels = 1;
+ while (width > 1 || height > 1) {
+ width = max(width / 2, 1u);
+ height = max(height / 2, 1u);
+ ++levels;
+ }
+ return levels;
+}
+
+void VideoWidget::paintGL()
+{
+ std::shared_ptr<Frame> frame = current_frame;
+ if (frame == nullptr) {
+ glClear(GL_COLOR_BUFFER_BIT);
+ return;
+ }
+
+ glUseProgram(ycbcr_program);
+ if (frame->width != last_width || frame->height != last_height) {
+ glTextureStorage2D(tex[0], num_levels(frame->width, frame->height), GL_R8, frame->width, frame->height);
+ }
+ if (frame->chroma_width != last_chroma_width || frame->chroma_height != last_chroma_height) {
+ for (GLuint num : { tex[1], tex[2] }) {
+ glTextureStorage2D(num, num_levels(frame->chroma_width, frame->chroma_height), GL_R8, frame->chroma_width, frame->chroma_height);
+ }
+ }
+
+ glTextureSubImage2D(tex[0], 0, 0, 0, frame->width, frame->height, GL_RED, GL_UNSIGNED_BYTE, frame->data.get());
+ glGenerateTextureMipmap(tex[0]);
+
+ glTextureSubImage2D(tex[1], 0, 0, 0, frame->chroma_width, frame->chroma_height, GL_RED, GL_UNSIGNED_BYTE, frame->data.get() + frame->width * frame->height);
+ glGenerateTextureMipmap(tex[1]);
+
+ glTextureSubImage2D(tex[2], 0, 0, 0, frame->chroma_width, frame->chroma_height, GL_RED, GL_UNSIGNED_BYTE, frame->data.get() + frame->width * frame->height + frame->chroma_width * frame->chroma_height);
+ glGenerateTextureMipmap(tex[2]);
+
+ glBindTextureUnit(0, tex[0]);
+ glBindTextureUnit(1, tex[1]);
+ glBindTextureUnit(2, tex[2]);
+ glBindSampler(0, bilinear_sampler);
+ glBindSampler(1, bilinear_sampler);
+ glBindSampler(2, bilinear_sampler);
+ glProgramUniform1i(ycbcr_program, 0, 0);
+ glProgramUniform1i(ycbcr_program, 1, 1);
+ glProgramUniform1i(ycbcr_program, 2, 2);
+ glProgramUniform2f(ycbcr_program, 3, cbcr_offset[0], -cbcr_offset[1]);
+
+ float tx1 = 0.0f;
+ float tx2 = 1.0f;
+ float ty1 = 0.0f;
+ float ty2 = 1.0f;
+
+ double video_aspect = double(frame->width) / frame->height;
+ if (display_aspect > video_aspect) {
+ double extra_width = frame->height * display_aspect - frame->width;
+ tx1 = -0.5 * extra_width / frame->width;
+ tx2 = 1.0 + 0.5 * extra_width / frame->width;
+ } else if (display_aspect < video_aspect) {
+ double extra_height = frame->width / display_aspect - frame->height;
+ ty1 = -0.5 * extra_height / frame->height;
+ ty2 = 1.0 + 0.5 * extra_height / frame->height;
+ }
+
+ glBegin(GL_QUADS);
+
+ glVertexAttrib2f(1, tx1, ty1);
+ glVertex2f(0.0f, 0.0f);
+
+ glVertexAttrib2f(1, tx1, ty2);
+ glVertex2f(0.0f, 1.0f);
+
+ glVertexAttrib2f(1, tx2, ty2);
+ glVertex2f(1.0f, 1.0f);
+
+ glVertexAttrib2f(1, tx2, ty1);
+ glVertex2f(1.0f, 0.0f);
+
+ glEnd();
+}
+
+void VideoWidget::open(const string &filename)
+{
+ stop();
+ internal_rewind();
+ pathname = filename;
+ play();
+}
+
+void VideoWidget::play()
+{
+ if (running) {
+ std::lock_guard<std::mutex> lock(queue_mu);
+ command_queue.push_back(QueuedCommand { QueuedCommand::RESUME });
+ producer_thread_should_quit.wakeup();
+ return;
+ }
+ running = true;
+ producer_thread_should_quit.unquit();
+ producer_thread = std::thread(&VideoWidget::producer_thread_func, this);
+}
+
+void VideoWidget::pause()
+{
+ if (!running) {
+ return;
+ }
+ std::lock_guard<std::mutex> lock(queue_mu);
+ command_queue.push_back(QueuedCommand { QueuedCommand::PAUSE });
+ producer_thread_should_quit.wakeup();
+}
+
+void VideoWidget::seek(int64_t relative_seek_ms)
+{
+ if (!running) {
+ return;
+ }
+ std::lock_guard<std::mutex> lock(queue_mu);
+ command_queue.push_back(QueuedCommand { QueuedCommand::SEEK, relative_seek_ms, 0, 0 });
+ producer_thread_should_quit.wakeup();
+}
+
+void VideoWidget::seek_frames(int64_t relative_seek_frames)
+{
+ if (!running) {
+ return;
+ }
+ std::lock_guard<std::mutex> lock(queue_mu);
+ command_queue.push_back(QueuedCommand { QueuedCommand::SEEK, 0, relative_seek_frames, 0 });
+ producer_thread_should_quit.wakeup();
+}
+
+void VideoWidget::seek_absolute(int64_t position_ms)
+{
+ if (!running) {
+ return;
+ }
+ std::lock_guard<std::mutex> lock(queue_mu);
+ command_queue.push_back(QueuedCommand { QueuedCommand::SEEK_ABSOLUTE, 0, 0, position_ms });
+ producer_thread_should_quit.wakeup();
+}
+
+void VideoWidget::stop()
+{
+ if (!running) {
+ return;
+ }
+ running = false;
+ producer_thread_should_quit.quit();
+ producer_thread.join();
+}
+
+void VideoWidget::producer_thread_func()
+{
+ if (!producer_thread_should_quit.should_quit()) {
+ if (!play_video(pathname)) {
+ // TODO: Send the error back to the UI somehow.
+ }
+ }
+}
+
+void VideoWidget::internal_rewind()
+{
+ pts_origin = last_pts = 0;
+ last_position = 0;
+ start = next_frame_start = steady_clock::now();
+}
+
+template<AVHWDeviceType type>
+AVPixelFormat get_hw_format(AVCodecContext *ctx, const AVPixelFormat *fmt)
+{
+ bool found_config_of_right_type = false;
+ for (int i = 0;; ++i) { // Termination condition inside loop.
+ const AVCodecHWConfig *config = avcodec_get_hw_config(ctx->codec, i);
+ if (config == nullptr) { // End of list.
+ break;
+ }
+ if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) ||
+ config->device_type != type) {
+ // Not interesting for us.
+ continue;
+ }
+
+ // We have a config of the right type, but does it actually support
+ // the pixel format we want? (Seemingly, FFmpeg's way of signaling errors
+ // is to just replace the pixel format with a software-decoded one,
+ // such as yuv420p.)
+ found_config_of_right_type = true;
+ for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
+ if (config->pix_fmt == *fmt_ptr) {
+ fprintf(stderr, "Initialized '%s' hardware decoding for codec '%s'.\n",
+ av_hwdevice_get_type_name(type), ctx->codec->name);
+ if (ctx->profile == FF_PROFILE_H264_BASELINE) {
+ fprintf(stderr, "WARNING: Stream claims to be H.264 Baseline, which is generally poorly supported in hardware decoders.\n");
+ fprintf(stderr, " Consider encoding it as Constrained Baseline, Main or High instead.\n");
+ fprintf(stderr, " Decoding might fail and fall back to software.\n");
+ }
+ return config->pix_fmt;
+ }
+ }
+ fprintf(stderr, "Decoder '%s' supports only these pixel formats:", ctx->codec->name);
+ unordered_set<AVPixelFormat> seen;
+ for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
+ if (!seen.count(*fmt_ptr)) {
+ fprintf(stderr, " %s", av_get_pix_fmt_name(*fmt_ptr));
+ seen.insert(*fmt_ptr);
+ }
+ }
+ fprintf(stderr, " (wanted %s for hardware acceleration)\n", av_get_pix_fmt_name(config->pix_fmt));
+
+ }
+
+ if (!found_config_of_right_type) {
+ fprintf(stderr, "Decoder '%s' does not support device type '%s'.\n", ctx->codec->name, av_hwdevice_get_type_name(type));
+ }
+
+ // We found no VA-API formats, so take the first software format.
+ for (const AVPixelFormat *fmt_ptr = fmt; *fmt_ptr != -1; ++fmt_ptr) {
+ if ((av_pix_fmt_desc_get(*fmt_ptr)->flags & AV_PIX_FMT_FLAG_HWACCEL) == 0) {
+ fprintf(stderr, "Falling back to software format %s.\n", av_get_pix_fmt_name(*fmt_ptr));
+ return *fmt_ptr;
+ }
+ }
+
+ // Fallback: Just return anything. (Should never really happen.)
+ return fmt[0];
+}
+
+AVFrameWithDeleter VideoWidget::decode_frame(AVFormatContext *format_ctx, AVCodecContext *video_codec_ctx,
+ const std::string &pathname, int video_stream_index,
+ bool *error)
+{
+ *error = false;
+
+ if (!queued_frames.empty()) {
+ AVFrameWithDeleter frame = std::move(queued_frames.front());
+ queued_frames.pop_front();
+ return frame;
+ }
+
+ // Read packets until we have a frame or there are none left.
+ bool frame_finished = false;
+ AVFrameWithDeleter video_avframe = av_frame_alloc_unique();
+ bool eof = false;
+ do {
+ AVPacket pkt;
+ unique_ptr<AVPacket, decltype(av_packet_unref)*> pkt_cleanup(
+ &pkt, av_packet_unref);
+ av_init_packet(&pkt);
+ pkt.data = nullptr;
+ pkt.size = 0;
+ if (av_read_frame(format_ctx, &pkt) == 0) {
+ if (pkt.stream_index == video_stream_index) {
+ if (avcodec_send_packet(video_codec_ctx, &pkt) < 0) {
+ fprintf(stderr, "%s: Cannot send packet to video codec.\n", pathname.c_str());
+ *error = true;
+ return AVFrameWithDeleter(nullptr);
+ }
+ }
+ } else {
+ eof = true; // Or error, but ignore that for the time being.
+ }
+
+ // Decode video, if we have a frame.
+ int err = avcodec_receive_frame(video_codec_ctx, video_avframe.get());
+ if (err == 0) {
+ frame_finished = true;
+ break;
+ } else if (err != AVERROR(EAGAIN)) {
+ fprintf(stderr, "%s: Cannot receive frame from video codec.\n", pathname.c_str());
+ *error = true;
+ return AVFrameWithDeleter(nullptr);
+ }
+ } while (!eof);
+
+ if (frame_finished)
+ return video_avframe;
+ else
+ return AVFrameWithDeleter(nullptr);
+}
+
+int find_stream_index(AVFormatContext *ctx, AVMediaType media_type)
+{
+ for (unsigned i = 0; i < ctx->nb_streams; ++i) {
+ if (ctx->streams[i]->codecpar->codec_type == media_type) {
+ return i;
+ }
+ }
+ return -1;
+}
+
+steady_clock::time_point compute_frame_start(int64_t frame_pts, int64_t pts_origin, const AVRational &video_timebase, const steady_clock::time_point &origin, double rate)
+{
+ const duration<double> pts((frame_pts - pts_origin) * double(video_timebase.num) / double(video_timebase.den));
+ return origin + duration_cast<steady_clock::duration>(pts / rate);
+}
+
+bool VideoWidget::play_video(const string &pathname)
+{
+ queued_frames.clear();
+ AVFormatContextWithCloser format_ctx = avformat_open_input_unique(pathname.c_str(), /*fmt=*/nullptr,
+ /*options=*/nullptr);
+ if (format_ctx == nullptr) {
+ fprintf(stderr, "%s: Error opening file\n", pathname.c_str());
+ return false;
+ }
+
+ if (avformat_find_stream_info(format_ctx.get(), nullptr) < 0) {
+ fprintf(stderr, "%s: Error finding stream info\n", pathname.c_str());
+ return false;
+ }
+
+ int video_stream_index = find_stream_index(format_ctx.get(), AVMEDIA_TYPE_VIDEO);
+ if (video_stream_index == -1) {
+ fprintf(stderr, "%s: No video stream found\n", pathname.c_str());
+ return false;
+ }
+
+ // Open video decoder.
+ const AVCodecParameters *video_codecpar = format_ctx->streams[video_stream_index]->codecpar;
+ const AVCodec *video_codec = avcodec_find_decoder(video_codecpar->codec_id);
+
+ video_timebase = format_ctx->streams[video_stream_index]->time_base;
+ AVCodecContextWithDeleter video_codec_ctx = avcodec_alloc_context3_unique(nullptr);
+ if (avcodec_parameters_to_context(video_codec_ctx.get(), video_codecpar) < 0) {
+ fprintf(stderr, "%s: Cannot fill video codec parameters\n", pathname.c_str());
+ return false;
+ }
+ if (video_codec == nullptr) {
+ fprintf(stderr, "%s: Cannot find video decoder\n", pathname.c_str());
+ return false;
+ }
+
+ // Seemingly, it's not too easy to make something that just initializes
+ // “whatever goes”, so we don't get CUDA or VULKAN or whatever here
+ // without enumerating through several different types.
+ // VA-API and VDPAU will do for now. We prioritize VDPAU for the
+ // simple reason that there's a VA-API-via-VDPAU emulation for NVidia
+ // cards that seems to work, but just hangs when trying to transfer the frame.
+ //
+ // Note that we don't actually check codec support beforehand,
+ // so if you have a low-end VDPAU device but a high-end VA-API device,
+ // you lose out on the extra codec support from the latter.
+ AVBufferRef *hw_device_ctx = nullptr;
+ if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VDPAU, nullptr, nullptr, 0) >= 0) {
+ video_codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
+ video_codec_ctx->get_format = get_hw_format<AV_HWDEVICE_TYPE_VDPAU>;
+ } else if (av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_VAAPI, nullptr, nullptr, 0) >= 0) {
+ video_codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
+ video_codec_ctx->get_format = get_hw_format<AV_HWDEVICE_TYPE_VAAPI>;
+ } else {
+ fprintf(stderr, "Failed to initialize VA-API or VDPAU for FFmpeg acceleration. Decoding video in software.\n");
+ }
+
+ if (avcodec_open2(video_codec_ctx.get(), video_codec, nullptr) < 0) {
+ fprintf(stderr, "%s: Cannot open video decoder\n", pathname.c_str());
+ return false;
+ }
+ unique_ptr<AVCodecContext, decltype(avcodec_close)*> video_codec_ctx_cleanup(
+ video_codec_ctx.get(), avcodec_close);
+
+ internal_rewind();
+
+ // Main loop.
+ int consecutive_errors = 0;
+ double rate = 1.0;
+ while (!producer_thread_should_quit.should_quit()) {
+ if (process_queued_commands(format_ctx.get(), video_codec_ctx.get(), video_stream_index, /*seeked=*/nullptr)) {
+ return true;
+ }
+ if (paused) {
+ producer_thread_should_quit.sleep_for(hours(1));
+ continue;
+ }
+
+ bool error;
+ AVFrameWithDeleter frame = decode_frame(format_ctx.get(), video_codec_ctx.get(),
+ pathname, video_stream_index, &error);
+ if (error) {
+ if (++consecutive_errors >= 100) {
+ fprintf(stderr, "More than 100 consecutive video frames, aborting playback.\n");
+ return false;
+ } else {
+ continue;
+ }
+ } else {
+ consecutive_errors = 0;
+ }
+ if (frame == nullptr) {
+ // EOF.
+ return false;
+ }
+
+ // Sleep until it's time to present this frame.
+ for ( ;; ) {
+ if (last_pts == 0 && pts_origin == 0) {
+ pts_origin = frame->pts;
+ }
+ steady_clock::time_point now = steady_clock::now();
+ next_frame_start = compute_frame_start(frame->pts, pts_origin, video_timebase, start, rate);
+
+ if (duration<double>(now - next_frame_start).count() >= 0.1) {
+ // If we don't have enough CPU to keep up, or if we have a live stream
+ // where the initial origin was somehow wrong, we could be behind indefinitely.
+ fprintf(stderr, "%s: Playback %.0f ms behind, resetting time scale\n",
+ pathname.c_str(),
+ 1e3 * duration<double>(now - next_frame_start).count());
+ pts_origin = frame->pts;
+ start = next_frame_start = now;
+ }
+ bool finished_wakeup;
+ finished_wakeup = producer_thread_should_quit.sleep_until(next_frame_start);
+ if (finished_wakeup) {
+ current_frame.reset(new Frame(make_video_frame(frame.get())));
+ last_frame = steady_clock::now();
+ update();
+ break;
+ } else {
+ if (producer_thread_should_quit.should_quit()) break;
+
+ bool seeked = false;
+ if (process_queued_commands(format_ctx.get(), video_codec_ctx.get(), video_stream_index, &seeked)) {
+ return true;
+ }
+
+ if (paused) {
+ // Just paused, so present the frame immediately and then go into deep sleep.
+ current_frame.reset(new Frame(make_video_frame(frame.get())));
+ last_frame = steady_clock::now();
+ update();
+ break;
+ }
+
+ // If we just seeked, drop this frame on the floor and be done.
+ if (seeked) {
+ break;
+ }
+ }
+ }
+ store_pts(frame->pts);
+ }
+ return true;
+}
+
+void VideoWidget::store_pts(int64_t pts)
+{
+ last_pts = pts;
+ last_position = lrint(pts * double(video_timebase.num) / double(video_timebase.den) * 1000);
+ emit position_changed(last_position);
+}
+
+// Taken from Movit (see the comment there for explanation)
+float compute_chroma_offset(float pos, unsigned subsampling_factor, unsigned resolution)
+{
+ float local_chroma_pos = (0.5 + pos * (subsampling_factor - 1)) / subsampling_factor;
+ if (fabs(local_chroma_pos - 0.5) < 1e-10) {
+ // x + (-0) can be optimized away freely, as opposed to x + 0.
+ return -0.0;
+ } else {
+ return (0.5 - local_chroma_pos) / resolution;
+ }
+}
+
+VideoWidget::Frame VideoWidget::make_video_frame(const AVFrame *frame)
+{
+ Frame video_frame;
+ AVFrameWithDeleter sw_frame;
+
+ if (frame->format == AV_PIX_FMT_VAAPI ||
+ frame->format == AV_PIX_FMT_VDPAU) {
+ // Get the frame down to the CPU. (TODO: See if we can keep it
+ // on the GPU all the way, since it will be going up again later.
+ // However, this only works if the OpenGL GPU is the same one.)
+ sw_frame = av_frame_alloc_unique();
+ int err = av_hwframe_transfer_data(sw_frame.get(), frame, 0);
+ if (err != 0) {
+ fprintf(stderr, "%s: Cannot transfer hardware video frame to software.\n", pathname.c_str());
+ } else {
+ sw_frame->pts = frame->pts;
+ sw_frame->pkt_duration = frame->pkt_duration;
+ frame = sw_frame.get();
+ }
+ }
+
+ if (sws_ctx == nullptr ||
+ sws_last_width != frame->width ||
+ sws_last_height != frame->height ||
+ sws_last_src_format != frame->format) {
+ sws_dst_format = decide_dst_format(AVPixelFormat(frame->format));
+ sws_ctx.reset(
+ sws_getContext(frame->width, frame->height, AVPixelFormat(frame->format),
+ frame->width, frame->height, sws_dst_format,
+ SWS_BICUBIC, nullptr, nullptr, nullptr));
+ sws_last_width = frame->width;
+ sws_last_height = frame->height;
+ sws_last_src_format = frame->format;
+ }
+ if (sws_ctx == nullptr) {
+ fprintf(stderr, "Could not create scaler context\n");
+ abort();
+ }
+
+ uint8_t *pic_data[4] = { nullptr, nullptr, nullptr, nullptr };
+ int linesizes[4] = { 0, 0, 0, 0 };
+ const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(sws_dst_format);
+
+ video_frame.width = frame->width;
+ video_frame.height = frame->height;
+ video_frame.chroma_width = AV_CEIL_RSHIFT(int(frame->width), desc->log2_chroma_w);
+ video_frame.chroma_height = AV_CEIL_RSHIFT(int(frame->height), desc->log2_chroma_h);
+
+ // We always assume left chroma placement for now.
+ cbcr_offset[0] = compute_chroma_offset(0.0f, 1 << desc->log2_chroma_w, video_frame.chroma_width);
+ cbcr_offset[1] = compute_chroma_offset(0.5f, 1 << desc->log2_chroma_h, video_frame.chroma_height);
+
+ size_t len = frame->width * frame->height + 2 * video_frame.chroma_width * video_frame.chroma_height;
+ video_frame.data.reset(new uint8_t[len]);
+
+ pic_data[0] = video_frame.data.get();
+ linesizes[0] = frame->width;
+
+ pic_data[1] = pic_data[0] + frame->width * frame->height;
+ linesizes[1] = video_frame.chroma_width;
+
+ pic_data[2] = pic_data[1] + video_frame.chroma_width * video_frame.chroma_height;
+ linesizes[2] = video_frame.chroma_width;
+
+ sws_scale(sws_ctx.get(), frame->data, frame->linesize, 0, frame->height, pic_data, linesizes);
+
+ return video_frame;
+}
+
projects / pkanalytics / commitdiff