#include <X11/Xlib.h>
#include <assert.h>
#include <epoxy/egl.h>
+extern "C" {
#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
#include <libavutil/channel_layout.h>
#include <libavutil/frame.h>
#include <libavutil/rational.h>
#include <libavutil/samplefmt.h>
+}
#include <libdrm/drm_fourcc.h>
#include <stdio.h>
#include <stdlib.h>
#include <va/va_x11.h>
#include <condition_variable>
#include <cstdint>
+#include <map>
#include <memory>
#include <mutex>
#include <queue>
VA_RC_VCM,
VA_RC_NONE,
};
-static unsigned long long current_IDR_display = 0;
static unsigned int current_frame_num = 0;
static int misc_priv_type = 0;
using namespace std;
+class H264EncoderImpl {
+public:
+ H264EncoderImpl(QSurface *surface, int width, int height, HTTPD *httpd);
+ ~H264EncoderImpl();
+ void add_audio(int64_t pts, std::vector<float> audio); // Needs to come before end_frame() of same pts.
+ bool begin_frame(GLuint *y_tex, GLuint *cbcr_tex);
+ void end_frame(RefCountedGLsync fence, int64_t pts, const std::vector<RefCountedFrame> &input_frames);
+
+private:
+ struct storage_task {
+ unsigned long long display_order;
+ int frame_type;
+ std::vector<float> audio;
+ int64_t pts, dts;
+ };
+ struct PendingFrame {
+ RefCountedGLsync fence;
+ std::vector<RefCountedFrame> input_frames;
+ int64_t pts;
+ };
+
+ void encode_thread_func();
+ void encode_remaining_frames_as_p(int encoding_frame_num, int gop_start_display_frame_num, int64_t last_dts);
+ void encode_frame(PendingFrame frame, int encoding_frame_num, int display_frame_num, int gop_start_display_frame_num,
+ int frame_type, int64_t pts, int64_t dts);
+ void storage_task_thread();
+ void storage_task_enqueue(storage_task task);
+ void save_codeddata(storage_task task);
+
+ std::thread encode_thread, storage_thread;
+
+ std::mutex storage_task_queue_mutex;
+ std::condition_variable storage_task_queue_changed;
+ int srcsurface_status[SURFACE_NUM]; // protected by storage_task_queue_mutex
+ std::queue<storage_task> storage_task_queue; // protected by storage_task_queue_mutex
+ bool storage_thread_should_quit = false; // protected by storage_task_queue_mutex
+
+ std::mutex frame_queue_mutex;
+ std::condition_variable frame_queue_nonempty;
+ bool encode_thread_should_quit = false; // under frame_queue_mutex
+
+ //int frame_width, frame_height;
+ //int ;
+ int current_storage_frame;
+
+ std::map<int, PendingFrame> pending_video_frames; // under frame_queue_mutex
+ std::map<int64_t, std::vector<float>> pending_audio_frames; // under frame_queue_mutex
+ QSurface *surface;
+
+ AVCodecContext *context_audio;
+ HTTPD *httpd;
+};
+
+
// Supposedly vaRenderPicture() is supposed to destroy the buffer implicitly,
// but if we don't delete it here, we get leaks. The GStreamer implementation
// does the same.
sort_one(ref, j+1, right, list1_ascending, frame_idx);
}
-static int update_ReferenceFrames(int frame_type)
+static void update_ReferenceFrames(int frame_type)
{
int i;
if (frame_type == FRAME_B)
- return 0;
+ return;
CurrentCurrPic.flags = VA_PICTURE_H264_SHORT_TERM_REFERENCE;
numShortTerm++;
current_frame_num++;
if (current_frame_num > MaxFrameNum)
current_frame_num = 0;
-
- return 0;
}
return TopFieldOrderCnt;
}
-static int render_picture(int frame_type, int display_frame_num)
+static int render_picture(int frame_type, int display_frame_num, int gop_start_display_frame_num)
{
VABufferID pic_param_buf;
VAStatus va_status;
pic_param.CurrPic.picture_id = gl_surfaces[display_frame_num % SURFACE_NUM].ref_surface;
pic_param.CurrPic.frame_idx = current_frame_num;
pic_param.CurrPic.flags = 0;
- pic_param.CurrPic.TopFieldOrderCnt = calc_poc((display_frame_num - current_IDR_display) % MaxPicOrderCntLsb, frame_type);
+ pic_param.CurrPic.TopFieldOrderCnt = calc_poc((display_frame_num - gop_start_display_frame_num) % MaxPicOrderCntLsb, frame_type);
pic_param.CurrPic.BottomFieldOrderCnt = pic_param.CurrPic.TopFieldOrderCnt;
CurrentCurrPic = pic_param.CurrPic;
free(packedslice_buffer);
}
-static int render_slice(int encoding_frame_num, int display_frame_num, int frame_type)
+static int render_slice(int encoding_frame_num, int display_frame_num, int gop_start_display_frame_num, int frame_type)
{
VABufferID slice_param_buf;
VAStatus va_status;
slice_param.slice_alpha_c0_offset_div2 = 0;
slice_param.slice_beta_offset_div2 = 0;
slice_param.direct_spatial_mv_pred_flag = 1;
- slice_param.pic_order_cnt_lsb = (display_frame_num - current_IDR_display) % MaxPicOrderCntLsb;
+ slice_param.pic_order_cnt_lsb = (display_frame_num - gop_start_display_frame_num) % MaxPicOrderCntLsb;
if (h264_packedheader &&
-void H264Encoder::save_codeddata(storage_task task)
+void H264EncoderImpl::save_codeddata(storage_task task)
{
VACodedBufferSegment *buf_list = NULL;
VAStatus va_status;
vector<float> audio;
{
unique_lock<mutex> lock(frame_queue_mutex);
- frame_queue_nonempty.wait(lock, [this]{ return copy_thread_should_quit || !pending_audio_frames.empty(); });
- if (copy_thread_should_quit) return;
+ frame_queue_nonempty.wait(lock, [this]{ return storage_thread_should_quit || !pending_audio_frames.empty(); });
+ if (storage_thread_should_quit && pending_audio_frames.empty()) return;
auto it = pending_audio_frames.begin();
if (it->first > task.pts) break;
audio_pts = it->first;
// this is weird. but it seems to put a new frame onto the queue
-void H264Encoder::storage_task_enqueue(storage_task task)
+void H264EncoderImpl::storage_task_enqueue(storage_task task)
{
unique_lock<mutex> lock(storage_task_queue_mutex);
storage_task_queue.push(move(task));
storage_task_queue_changed.notify_all();
}
-void H264Encoder::storage_task_thread()
+void H264EncoderImpl::storage_task_thread()
{
for ( ;; ) {
storage_task current;
// wait until there's an encoded frame
unique_lock<mutex> lock(storage_task_queue_mutex);
storage_task_queue_changed.wait(lock, [this]{ return storage_thread_should_quit || !storage_task_queue.empty(); });
- if (storage_thread_should_quit) return;
+ if (storage_thread_should_quit && storage_task_queue.empty()) return;
current = move(storage_task_queue.front());
storage_task_queue.pop();
}
}
-H264Encoder::H264Encoder(QSurface *surface, int width, int height, HTTPD *httpd)
+H264EncoderImpl::H264EncoderImpl(QSurface *surface, int width, int height, HTTPD *httpd)
: current_storage_frame(0), surface(surface), httpd(httpd)
{
AVCodec *codec_audio = avcodec_find_encoder(AUDIO_OUTPUT_CODEC);
memset(&pic_param, 0, sizeof(pic_param));
memset(&slice_param, 0, sizeof(slice_param));
- storage_thread = thread(&H264Encoder::storage_task_thread, this);
+ storage_thread = thread(&H264EncoderImpl::storage_task_thread, this);
- copy_thread = thread([this]{
+ encode_thread = thread([this]{
//SDL_GL_MakeCurrent(window, context);
QOpenGLContext *context = create_context(this->surface);
eglBindAPI(EGL_OPENGL_API);
eglGetError());
exit(1);
}
- copy_thread_func();
+ encode_thread_func();
});
}
-H264Encoder::~H264Encoder()
+H264EncoderImpl::~H264EncoderImpl()
{
{
- unique_lock<mutex> lock(storage_task_queue_mutex);
- storage_thread_should_quit = true;
- storage_task_queue_changed.notify_all();
+ unique_lock<mutex> lock(frame_queue_mutex);
+ encode_thread_should_quit = true;
+ frame_queue_nonempty.notify_all();
}
+ encode_thread.join();
{
- unique_lock<mutex> lock(frame_queue_mutex);
- copy_thread_should_quit = true;
+ unique_lock<mutex> lock(storage_task_queue_mutex);
+ storage_thread_should_quit = true;
frame_queue_nonempty.notify_all();
+ storage_task_queue_changed.notify_all();
}
storage_thread.join();
- copy_thread.join();
release_encode();
deinit_va();
}
-bool H264Encoder::begin_frame(GLuint *y_tex, GLuint *cbcr_tex)
+bool H264EncoderImpl::begin_frame(GLuint *y_tex, GLuint *cbcr_tex)
{
{
// Wait until this frame slot is done encoding.
return true;
}
-void H264Encoder::add_audio(int64_t pts, vector<float> audio)
+void H264EncoderImpl::add_audio(int64_t pts, vector<float> audio)
{
{
unique_lock<mutex> lock(frame_queue_mutex);
frame_queue_nonempty.notify_all();
}
-void H264Encoder::end_frame(RefCountedGLsync fence, int64_t pts, const vector<RefCountedFrame> &input_frames)
+void H264EncoderImpl::end_frame(RefCountedGLsync fence, int64_t pts, const vector<RefCountedFrame> &input_frames)
{
{
unique_lock<mutex> lock(frame_queue_mutex);
frame_queue_nonempty.notify_all();
}
-void H264Encoder::copy_thread_func()
+void H264EncoderImpl::encode_thread_func()
{
int64_t last_dts = -1;
+ int gop_start_display_frame_num = 0;
for (int encoding_frame_num = 0; ; ++encoding_frame_num) {
PendingFrame frame;
int pts_lag;
if (frame_type == FRAME_IDR) {
numShortTerm = 0;
current_frame_num = 0;
- current_IDR_display = display_frame_num;
+ gop_start_display_frame_num = display_frame_num;
}
{
unique_lock<mutex> lock(frame_queue_mutex);
frame_queue_nonempty.wait(lock, [this, display_frame_num]{
- return copy_thread_should_quit || pending_video_frames.count(display_frame_num) != 0;
+ return encode_thread_should_quit || pending_video_frames.count(display_frame_num) != 0;
});
- if (copy_thread_should_quit) {
+ if (encode_thread_should_quit && pending_video_frames.count(display_frame_num) == 0) {
+ // We have queued frames that were supposed to be B-frames,
+ // but will be no P-frame to encode them against. Encode them all
+ // as P-frames instead. Note that this happens under the mutex,
+ // but nobody else uses it at this point, since we're shutting down,
+ // so there's no contention.
+ encode_remaining_frames_as_p(encoding_frame_num, gop_start_display_frame_num, last_dts);
return;
} else {
frame = move(pending_video_frames[display_frame_num]);
}
last_dts = dts;
- encode_frame(frame, encoding_frame_num, display_frame_num, frame_type, frame.pts, dts);
+ encode_frame(frame, encoding_frame_num, display_frame_num, gop_start_display_frame_num, frame_type, frame.pts, dts);
+ }
+}
+
+void H264EncoderImpl::encode_remaining_frames_as_p(int encoding_frame_num, int gop_start_display_frame_num, int64_t last_dts)
+{
+ if (pending_video_frames.empty()) {
+ return;
+ }
+
+ for (auto &pending_frame : pending_video_frames) {
+ int display_frame_num = pending_frame.first;
+ assert(display_frame_num > 0);
+ PendingFrame frame = move(pending_frame.second);
+ int64_t dts = last_dts + (TIMEBASE / MAX_FPS);
+ printf("Finalizing encode: Encoding leftover frame %d as P-frame instead of B-frame.\n", display_frame_num);
+ encode_frame(frame, encoding_frame_num++, display_frame_num, gop_start_display_frame_num, FRAME_P, frame.pts, dts);
+ last_dts = dts;
}
}
-void H264Encoder::encode_frame(H264Encoder::PendingFrame frame, int encoding_frame_num, int display_frame_num,
- int frame_type, int64_t pts, int64_t dts)
+void H264EncoderImpl::encode_frame(H264EncoderImpl::PendingFrame frame, int encoding_frame_num, int display_frame_num, int gop_start_display_frame_num,
+ int frame_type, int64_t pts, int64_t dts)
{
// Wait for the GPU to be done with the frame.
glClientWaitSync(frame.fence.get(), 0, 0);
if (frame_type == FRAME_IDR) {
render_sequence();
- render_picture(frame_type, display_frame_num);
+ render_picture(frame_type, display_frame_num, gop_start_display_frame_num);
if (h264_packedheader) {
render_packedsequence();
render_packedpicture();
}
} else {
//render_sequence();
- render_picture(frame_type, display_frame_num);
+ render_picture(frame_type, display_frame_num, gop_start_display_frame_num);
}
- render_slice(encoding_frame_num, display_frame_num, frame_type);
+ render_slice(encoding_frame_num, display_frame_num, gop_start_display_frame_num, frame_type);
va_status = vaEndPicture(va_dpy, context_id);
CHECK_VASTATUS(va_status, "vaEndPicture");
update_ReferenceFrames(frame_type);
}
+
+// Proxy object.
+H264Encoder::H264Encoder(QSurface *surface, int width, int height, HTTPD *httpd)
+ : impl(new H264EncoderImpl(surface, width, height, httpd)) {}
+
+// Must be defined here because unique_ptr<> destructor needs to know the impl.
+H264Encoder::~H264Encoder() {}
+
+void H264Encoder::add_audio(int64_t pts, std::vector<float> audio)
+{
+ impl->add_audio(pts, audio);
+}
+
+bool H264Encoder::begin_frame(GLuint *y_tex, GLuint *cbcr_tex)
+{
+ return impl->begin_frame(y_tex, cbcr_tex);
+}
+
+void H264Encoder::end_frame(RefCountedGLsync fence, int64_t pts, const std::vector<RefCountedFrame> &input_frames)
+{
+ impl->end_frame(fence, pts, input_frames);
+}
+
+// Real class.