X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=h264encode.cpp;h=51ad5178704922c56dd474db247405077e45aae3;hb=d7bba4abb3d56893399578f0540b9ded0a28380f;hp=487a5d681ae850d3fa01d89e3ad14052118b1457;hpb=169ed7e113610c47394def16857541e20160b254;p=nageru diff --git a/h264encode.cpp b/h264encode.cpp index 487a5d6..51ad517 100644 --- a/h264encode.cpp +++ b/h264encode.cpp @@ -7,11 +7,11 @@ #include #include #include -#include -#include +#include +#include #include +#include #include -#include #include #include #include @@ -20,12 +20,17 @@ #include #include #include +#include +#include #include #include #include #include +#include #include "context.h" +#include "httpd.h" +#include "timebase.h" class QOpenGLContext; class QSurface; @@ -117,7 +122,7 @@ static int initial_qp = 15; static int minimal_qp = 0; static int intra_period = 30; static int intra_idr_period = 60; -static int ip_period = 1; +static int ip_period = 3; static int rc_mode = -1; static int rc_default_modes[] = { VA_RC_VBR, @@ -338,12 +343,22 @@ static void sps_rbsp(bitstream *bs) bitstream_put_ui(bs, 1, 1); /* vui_parameters_present_flag */ bitstream_put_ui(bs, 0, 1); /* aspect_ratio_info_present_flag */ bitstream_put_ui(bs, 0, 1); /* overscan_info_present_flag */ - bitstream_put_ui(bs, 0, 1); /* video_signal_type_present_flag */ + bitstream_put_ui(bs, 1, 1); /* video_signal_type_present_flag */ + { + bitstream_put_ui(bs, 5, 3); /* video_format (5 = Unspecified) */ + bitstream_put_ui(bs, 0, 1); /* video_full_range_flag */ + bitstream_put_ui(bs, 1, 1); /* colour_description_present_flag */ + { + bitstream_put_ui(bs, 1, 8); /* colour_primaries (1 = BT.709) */ + bitstream_put_ui(bs, 2, 8); /* transfer_characteristics (2 = unspecified, since we use sRGB) */ + bitstream_put_ui(bs, 6, 8); /* matrix_coefficients (6 = BT.601/SMPTE 170M) */ + } + } bitstream_put_ui(bs, 0, 1); /* chroma_loc_info_present_flag */ bitstream_put_ui(bs, 1, 1); /* timing_info_present_flag */ { bitstream_put_ui(bs, 1, 32); // FPS - bitstream_put_ui(bs, frame_rate * 2, 32); // FPS + bitstream_put_ui(bs, TIMEBASE * 2, 32); // FPS bitstream_put_ui(bs, 1, 1); } bitstream_put_ui(bs, 1, 1); /* nal_hrd_parameters_present_flag */ @@ -585,11 +600,51 @@ static int build_packed_slice_buffer(unsigned char **header_buffer) {IDR(PBB)(PBB)}. */ -/* - * Return displaying order with specified periods and encoding order - * displaying_order: displaying order - * frame_type: frame type - */ +// General pts/dts strategy: +// +// Getting pts and dts right with variable frame rate (VFR) and B-frames can be a +// bit tricky. We assume first of all that the frame rate never goes _above_ +// , which gives us a frame period N. The decoder can always decode +// in at least this speed, as long at dts <= pts (the frame is not attempted +// presented before it is decoded). Furthermore, we never have longer chains of +// B-frames than a fixed constant C. (In a B-frame chain, we say that the base +// I/P-frame has order O=0, the B-frame depending on it directly has order O=1, +// etc. The last frame in the chain, which no B-frames depend on, is the “tip” +// frame, with an order O <= C.) +// +// Many strategies are possible, but we establish these rules: +// +// - Tip frames have dts = pts - (C-O)*N. +// - Non-tip frames have dts = dts_last + N. +// +// An example, with C=2 and N=10 and the data flow showed with arrows: +// +// I B P B B P +// pts: 30 40 50 60 70 80 +// ↓ ↓ ↓ +// dts: 10 30 20 60 50←40 +// | | ↑ ↑ +// `--|--' | +// `----------' +// +// To show that this works fine also with irregular spacings, let's say that +// the third frame is delayed a bit (something earlier was dropped). Now the +// situation looks like this: +// +// I B P B B P +// pts: 30 40 80 90 100 110 +// ↓ ↓ ↓ +// dts: 10 30 20 90 50←40 +// | | ↑ ↑ +// `--|--' | +// `----------' +// +// The resetting on every tip frame makes sure dts never ends up lagging a lot +// behind pts, and the subtraction of (C-O)*N makes sure pts <= dts. +// +// In the output of this function, if is >= 0, it means to reset the +// dts from the current pts minus , while if it's -1, the frame is not +// a tip frame and should be given a dts based on the previous one. #define FRAME_P 0 #define FRAME_B 1 #define FRAME_I 2 @@ -598,10 +653,12 @@ void encoding2display_order( unsigned long long encoding_order, int intra_period, int intra_idr_period, int ip_period, unsigned long long *displaying_order, - int *frame_type) + int *frame_type, int *pts_lag) { int encoding_order_gop = 0; + *pts_lag = 0; + if (intra_period == 1) { /* all are I/IDR frames */ *displaying_order = encoding_order; if (intra_idr_period == 0) @@ -614,29 +671,47 @@ void encoding2display_order( if (intra_period == 0) intra_idr_period = 0; - /* new sequence like - * IDR PPPPP IPPPPP - * IDR (PBB)(PBB)(IBB)(PBB) - */ - encoding_order_gop = (intra_idr_period == 0)? encoding_order: - (encoding_order % (intra_idr_period + ((ip_period == 1)?0:1))); + if (ip_period == 1) { + // No B-frames, sequence is like IDR PPPPP IPPPPP. + encoding_order_gop = (intra_idr_period == 0) ? encoding_order : (encoding_order % intra_idr_period); + *displaying_order = encoding_order; + + if (encoding_order_gop == 0) { /* the first frame */ + *frame_type = FRAME_IDR; + } else if (intra_period != 0 && /* have I frames */ + encoding_order_gop >= 2 && + (encoding_order_gop % intra_period == 0)) { + *frame_type = FRAME_I; + } else { + *frame_type = FRAME_P; + } + return; + } + + // We have B-frames. Sequence is like IDR (PBB)(PBB)(IBB)(PBB). + encoding_order_gop = (intra_idr_period == 0) ? encoding_order : (encoding_order % (intra_idr_period + 1)); + *pts_lag = -1; // Most frames are not tip frames. if (encoding_order_gop == 0) { /* the first frame */ *frame_type = FRAME_IDR; *displaying_order = encoding_order; + // IDR frames are a special case; I honestly can't find the logic behind + // why this is the right thing, but it seems to line up nicely in practice :-) + *pts_lag = TIMEBASE / frame_rate; } else if (((encoding_order_gop - 1) % ip_period) != 0) { /* B frames */ - *frame_type = FRAME_B; + *frame_type = FRAME_B; *displaying_order = encoding_order - 1; - } else if ((intra_period != 0) && /* have I frames */ - (encoding_order_gop >= 2) && - ((ip_period == 1 && encoding_order_gop % intra_period == 0) || /* for IDR PPPPP IPPPP */ - /* for IDR (PBB)(PBB)(IBB) */ - (ip_period >= 2 && ((encoding_order_gop - 1) / ip_period % (intra_period / ip_period)) == 0))) { - *frame_type = FRAME_I; - *displaying_order = encoding_order + ip_period - 1; + if ((encoding_order_gop % ip_period) == 0) { + *pts_lag = 0; // Last B-frame. + } + } else if (intra_period != 0 && /* have I frames */ + encoding_order_gop >= 2 && + ((encoding_order_gop - 1) / ip_period % (intra_period / ip_period)) == 0) { + *frame_type = FRAME_I; + *displaying_order = encoding_order + ip_period - 1; } else { - *frame_type = FRAME_P; - *displaying_order = encoding_order + ip_period - 1; + *frame_type = FRAME_P; + *displaying_order = encoding_order + ip_period - 1; } } @@ -1252,7 +1327,7 @@ static int render_sequence(void) seq_param.max_num_ref_frames = num_ref_frames; seq_param.seq_fields.bits.frame_mbs_only_flag = 1; - seq_param.time_scale = frame_rate * 2; + seq_param.time_scale = TIMEBASE * 2; seq_param.num_units_in_tick = 1; /* Tc = num_units_in_tick / scale */ seq_param.seq_fields.bits.log2_max_pic_order_cnt_lsb_minus4 = Log2MaxPicOrderCntLsb - 4; seq_param.seq_fields.bits.log2_max_frame_num_minus4 = Log2MaxFrameNum - 4;; @@ -1577,6 +1652,8 @@ int H264Encoder::save_codeddata(storage_task task) string data; + const int64_t global_delay = (ip_period - 1) * (TIMEBASE / frame_rate); // So we never get negative dts. + va_status = vaMapBuffer(va_dpy, gl_surfaces[task.display_order % SURFACE_NUM].coded_buf, (void **)(&buf_list)); CHECK_VASTATUS(va_status, "vaMapBuffer"); while (buf_list != NULL) { @@ -1587,15 +1664,11 @@ int H264Encoder::save_codeddata(storage_task task) } vaUnmapBuffer(va_dpy, gl_surfaces[task.display_order % SURFACE_NUM].coded_buf); - const int pts_dts_delay = ip_period - 1; - const int av_delay = 2; // Corresponds to the fixed delay in resampler.h. TODO: Make less hard-coded. { // Add video. AVPacket pkt; memset(&pkt, 0, sizeof(pkt)); pkt.buf = nullptr; - pkt.pts = av_rescale_q(task.display_order + av_delay + pts_dts_delay, AVRational{1, frame_rate}, avstream_video->time_base); - pkt.dts = av_rescale_q(task.encode_order + av_delay, AVRational{1, frame_rate}, avstream_video->time_base); pkt.data = reinterpret_cast(&data[0]); pkt.size = data.size(); pkt.stream_index = 0; @@ -1605,22 +1678,23 @@ int H264Encoder::save_codeddata(storage_task task) pkt.flags = 0; } //pkt.duration = 1; - av_interleaved_write_frame(avctx, &pkt); + httpd->add_packet(pkt, task.pts + global_delay, task.dts + global_delay); } // Encode and add all audio frames up to and including the pts of this video frame. // (They can never be queued to us after the video frame they belong to, only before.) for ( ;; ) { - int display_order; + int64_t audio_pts; std::vector audio; { unique_lock lock(frame_queue_mutex); if (pending_audio_frames.empty()) break; auto it = pending_audio_frames.begin(); - if (it->first > int(task.display_order)) break; - display_order = it->first; + if (it->first > task.pts) break; + audio_pts = it->first; audio = move(it->second); pending_audio_frames.erase(it); } + AVFrame *frame = avcodec_alloc_frame(); frame->nb_samples = audio.size() / 2; frame->format = AV_SAMPLE_FMT_FLT; @@ -1638,12 +1712,10 @@ int H264Encoder::save_codeddata(storage_task task) pkt.data = nullptr; pkt.size = 0; int got_output; - avcodec_encode_audio2(avstream_audio->codec, &pkt, frame, &got_output); + avcodec_encode_audio2(context_audio, &pkt, frame, &got_output); if (got_output) { - pkt.pts = av_rescale_q(display_order + pts_dts_delay, AVRational{1, frame_rate}, avstream_audio->time_base); // FIXME - pkt.dts = pkt.pts; pkt.stream_index = 1; - av_interleaved_write_frame(avctx, &pkt); + httpd->add_packet(pkt, audio_pts + global_delay, audio_pts + global_delay); } // TODO: Delayed frames. avcodec_free_frame(&frame); @@ -1757,57 +1829,22 @@ static int print_input() return 0; } - -//H264Encoder::H264Encoder(SDL_Window *window, SDL_GLContext context, int width, int height, const char *output_filename) -H264Encoder::H264Encoder(QSurface *surface, int width, int height, const char *output_filename) - : current_storage_frame(0), surface(surface) - //: width(width), height(height), current_encoding_frame(0) +H264Encoder::H264Encoder(QSurface *surface, int width, int height, HTTPD *httpd) + : current_storage_frame(0), surface(surface), httpd(httpd) { - av_register_all(); - avctx = avformat_alloc_context(); - avctx->oformat = av_guess_format(NULL, output_filename, NULL); - strcpy(avctx->filename, output_filename); - if (avio_open2(&avctx->pb, output_filename, AVIO_FLAG_WRITE, &avctx->interrupt_callback, NULL) < 0) { - fprintf(stderr, "%s: avio_open2() failed\n", output_filename); - exit(1); - } - AVCodec *codec_video = avcodec_find_encoder(AV_CODEC_ID_H264); - avstream_video = avformat_new_stream(avctx, codec_video); - if (avstream_video == nullptr) { - fprintf(stderr, "%s: avformat_new_stream() failed\n", output_filename); - exit(1); - } - avstream_video->time_base = AVRational{1, frame_rate}; - avstream_video->codec->width = width; - avstream_video->codec->height = height; - avstream_video->codec->time_base = AVRational{1, frame_rate}; - avstream_video->codec->ticks_per_frame = 1; // or 2? - AVCodec *codec_audio = avcodec_find_encoder(AV_CODEC_ID_MP3); - avstream_audio = avformat_new_stream(avctx, codec_audio); - if (avstream_audio == nullptr) { - fprintf(stderr, "%s: avformat_new_stream() failed\n", output_filename); - exit(1); - } - avstream_audio->time_base = AVRational{1, frame_rate}; - avstream_audio->codec->bit_rate = 256000; - avstream_audio->codec->sample_rate = 48000; - avstream_audio->codec->sample_fmt = AV_SAMPLE_FMT_FLTP; - avstream_audio->codec->channels = 2; - avstream_audio->codec->channel_layout = AV_CH_LAYOUT_STEREO; - avstream_audio->codec->time_base = AVRational{1, frame_rate}; - - /* open it */ - if (avcodec_open2(avstream_audio->codec, codec_audio, NULL) < 0) { + context_audio = avcodec_alloc_context3(codec_audio); + context_audio->bit_rate = 256000; + context_audio->sample_rate = 48000; + context_audio->sample_fmt = AV_SAMPLE_FMT_FLTP; + context_audio->channels = 2; + context_audio->channel_layout = AV_CH_LAYOUT_STEREO; + context_audio->time_base = AVRational{1, TIMEBASE}; + if (avcodec_open2(context_audio, codec_audio, NULL) < 0) { fprintf(stderr, "Could not open codec\n"); exit(1); } - if (avformat_write_header(avctx, NULL) < 0) { - fprintf(stderr, "%s: avformat_write_header() failed\n", output_filename); - exit(1); - } - frame_width = width; frame_height = height; frame_width_mbaligned = (frame_width + 15) & (~15); @@ -1859,9 +1896,6 @@ H264Encoder::~H264Encoder() release_encode(); deinit_va(); - - av_write_trailer(avctx); - avformat_free_context(avctx); } bool H264Encoder::begin_frame(GLuint *y_tex, GLuint *cbcr_tex) @@ -1928,12 +1962,21 @@ bool H264Encoder::begin_frame(GLuint *y_tex, GLuint *cbcr_tex) return true; } -void H264Encoder::end_frame(RefCountedGLsync fence, std::vector audio, const std::vector &input_frames) +void H264Encoder::add_audio(int64_t pts, std::vector audio) +{ + { + unique_lock lock(frame_queue_mutex); + pending_audio_frames[pts] = move(audio); + } + frame_queue_nonempty.notify_one(); +} + + +void H264Encoder::end_frame(RefCountedGLsync fence, int64_t pts, const std::vector &input_frames) { { unique_lock lock(frame_queue_mutex); - pending_video_frames[current_storage_frame] = PendingFrame{ fence, input_frames }; - pending_audio_frames[current_storage_frame] = move(audio); + pending_video_frames[current_storage_frame] = PendingFrame{ fence, input_frames, pts }; ++current_storage_frame; } frame_queue_nonempty.notify_one(); @@ -1941,10 +1984,12 @@ void H264Encoder::end_frame(RefCountedGLsync fence, std::vector audio, co void H264Encoder::copy_thread_func() { + int64_t last_dts = -1; for ( ;; ) { PendingFrame frame; + int pts_lag; encoding2display_order(current_frame_encoding, intra_period, intra_idr_period, ip_period, - ¤t_frame_display, ¤t_frame_type); + ¤t_frame_display, ¤t_frame_type, &pts_lag); if (current_frame_type == FRAME_IDR) { numShortTerm = 0; current_frame_num = 0; @@ -1996,12 +2041,24 @@ void H264Encoder::copy_thread_func() va_status = vaEndPicture(va_dpy, context_id); CHECK_VASTATUS(va_status, "vaEndPicture"); + // Determine the pts and dts of this frame. + int64_t pts = frame.pts; + int64_t dts; + if (pts_lag == -1) { + assert(last_dts != -1); + dts = last_dts + (TIMEBASE / frame_rate); + } else { + dts = pts - pts_lag; + } + last_dts = dts; + // so now the data is done encoding (well, async job kicked off)... // we send that to the storage thread storage_task tmp; tmp.display_order = current_frame_display; - tmp.encode_order = current_frame_encoding; tmp.frame_type = current_frame_type; + tmp.pts = pts; + tmp.dts = dts; storage_task_enqueue(move(tmp)); update_ReferenceFrames();