X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=h264encode.cpp;h=51ad5178704922c56dd474db247405077e45aae3;hb=d7bba4abb3d56893399578f0540b9ded0a28380f;hp=db0ba93588fc72f9061e491c694910bc0f4f3180;hpb=d147175aaa3e5abfe271940c4518e9ef8dbbc932;p=nageru diff --git a/h264encode.cpp b/h264encode.cpp index db0ba93..51ad517 100644 --- a/h264encode.cpp +++ b/h264encode.cpp @@ -7,11 +7,11 @@ #include #include #include -#include -#include +#include +#include #include +#include #include -#include #include #include #include @@ -20,12 +20,17 @@ #include #include #include +#include +#include #include #include #include #include +#include #include "context.h" +#include "httpd.h" +#include "timebase.h" class QOpenGLContext; class QSurface; @@ -103,7 +108,6 @@ static int h264_maxref = (1<<16|1); static int h264_entropy_mode = 1; /* cabac */ static char *coded_fn = NULL; -static FILE *coded_fp = NULL; static int frame_width = 176; static int frame_height = 144; @@ -118,7 +122,7 @@ static int initial_qp = 15; static int minimal_qp = 0; static int intra_period = 30; static int intra_idr_period = 60; -static int ip_period = 1; +static int ip_period = 3; static int rc_mode = -1; static int rc_default_modes[] = { VA_RC_VBR, @@ -339,12 +343,22 @@ static void sps_rbsp(bitstream *bs) bitstream_put_ui(bs, 1, 1); /* vui_parameters_present_flag */ bitstream_put_ui(bs, 0, 1); /* aspect_ratio_info_present_flag */ bitstream_put_ui(bs, 0, 1); /* overscan_info_present_flag */ - bitstream_put_ui(bs, 0, 1); /* video_signal_type_present_flag */ + bitstream_put_ui(bs, 1, 1); /* video_signal_type_present_flag */ + { + bitstream_put_ui(bs, 5, 3); /* video_format (5 = Unspecified) */ + bitstream_put_ui(bs, 0, 1); /* video_full_range_flag */ + bitstream_put_ui(bs, 1, 1); /* colour_description_present_flag */ + { + bitstream_put_ui(bs, 1, 8); /* colour_primaries (1 = BT.709) */ + bitstream_put_ui(bs, 2, 8); /* transfer_characteristics (2 = unspecified, since we use sRGB) */ + bitstream_put_ui(bs, 6, 8); /* matrix_coefficients (6 = BT.601/SMPTE 170M) */ + } + } bitstream_put_ui(bs, 0, 1); /* chroma_loc_info_present_flag */ bitstream_put_ui(bs, 1, 1); /* timing_info_present_flag */ { bitstream_put_ui(bs, 1, 32); // FPS - bitstream_put_ui(bs, frame_rate * 2, 32); // FPS + bitstream_put_ui(bs, TIMEBASE * 2, 32); // FPS bitstream_put_ui(bs, 1, 1); } bitstream_put_ui(bs, 1, 1); /* nal_hrd_parameters_present_flag */ @@ -586,11 +600,51 @@ static int build_packed_slice_buffer(unsigned char **header_buffer) {IDR(PBB)(PBB)}. */ -/* - * Return displaying order with specified periods and encoding order - * displaying_order: displaying order - * frame_type: frame type - */ +// General pts/dts strategy: +// +// Getting pts and dts right with variable frame rate (VFR) and B-frames can be a +// bit tricky. We assume first of all that the frame rate never goes _above_ +// , which gives us a frame period N. The decoder can always decode +// in at least this speed, as long at dts <= pts (the frame is not attempted +// presented before it is decoded). Furthermore, we never have longer chains of +// B-frames than a fixed constant C. (In a B-frame chain, we say that the base +// I/P-frame has order O=0, the B-frame depending on it directly has order O=1, +// etc. The last frame in the chain, which no B-frames depend on, is the “tip” +// frame, with an order O <= C.) +// +// Many strategies are possible, but we establish these rules: +// +// - Tip frames have dts = pts - (C-O)*N. +// - Non-tip frames have dts = dts_last + N. +// +// An example, with C=2 and N=10 and the data flow showed with arrows: +// +// I B P B B P +// pts: 30 40 50 60 70 80 +// ↓ ↓ ↓ +// dts: 10 30 20 60 50←40 +// | | ↑ ↑ +// `--|--' | +// `----------' +// +// To show that this works fine also with irregular spacings, let's say that +// the third frame is delayed a bit (something earlier was dropped). Now the +// situation looks like this: +// +// I B P B B P +// pts: 30 40 80 90 100 110 +// ↓ ↓ ↓ +// dts: 10 30 20 90 50←40 +// | | ↑ ↑ +// `--|--' | +// `----------' +// +// The resetting on every tip frame makes sure dts never ends up lagging a lot +// behind pts, and the subtraction of (C-O)*N makes sure pts <= dts. +// +// In the output of this function, if is >= 0, it means to reset the +// dts from the current pts minus , while if it's -1, the frame is not +// a tip frame and should be given a dts based on the previous one. #define FRAME_P 0 #define FRAME_B 1 #define FRAME_I 2 @@ -599,10 +653,12 @@ void encoding2display_order( unsigned long long encoding_order, int intra_period, int intra_idr_period, int ip_period, unsigned long long *displaying_order, - int *frame_type) + int *frame_type, int *pts_lag) { int encoding_order_gop = 0; + *pts_lag = 0; + if (intra_period == 1) { /* all are I/IDR frames */ *displaying_order = encoding_order; if (intra_idr_period == 0) @@ -615,29 +671,47 @@ void encoding2display_order( if (intra_period == 0) intra_idr_period = 0; - /* new sequence like - * IDR PPPPP IPPPPP - * IDR (PBB)(PBB)(IBB)(PBB) - */ - encoding_order_gop = (intra_idr_period == 0)? encoding_order: - (encoding_order % (intra_idr_period + ((ip_period == 1)?0:1))); + if (ip_period == 1) { + // No B-frames, sequence is like IDR PPPPP IPPPPP. + encoding_order_gop = (intra_idr_period == 0) ? encoding_order : (encoding_order % intra_idr_period); + *displaying_order = encoding_order; + + if (encoding_order_gop == 0) { /* the first frame */ + *frame_type = FRAME_IDR; + } else if (intra_period != 0 && /* have I frames */ + encoding_order_gop >= 2 && + (encoding_order_gop % intra_period == 0)) { + *frame_type = FRAME_I; + } else { + *frame_type = FRAME_P; + } + return; + } + + // We have B-frames. Sequence is like IDR (PBB)(PBB)(IBB)(PBB). + encoding_order_gop = (intra_idr_period == 0) ? encoding_order : (encoding_order % (intra_idr_period + 1)); + *pts_lag = -1; // Most frames are not tip frames. if (encoding_order_gop == 0) { /* the first frame */ *frame_type = FRAME_IDR; *displaying_order = encoding_order; + // IDR frames are a special case; I honestly can't find the logic behind + // why this is the right thing, but it seems to line up nicely in practice :-) + *pts_lag = TIMEBASE / frame_rate; } else if (((encoding_order_gop - 1) % ip_period) != 0) { /* B frames */ - *frame_type = FRAME_B; + *frame_type = FRAME_B; *displaying_order = encoding_order - 1; - } else if ((intra_period != 0) && /* have I frames */ - (encoding_order_gop >= 2) && - ((ip_period == 1 && encoding_order_gop % intra_period == 0) || /* for IDR PPPPP IPPPP */ - /* for IDR (PBB)(PBB)(IBB) */ - (ip_period >= 2 && ((encoding_order_gop - 1) / ip_period % (intra_period / ip_period)) == 0))) { - *frame_type = FRAME_I; - *displaying_order = encoding_order + ip_period - 1; + if ((encoding_order_gop % ip_period) == 0) { + *pts_lag = 0; // Last B-frame. + } + } else if (intra_period != 0 && /* have I frames */ + encoding_order_gop >= 2 && + ((encoding_order_gop - 1) / ip_period % (intra_period / ip_period)) == 0) { + *frame_type = FRAME_I; + *displaying_order = encoding_order + ip_period - 1; } else { - *frame_type = FRAME_P; - *displaying_order = encoding_order + ip_period - 1; + *frame_type = FRAME_P; + *displaying_order = encoding_order + ip_period - 1; } } @@ -796,12 +870,6 @@ static int process_cmdline(int argc, char *argv[]) coded_fn = strdup("./test.264"); } - /* store coded data into a file */ - coded_fp = fopen(coded_fn, "w+"); - if (coded_fp == NULL) { - printf("Open file %s failed, exit\n", coded_fn); - exit(1); - } frame_width_mbaligned = (frame_width + 15) & (~15); frame_height_mbaligned = (frame_height + 15) & (~15); @@ -1259,7 +1327,7 @@ static int render_sequence(void) seq_param.max_num_ref_frames = num_ref_frames; seq_param.seq_fields.bits.frame_mbs_only_flag = 1; - seq_param.time_scale = frame_rate * 2; + seq_param.time_scale = TIMEBASE * 2; seq_param.num_units_in_tick = 1; /* Tc = num_units_in_tick / scale */ seq_param.seq_fields.bits.log2_max_pic_order_cnt_lsb_minus4 = Log2MaxPicOrderCntLsb - 4; seq_param.seq_fields.bits.log2_max_frame_num_minus4 = Log2MaxFrameNum - 4;; @@ -1576,7 +1644,7 @@ static int render_slice(void) -int H264Encoder::save_codeddata(unsigned long long display_order, unsigned long long encode_order, int frame_type) +int H264Encoder::save_codeddata(storage_task task) { VACodedBufferSegment *buf_list = NULL; VAStatus va_status; @@ -1584,33 +1652,74 @@ int H264Encoder::save_codeddata(unsigned long long display_order, unsigned long string data; - va_status = vaMapBuffer(va_dpy, gl_surfaces[display_order % SURFACE_NUM].coded_buf, (void **)(&buf_list)); + const int64_t global_delay = (ip_period - 1) * (TIMEBASE / frame_rate); // So we never get negative dts. + + va_status = vaMapBuffer(va_dpy, gl_surfaces[task.display_order % SURFACE_NUM].coded_buf, (void **)(&buf_list)); CHECK_VASTATUS(va_status, "vaMapBuffer"); while (buf_list != NULL) { data.append(reinterpret_cast(buf_list->buf), buf_list->size); - if (coded_fp != nullptr) - coded_size += fwrite(buf_list->buf, 1, buf_list->size, coded_fp); buf_list = (VACodedBufferSegment *) buf_list->next; frame_size += coded_size; } - vaUnmapBuffer(va_dpy, gl_surfaces[display_order % SURFACE_NUM].coded_buf); - - AVPacket pkt; - memset(&pkt, 0, sizeof(pkt)); - pkt.buf = nullptr; - pkt.pts = av_rescale_q(display_order, AVRational{1, frame_rate}, avstream->time_base); - pkt.dts = av_rescale_q(encode_order, AVRational{1, frame_rate}, avstream->time_base); - pkt.data = reinterpret_cast(&data[0]); - pkt.size = data.size(); - pkt.stream_index = 0; - if (frame_type == FRAME_IDR || frame_type == FRAME_I) { - pkt.flags = AV_PKT_FLAG_KEY; - } else { - pkt.flags = 0; + vaUnmapBuffer(va_dpy, gl_surfaces[task.display_order % SURFACE_NUM].coded_buf); + + { + // Add video. + AVPacket pkt; + memset(&pkt, 0, sizeof(pkt)); + pkt.buf = nullptr; + pkt.data = reinterpret_cast(&data[0]); + pkt.size = data.size(); + pkt.stream_index = 0; + if (task.frame_type == FRAME_IDR || task.frame_type == FRAME_I) { + pkt.flags = AV_PKT_FLAG_KEY; + } else { + pkt.flags = 0; + } + //pkt.duration = 1; + httpd->add_packet(pkt, task.pts + global_delay, task.dts + global_delay); + } + // Encode and add all audio frames up to and including the pts of this video frame. + // (They can never be queued to us after the video frame they belong to, only before.) + for ( ;; ) { + int64_t audio_pts; + std::vector audio; + { + unique_lock lock(frame_queue_mutex); + if (pending_audio_frames.empty()) break; + auto it = pending_audio_frames.begin(); + if (it->first > task.pts) break; + audio_pts = it->first; + audio = move(it->second); + pending_audio_frames.erase(it); + } + + AVFrame *frame = avcodec_alloc_frame(); + frame->nb_samples = audio.size() / 2; + frame->format = AV_SAMPLE_FMT_FLT; + frame->channel_layout = AV_CH_LAYOUT_STEREO; + + unique_ptr planar_samples(new float[audio.size()]); + avcodec_fill_audio_frame(frame, 2, AV_SAMPLE_FMT_FLTP, (const uint8_t*)planar_samples.get(), audio.size() * sizeof(float), 0); + for (int i = 0; i < frame->nb_samples; ++i) { + planar_samples[i] = audio[i * 2 + 0]; + planar_samples[i + frame->nb_samples] = audio[i * 2 + 1]; + } + + AVPacket pkt; + av_init_packet(&pkt); + pkt.data = nullptr; + pkt.size = 0; + int got_output; + avcodec_encode_audio2(context_audio, &pkt, frame, &got_output); + if (got_output) { + pkt.stream_index = 1; + httpd->add_packet(pkt, audio_pts + global_delay, audio_pts + global_delay); + } + // TODO: Delayed frames. + avcodec_free_frame(&frame); } - pkt.duration = 1; - av_interleaved_write_frame(avctx, &pkt); #if 0 printf("\r "); /* return back to startpoint */ @@ -1637,17 +1746,11 @@ int H264Encoder::save_codeddata(unsigned long long display_order, unsigned long // this is weird. but it seems to put a new frame onto the queue -void H264Encoder::storage_task_enqueue(unsigned long long display_order, unsigned long long encode_order, int frame_type) +void H264Encoder::storage_task_enqueue(storage_task task) { std::unique_lock lock(storage_task_queue_mutex); - - storage_task tmp; - tmp.display_order = display_order; - tmp.encode_order = encode_order; - tmp.frame_type = frame_type; - storage_task_queue.push(tmp); - srcsurface_status[display_order % SURFACE_NUM] = SRC_SURFACE_IN_ENCODING; - + storage_task_queue.push(move(task)); + srcsurface_status[task.display_order % SURFACE_NUM] = SRC_SURFACE_IN_ENCODING; storage_task_queue_changed.notify_all(); } @@ -1660,7 +1763,7 @@ void H264Encoder::storage_task_thread() std::unique_lock lock(storage_task_queue_mutex); storage_task_queue_changed.wait(lock, [this]{ return storage_thread_should_quit || !storage_task_queue.empty(); }); if (storage_thread_should_quit) return; - current = storage_task_queue.front(); + current = move(storage_task_queue.front()); storage_task_queue.pop(); } @@ -1669,7 +1772,7 @@ void H264Encoder::storage_task_thread() // waits for data, then saves it to disk. va_status = vaSyncSurface(va_dpy, gl_surfaces[current.display_order % SURFACE_NUM].src_surface); CHECK_VASTATUS(va_status, "vaSyncSurface"); - save_codeddata(current.display_order, current.encode_order, current.frame_type); + save_codeddata(move(current)); { std::unique_lock lock(storage_task_queue_mutex); @@ -1726,40 +1829,22 @@ static int print_input() return 0; } - -//H264Encoder::H264Encoder(SDL_Window *window, SDL_GLContext context, int width, int height, const char *output_filename) -H264Encoder::H264Encoder(QSurface *surface, int width, int height, const char *output_filename) - : current_storage_frame(0), surface(surface) - //: width(width), height(height), current_encoding_frame(0) +H264Encoder::H264Encoder(QSurface *surface, int width, int height, HTTPD *httpd) + : current_storage_frame(0), surface(surface), httpd(httpd) { - av_register_all(); - avctx = avformat_alloc_context(); - avctx->oformat = av_guess_format(NULL, output_filename, NULL); - strcpy(avctx->filename, output_filename); - if (avio_open2(&avctx->pb, output_filename, AVIO_FLAG_WRITE, &avctx->interrupt_callback, NULL) < 0) { - fprintf(stderr, "%s: avio_open2() failed\n", output_filename); - exit(1); - } - AVCodec *codec = avcodec_find_encoder(AV_CODEC_ID_H264); - avstream = avformat_new_stream(avctx, codec); - if (avstream == nullptr) { - fprintf(stderr, "%s: avformat_new_stream() failed\n", output_filename); - exit(1); - } - avstream->time_base = AVRational{1, frame_rate}; - avstream->codec->width = width; - avstream->codec->height = height; - avstream->codec->time_base = AVRational{1, frame_rate}; - avstream->codec->ticks_per_frame = 1; // or 2? - - if (avformat_write_header(avctx, NULL) < 0) { - fprintf(stderr, "%s: avformat_write_header() failed\n", output_filename); + AVCodec *codec_audio = avcodec_find_encoder(AV_CODEC_ID_MP3); + context_audio = avcodec_alloc_context3(codec_audio); + context_audio->bit_rate = 256000; + context_audio->sample_rate = 48000; + context_audio->sample_fmt = AV_SAMPLE_FMT_FLTP; + context_audio->channels = 2; + context_audio->channel_layout = AV_CH_LAYOUT_STEREO; + context_audio->time_base = AVRational{1, TIMEBASE}; + if (avcodec_open2(context_audio, codec_audio, NULL) < 0) { + fprintf(stderr, "Could not open codec\n"); exit(1); } - coded_fp = fopen("dump.h264", "wb"); - assert(coded_fp != NULL); - frame_width = width; frame_height = height; frame_width_mbaligned = (frame_width + 15) & (~15); @@ -1811,9 +1896,6 @@ H264Encoder::~H264Encoder() release_encode(); deinit_va(); - - av_write_trailer(avctx); - avformat_free_context(avctx); } bool H264Encoder::begin_frame(GLuint *y_tex, GLuint *cbcr_tex) @@ -1880,21 +1962,34 @@ bool H264Encoder::begin_frame(GLuint *y_tex, GLuint *cbcr_tex) return true; } -void H264Encoder::end_frame(RefCountedGLsync fence, const std::vector &input_frames_to_release) +void H264Encoder::add_audio(int64_t pts, std::vector audio) { { unique_lock lock(frame_queue_mutex); - pending_frames[current_storage_frame++] = PendingFrame{ fence, input_frames_to_release }; + pending_audio_frames[pts] = move(audio); + } + frame_queue_nonempty.notify_one(); +} + + +void H264Encoder::end_frame(RefCountedGLsync fence, int64_t pts, const std::vector &input_frames) +{ + { + unique_lock lock(frame_queue_mutex); + pending_video_frames[current_storage_frame] = PendingFrame{ fence, input_frames, pts }; + ++current_storage_frame; } frame_queue_nonempty.notify_one(); } void H264Encoder::copy_thread_func() { + int64_t last_dts = -1; for ( ;; ) { PendingFrame frame; + int pts_lag; encoding2display_order(current_frame_encoding, intra_period, intra_idr_period, ip_period, - ¤t_frame_display, ¤t_frame_type); + ¤t_frame_display, ¤t_frame_type, &pts_lag); if (current_frame_type == FRAME_IDR) { numShortTerm = 0; current_frame_num = 0; @@ -1903,20 +1998,17 @@ void H264Encoder::copy_thread_func() { unique_lock lock(frame_queue_mutex); - frame_queue_nonempty.wait(lock, [this]{ return copy_thread_should_quit || pending_frames.count(current_frame_display) != 0; }); + frame_queue_nonempty.wait(lock, [this]{ return copy_thread_should_quit || pending_video_frames.count(current_frame_display) != 0; }); if (copy_thread_should_quit) return; - frame = pending_frames[current_frame_display]; - pending_frames.erase(current_frame_display); + frame = move(pending_video_frames[current_frame_display]); + pending_video_frames.erase(current_frame_display); } // Wait for the GPU to be done with the frame. glClientWaitSync(frame.fence.get(), 0, 0); // Release back any input frames we needed to render this frame. - // (Actually, those that were needed one output frame ago.) - for (FrameAllocator::Frame input_frame : frame.input_frames_to_release) { - input_frame.owner->release_frame(input_frame); - } + frame.input_frames.clear(); // Unmap the image. GLSurface *surf = &gl_surfaces[current_frame_display % SURFACE_NUM]; @@ -1949,9 +2041,25 @@ void H264Encoder::copy_thread_func() va_status = vaEndPicture(va_dpy, context_id); CHECK_VASTATUS(va_status, "vaEndPicture"); + // Determine the pts and dts of this frame. + int64_t pts = frame.pts; + int64_t dts; + if (pts_lag == -1) { + assert(last_dts != -1); + dts = last_dts + (TIMEBASE / frame_rate); + } else { + dts = pts - pts_lag; + } + last_dts = dts; + // so now the data is done encoding (well, async job kicked off)... // we send that to the storage thread - storage_task_enqueue(current_frame_display, current_frame_encoding, current_frame_type); + storage_task tmp; + tmp.display_order = current_frame_display; + tmp.frame_type = current_frame_type; + tmp.pts = pts; + tmp.dts = dts; + storage_task_enqueue(move(tmp)); update_ReferenceFrames(); ++current_frame_encoding;