X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=futatabi%2Fvideo_stream.cpp;h=9647836feaa4294a4653bdb8eb7b84b8af77da27;hb=f014bc9a81cc4fae068172a84f0491efef38135e;hp=745da27486971b13414549033b3270fa00a37174;hpb=e20bf0f86703779f11d575f44173ff73544e1c2d;p=nageru diff --git a/futatabi/video_stream.cpp b/futatabi/video_stream.cpp index 745da27..9647836 100644 --- a/futatabi/video_stream.cpp +++ b/futatabi/video_stream.cpp @@ -13,6 +13,7 @@ extern "C" { #include "player.h" #include "shared/context.h" #include "shared/httpd.h" +#include "shared/shared_defs.h" #include "shared/mux.h" #include "util.h" #include "ycbcr_converter.h" @@ -28,7 +29,7 @@ extern HTTPD *global_httpd; struct VectorDestinationManager { jpeg_destination_mgr pub; - std::vector dest; + string dest; VectorDestinationManager() { @@ -62,7 +63,7 @@ struct VectorDestinationManager { { dest.resize(bytes_used + 4096); dest.resize(dest.capacity()); - pub.next_output_byte = dest.data() + bytes_used; + pub.next_output_byte = (uint8_t *)dest.data() + bytes_used; pub.free_in_buffer = dest.size() - bytes_used; } @@ -78,7 +79,7 @@ struct VectorDestinationManager { }; static_assert(std::is_standard_layout::value, ""); -vector encode_jpeg(const uint8_t *y_data, const uint8_t *cb_data, const uint8_t *cr_data, unsigned width, unsigned height) +string encode_jpeg(const uint8_t *y_data, const uint8_t *cb_data, const uint8_t *cr_data, unsigned width, unsigned height) { VectorDestinationManager dest; @@ -241,6 +242,30 @@ VideoStream::~VideoStream() if (last_flow_tex != 0) { compute_flow->release_texture(last_flow_tex); } + + for (const unique_ptr &resource : interpolate_resources) { + glUnmapNamedBuffer(resource->pbo); + check_error(); + glDeleteBuffers(1, &resource->pbo); + check_error(); + glDeleteFramebuffers(2, resource->input_fbos); + check_error(); + glDeleteFramebuffers(1, &resource->fade_fbo); + check_error(); + glDeleteTextures(1, &resource->input_tex); + check_error(); + glDeleteTextures(1, &resource->gray_tex); + check_error(); + glDeleteTextures(1, &resource->fade_y_output_tex); + check_error(); + glDeleteTextures(1, &resource->fade_cbcr_output_tex); + check_error(); + glDeleteTextures(1, &resource->cb_tex); + check_error(); + glDeleteTextures(1, &resource->cr_tex); + check_error(); + } + assert(interpolate_resources.size() == num_interpolate_slots); } void VideoStream::start() @@ -262,10 +287,19 @@ void VideoStream::start() avctx->flags = AVFMT_FLAG_CUSTOM_IO; } + AVCodecParameters *audio_codecpar = avcodec_parameters_alloc(); + + audio_codecpar->codec_type = AVMEDIA_TYPE_AUDIO; + audio_codecpar->codec_id = AV_CODEC_ID_PCM_S32LE; + audio_codecpar->channel_layout = AV_CH_LAYOUT_STEREO; + audio_codecpar->channels = 2; + audio_codecpar->sample_rate = OUTPUT_FREQUENCY; + size_t width = global_flags.width, height = global_flags.height; // Doesn't matter for MJPEG. - mux.reset(new Mux(avctx, width, height, Mux::CODEC_MJPEG, /*video_extradata=*/"", /*audio_codec_parameters=*/nullptr, + mux.reset(new Mux(avctx, width, height, Mux::CODEC_MJPEG, /*video_extradata=*/"", audio_codecpar, AVCOL_SPC_BT709, COARSE_TIMEBASE, /*write_callback=*/nullptr, Mux::WRITE_FOREGROUND, {}, Mux::WITH_SUBTITLES)); + avcodec_parameters_free(&audio_codecpar); encode_thread = thread(&VideoStream::encode_thread_func, this); } @@ -307,22 +341,20 @@ void VideoStream::clear_queue() void VideoStream::schedule_original_frame(steady_clock::time_point local_pts, int64_t output_pts, function &&display_func, QueueSpotHolder &&queue_spot_holder, - FrameOnDisk frame, const string &subtitle) + FrameOnDisk frame, const string &subtitle, bool include_audio) { - fprintf(stderr, "output_pts=%ld original input_pts=%ld\n", output_pts, frame.pts); - - // Preload the file from disk, so that the encoder thread does not get stalled. - // TODO: Consider sending it through the queue instead. - (void)frame_reader.read_frame(frame); + fprintf(stderr, "output_pts=%" PRId64 " original input_pts=%" PRId64 "\n", output_pts, frame.pts); QueuedFrame qf; qf.local_pts = local_pts; qf.type = QueuedFrame::ORIGINAL; qf.output_pts = output_pts; - qf.frame1 = frame; qf.display_func = move(display_func); qf.queue_spot_holder = move(queue_spot_holder); qf.subtitle = subtitle; + FrameReader::Frame read_frame = frame_reader.read_frame(frame, /*read_video=*/true, include_audio); + qf.encoded_jpeg.reset(new string(move(read_frame.video))); + qf.audio = move(read_frame.audio); lock_guard lock(queue_lock); frame_queue.push_back(move(qf)); @@ -335,7 +367,7 @@ void VideoStream::schedule_faded_frame(steady_clock::time_point local_pts, int64 FrameOnDisk frame1_spec, FrameOnDisk frame2_spec, float fade_alpha, const string &subtitle) { - fprintf(stderr, "output_pts=%ld faded input_pts=%ld,%ld fade_alpha=%.2f\n", output_pts, frame1_spec.pts, frame2_spec.pts, fade_alpha); + fprintf(stderr, "output_pts=%" PRId64 " faded input_pts=%" PRId64 ",%" PRId64 " fade_alpha=%.2f\n", output_pts, frame1_spec.pts, frame2_spec.pts, fade_alpha); // Get the temporary OpenGL resources we need for doing the fade. // (We share these with interpolated frames, which is slightly @@ -402,12 +434,13 @@ void VideoStream::schedule_interpolated_frame(steady_clock::time_point local_pts int64_t output_pts, function)> &&display_func, QueueSpotHolder &&queue_spot_holder, FrameOnDisk frame1, FrameOnDisk frame2, - float alpha, FrameOnDisk secondary_frame, float fade_alpha, const string &subtitle) + float alpha, FrameOnDisk secondary_frame, float fade_alpha, const string &subtitle, + bool play_audio) { if (secondary_frame.pts != -1) { - fprintf(stderr, "output_pts=%ld interpolated input_pts1=%ld input_pts2=%ld alpha=%.3f secondary_pts=%ld fade_alpha=%.2f\n", output_pts, frame1.pts, frame2.pts, alpha, secondary_frame.pts, fade_alpha); + fprintf(stderr, "output_pts=%" PRId64 " interpolated input_pts1=%" PRId64 " input_pts2=%" PRId64 " alpha=%.3f secondary_pts=%" PRId64 " fade_alpha=%.2f\n", output_pts, frame1.pts, frame2.pts, alpha, secondary_frame.pts, fade_alpha); } else { - fprintf(stderr, "output_pts=%ld interpolated input_pts1=%ld input_pts2=%ld alpha=%.3f\n", output_pts, frame1.pts, frame2.pts, alpha); + fprintf(stderr, "output_pts=%" PRId64 " interpolated input_pts1=%" PRId64 " input_pts2=%" PRId64 " alpha=%.3f\n", output_pts, frame1.pts, frame2.pts, alpha); } // Get the temporary OpenGL resources we need for doing the interpolation. @@ -430,6 +463,10 @@ void VideoStream::schedule_interpolated_frame(steady_clock::time_point local_pts qf.local_pts = local_pts; qf.subtitle = subtitle; + if (play_audio) { + qf.audio = frame_reader.read_frame(frame1, /*read_video=*/false, /*read_audio=*/true).audio; + } + check_error(); // Convert frame0 and frame1 to OpenGL textures. @@ -541,6 +578,20 @@ void VideoStream::schedule_refresh_frame(steady_clock::time_point local_pts, queue_changed.notify_all(); } +void VideoStream::schedule_silence(steady_clock::time_point local_pts, int64_t output_pts, + int64_t length_pts, QueueSpotHolder &&queue_spot_holder) +{ + QueuedFrame qf; + qf.type = QueuedFrame::SILENCE; + qf.output_pts = output_pts; + qf.queue_spot_holder = move(queue_spot_holder); + qf.silence_length_pts = length_pts; + + lock_guard lock(queue_lock); + frame_queue.push_back(move(qf)); + queue_changed.notify_all(); +} + namespace { shared_ptr frame_from_pbo(void *contents, size_t width, size_t height) @@ -580,7 +631,7 @@ void VideoStream::encode_thread_func() bool ok = make_current(context, surface); if (!ok) { fprintf(stderr, "Video stream couldn't get an OpenGL context\n"); - exit(1); + abort(); } while (!should_quit) { @@ -615,9 +666,23 @@ void VideoStream::encode_thread_func() frame_queue.pop_front(); } + // Hack: We mux the subtitle packet one time unit before the actual frame, + // so that Nageru is sure to get it first. + if (!qf.subtitle.empty()) { + AVPacket pkt; + av_init_packet(&pkt); + pkt.stream_index = mux->get_subtitle_stream_idx(); + assert(pkt.stream_index != -1); + pkt.data = (uint8_t *)qf.subtitle.data(); + pkt.size = qf.subtitle.size(); + pkt.flags = 0; + pkt.duration = lrint(TIMEBASE / global_flags.output_framerate); // Doesn't really matter for Nageru. + mux->add_packet(pkt, qf.output_pts - 1, qf.output_pts - 1); + } + if (qf.type == QueuedFrame::ORIGINAL) { // Send the JPEG frame on, unchanged. - string jpeg = frame_reader.read_frame(qf.frame1); + string jpeg = move(*qf.encoded_jpeg); AVPacket pkt; av_init_packet(&pkt); pkt.stream_index = 0; @@ -625,15 +690,16 @@ void VideoStream::encode_thread_func() pkt.size = jpeg.size(); pkt.flags = AV_PKT_FLAG_KEY; mux->add_packet(pkt, qf.output_pts, qf.output_pts); + last_frame = move(jpeg); - last_frame.assign(&jpeg[0], &jpeg[0] + jpeg.size()); + add_audio_or_silence(qf); } else if (qf.type == QueuedFrame::FADED) { glClientWaitSync(qf.fence.get(), /*flags=*/0, GL_TIMEOUT_IGNORED); shared_ptr frame = frame_from_pbo(qf.resources->pbo_contents, global_flags.width, global_flags.height); // Now JPEG encode it, and send it on to the stream. - vector jpeg = encode_jpeg(frame->y.get(), frame->cb.get(), frame->cr.get(), global_flags.width, global_flags.height); + string jpeg = encode_jpeg(frame->y.get(), frame->cb.get(), frame->cr.get(), global_flags.width, global_flags.height); AVPacket pkt; av_init_packet(&pkt); @@ -643,6 +709,8 @@ void VideoStream::encode_thread_func() pkt.flags = AV_PKT_FLAG_KEY; mux->add_packet(pkt, qf.output_pts, qf.output_pts); last_frame = move(jpeg); + + add_audio_or_silence(qf); } else if (qf.type == QueuedFrame::INTERPOLATED || qf.type == QueuedFrame::FADED_INTERPOLATED) { glClientWaitSync(qf.fence.get(), /*flags=*/0, GL_TIMEOUT_IGNORED); @@ -653,7 +721,7 @@ void VideoStream::encode_thread_func() } // Now JPEG encode it, and send it on to the stream. - vector jpeg = encode_jpeg(frame->y.get(), frame->cb.get(), frame->cr.get(), global_flags.width, global_flags.height); + string jpeg = encode_jpeg(frame->y.get(), frame->cb.get(), frame->cr.get(), global_flags.width, global_flags.height); if (qf.flow_tex != 0) { compute_flow->release_texture(qf.flow_tex); } @@ -670,6 +738,8 @@ void VideoStream::encode_thread_func() pkt.flags = AV_PKT_FLAG_KEY; mux->add_packet(pkt, qf.output_pts, qf.output_pts); last_frame = move(jpeg); + + add_audio_or_silence(qf); } else if (qf.type == QueuedFrame::REFRESH) { AVPacket pkt; av_init_packet(&pkt); @@ -678,22 +748,13 @@ void VideoStream::encode_thread_func() pkt.size = last_frame.size(); pkt.flags = AV_PKT_FLAG_KEY; mux->add_packet(pkt, qf.output_pts, qf.output_pts); + + add_audio_or_silence(qf); // Definitely silence. + } else if (qf.type == QueuedFrame::SILENCE) { + add_silence(qf.output_pts, qf.silence_length_pts); } else { assert(false); } - - if (!qf.subtitle.empty()) { - AVPacket pkt; - av_init_packet(&pkt); - pkt.stream_index = mux->get_subtitle_stream_idx(); - assert(pkt.stream_index != -1); - pkt.data = (uint8_t *)qf.subtitle.data(); - pkt.size = qf.subtitle.size(); - pkt.flags = 0; - pkt.duration = lrint(TIMEBASE / global_flags.output_framerate); // Doesn't really matter for Nageru. - mux->add_packet(pkt, qf.output_pts, qf.output_pts); - } - if (qf.display_func != nullptr) { qf.display_func(); } @@ -724,3 +785,38 @@ int VideoStream::write_packet2(uint8_t *buf, int buf_size, AVIODataMarkerType ty } return buf_size; } + +void VideoStream::add_silence(int64_t pts, int64_t length_pts) +{ + // At 59.94, this will never quite add up (even discounting refresh frames, + // which have unpredictable length), but hopefully, the player in the other + // end should be able to stretch silence easily enough. + long num_samples = lrint(length_pts * double(OUTPUT_FREQUENCY) / double(TIMEBASE)) * 2; + uint8_t *zero = (uint8_t *)calloc(num_samples, sizeof(int32_t)); + + AVPacket pkt; + av_init_packet(&pkt); + pkt.stream_index = 1; + pkt.data = zero; + pkt.size = num_samples * sizeof(int32_t); + pkt.flags = AV_PKT_FLAG_KEY; + mux->add_packet(pkt, pts, pts); + + free(zero); +} + +void VideoStream::add_audio_or_silence(const QueuedFrame &qf) +{ + if (qf.audio.empty()) { + int64_t frame_length = lrint(double(TIMEBASE) / global_flags.output_framerate); + add_silence(qf.output_pts, frame_length); + } else { + AVPacket pkt; + av_init_packet(&pkt); + pkt.stream_index = 1; + pkt.data = (uint8_t *)qf.audio.data(); + pkt.size = qf.audio.size(); + pkt.flags = AV_PKT_FLAG_KEY; + mux->add_packet(pkt, qf.output_pts, qf.output_pts); + } +}