From 581694ded2a2de99cad7383f04a0ee1fb588f27d Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Mon, 3 Apr 2023 21:35:04 +0200 Subject: [PATCH 01/16] Bump version number. It's pretty clear next version is not going to be a simple subrevision. --- configure.ac | 2 +- version.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 0e51aed..3155873 100644 --- a/configure.ac +++ b/configure.ac @@ -1,5 +1,5 @@ AC_CONFIG_MACRO_DIR([m4]) -AC_INIT(cubemap, 1.4.3) +AC_INIT(cubemap, 1.5.0) AC_CONFIG_SRCDIR(main.cpp) diff --git a/version.h b/version.h index 4a2a26f..257e4b4 100644 --- a/version.h +++ b/version.h @@ -3,7 +3,7 @@ // Version number. Don't expect this to change all that often. -#define SERVER_VERSION "1.4.4-pre" +#define SERVER_VERSION "1.5.0-pre" #define SERVER_IDENTIFICATION "Cubemap/" SERVER_VERSION #endif // !defined(_VERSION_H) -- 2.39.2 From 3a1d58dee269fdca9cab7f482e4e8695b3fb2ffd Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Mon, 3 Apr 2023 22:09:07 +0200 Subject: [PATCH 02/16] Add a LD_PRELOAD-able library to force Metacube output from FFmpeg. This hooks just the right amount of functions to add Metacube output to arbitrary FFmpeg programs, but is obviously very brittle. (Native FFmpeg support would be better, but a patch did not go through when I tried a while back.) It is only lightly tested. Documentation in the README and cubemap.config.sample. --- Makefile.in | 13 +- README | 9 ++ configure.ac | 8 + cubemap.config.sample | 14 ++ ffmpeg_metacube_hack.c | 356 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 399 insertions(+), 1 deletion(-) create mode 100644 ffmpeg_metacube_hack.c diff --git a/Makefile.in b/Makefile.in index b4d8456..5f9f053 100644 --- a/Makefile.in +++ b/Makefile.in @@ -5,12 +5,17 @@ PROTOC=protoc CPPFLAGS=@CPPFLAGS@ CPPFLAGS += -Itlse -DWITH_KTLS -DNO_TLS_LEGACY_SUPPORT -DNO_SSL_COMPATIBLE_INTERFACE -DLTM_DESC -DTLS_REEXPORTABLE -DNO_TLS_WITH_CHACHA20_POLY1305 CXXFLAGS=-Wall @CXXFLAGS@ @protobuf_CFLAGS@ @libsystemd_CFLAGS@ @libtomcrypt_CFLAGS@ -pthread +CFLAGS=-Wall @CFLAGS@ LDFLAGS=@LDFLAGS@ -pthread LIBS=@LIBS@ @protobuf_LIBS@ @libsystemd_LIBS@ @libtomcrypt_LIBS@ OBJS=main.o client.o server.o stream.o udpstream.o serverpool.o input.o input_stats.o httpinput.o udpinput.o parse.o config.o acceptor.o stats.o accesslog.o thread.o util.o log.o metacube2.o sa_compare.o timespec.o state.pb.o tlse/tlse.o +ifeq (@have_ffmpeg@,yes) +all: cubemap ffmpeg_metacube_hack.so +else all: cubemap +endif %.pb.cc %.pb.h : %.proto $(PROTOC) --cpp_out=. $< @@ -23,16 +28,19 @@ all: cubemap $(CXX) -MMD -MP $(CPPFLAGS) $(CXXFLAGS) -o $@ -c $< cubemap: $(OBJS) $(CXX) -o cubemap $(OBJS) $(LIBS) $(LDFLAGS) +ffmpeg_metacube_hack.so: ffmpeg_metacube_hack.c + $(CC) -shared $(CPPFLAGS) $(CFLAGS) -fPIC -o $@ $< -ldl DEPS=$(OBJS:.o=.d) -include $(DEPS) clean: - $(RM) cubemap $(OBJS) $(DEPS) state.pb.h state.pb.cc + $(RM) cubemap $(OBJS) $(DEPS) state.pb.h state.pb.cc ffmpeg_metacube_hack.so PREFIX=@prefix@ SYSCONFDIR=@sysconfdir@ LOCALSTATEDIR=@localstatedir@ +LIBDIR=@libdir@ install: $(INSTALL) -m 755 -o root -g root -d \ $(DESTDIR)$(PREFIX)/bin \ @@ -44,6 +52,9 @@ install: $(DESTDIR)/lib/systemd/system $(INSTALL) -m 755 -o root -g root cubemap $(DESTDIR)$(PREFIX)/bin/cubemap $(INSTALL) -m 755 -o root -g root munin/cubemap munin/cubemap_input $(DESTDIR)$(PREFIX)/share/munin/plugins/ +ifeq (@have_ffmpeg@,yes) + $(INSTALL) -m 755 -o root -g root ffmpeg_metacube_hack.so $(DESTDIR)$(LIBDIR)/ +endif gzip -c cubemap.1 > $(DESTDIR)$(PREFIX)/share/man/man1/cubemap.1.gz sed \ -e "s,@prefix[@],$(PREFIX),g" \ diff --git a/README b/README index e9520d3..15913a1 100644 --- a/README +++ b/README @@ -43,6 +43,15 @@ compile and start cubemap. Nageru, my free video mixer, can also produce Metacube streams natively. See the manual at https://nageru.sesse.net/doc/ for more information. +If you feel very adventurous, you can use LD_PRELOAD to load +ffmpeg_metacube_hack.so into an FFmpeg-using binary. (This is +experimental; native Metacube support is vastly preferred.) +For instance, here's one way you can use the ffmpeg(1) binary +to serve your webcam to Cubemap: + + LD_PRELOAD=ffmpeg_metacube_hack.so ffmpeg -i /dev/video0 -f mpegts -listen 1 'http://[::]:9095' + + To upgrade cubemap (after you've compiled a new binary), or to pick up new config: diff --git a/configure.ac b/configure.ac index 3155873..189e45f 100644 --- a/configure.ac +++ b/configure.ac @@ -10,6 +10,14 @@ PKG_CHECK_MODULES([libsystemd], [libsystemd]) PKG_CHECK_MODULES([protobuf], [protobuf]) PKG_CHECK_MODULES([libtomcrypt], [libtomcrypt]) +have_ffmpeg=yes +PKG_CHECK_MODULES([libavformat], [libavformat], [true], [have_ffmpeg=no]) +PKG_CHECK_MODULES([libavutil], [libavutil], [true], [have_ffmpeg=no]) +AC_SUBST([have_ffmpeg]) + +AS_IF([test "x$have_ffmpeg" = "xno" ], + [AC_MSG_WARN([FFmpeg libraries not found. ffmpeg_metacube_hack.so will not be built.])]) + CXXFLAGS="$CXXFLAGS -std=gnu++11" AC_CONFIG_FILES([Makefile]) diff --git a/cubemap.config.sample b/cubemap.config.sample index 51c46d3..967059c 100644 --- a/cubemap.config.sample +++ b/cubemap.config.sample @@ -81,6 +81,20 @@ stream /test.ts src=http://gruessi.zrh.sesse.net:4013/test.ts src_encoding=raw # to be some reasonable fraction of your fragment length. stream /stream.mp4 src=http://gruessi.zrh.sesse.net:9095/test.mp4.metacube hls_playlist=/stream.m3u8 hls_frag_duration=6 backlog_size=20971520 hls_backlog_margin=1048576 allow_origin=* +# An example of using pipe: to read from a program that generates Metacube +# on standard output (or you can use src_encoding=raw for MPEG_TS). +# This shows how to use FFmpeg, combined with a LD_PRELOAD hack +# (the .so file is built with Cubemap if you have the FFmpeg development +# libraries instealled), to receive MPEG-TS over SRT and remux it into MP4. +# Note that using ffmpeg_metacube_hack.so is experimental and not recommended +# if you can avoid it. +# +# Only one instance of the command will be started at a time (it will drive a +# stream just like an HTTP input will). Such subprocesses will survive +# a Cubemap reload, but can of course themselves not be live-reloaded +# like Cubemap can. Standard error will be passed through unchanged. +stream /srt.mp4 src=pipe:"LD_PRELOAD=ffmpeg_metacube_hack.so ffmpeg -loglevel warning -i 'srt://[::]:9710?mode=listener' -c:v copy -c:a copy -f mp4 -movflags empty_moov+frag_keyframe+default_base_moof+skip_trailer -frag_duration 125000 -bsf:a aac_adtstoasc -" + # UDP input. TS is the most common container to use over UDP (you cannot # take any arbitrary container and expect it to work). # backlog_size= overrides the backlog, which is normally 10 MB. diff --git a/ffmpeg_metacube_hack.c b/ffmpeg_metacube_hack.c new file mode 100644 index 0000000..4398cd6 --- /dev/null +++ b/ffmpeg_metacube_hack.c @@ -0,0 +1,356 @@ +// A shared library that you can LD_PRELOAD into an FFmpeg-using process +// (most likely ffmpeg(1)) to make it output Metacube. This is obviously +// pretty hacky, since it needs to override various FFmpeg functions, +// so there are few guarantees here. It is written in C to avoid pulling +// in the C++ runtime into FFmpeg's C-only world. +// +// You should not link to this library. It does not have ABI stability. +// It is licensed the same as the rest of Cubemap. + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "metacube2.h" + +static pthread_once_t metacube2_crc_once_control = PTHREAD_ONCE_INIT; +static AVCRC metacube2_crc_table[257]; + +// We need to store some extra information for each context, +// so this is where we do it. The “opaque” field in the AVIOContext +// points to this struct, but we can also look it up by the AVIOContext +// pointer by scanning through the singly linked list starting with +// first_extra_data. +struct ContextExtraData { + struct ContextExtraData *next; // NULL for last entry. + AVIOContext *ctx; // The context we are associating data with. + bool seen_sync_point; + void *old_opaque; + int (*old_write_data_type)(void *opaque, uint8_t * buf, + int buf_size, + enum AVIODataMarkerType type, + int64_t time); + + // Used during avformat_write_header(), to combine adjacent header blocks + // into one (in particular, the MP4 mux has an unneeded avio_flush() + // halfway throughout). + bool in_header; + int64_t header_first_time; + uint8_t *buffered_header; + size_t buffered_header_bytes; +}; +static struct ContextExtraData *first_extra_data = NULL; + +// Look up ContextExtraData for the given context, creating a new one if needed. +static struct ContextExtraData *get_extra_data(AVIOContext * ctx) +{ + for (struct ContextExtraData * ed = first_extra_data; ed != NULL; + ed = ed->next) { + if (ed->ctx == ctx) { + return ed; + } + } + struct ContextExtraData *ed = (struct ContextExtraData *) + malloc(sizeof(struct ContextExtraData)); + ed->ctx = ctx; + ed->seen_sync_point = false; + ed->old_write_data_type = NULL; + ed->in_header = false; + ed->buffered_header = NULL; + ed->buffered_header_bytes = 0; + + ed->next = first_extra_data; + first_extra_data = ed; + return ed; +} + +// Clear ContextExtraData for a given context (presumably before it's freed). +static void free_extra_data(AVIOContext * ctx) +{ + if (first_extra_data == NULL) { + return; + } + if (first_extra_data->ctx == ctx) { + struct ContextExtraData *to_free = first_extra_data; + first_extra_data = to_free->next; + free(to_free); + return; + } + for (struct ContextExtraData * ed = first_extra_data; ed != NULL; + ed = ed->next) { + if (ed->next != NULL && ed->next->ctx == ctx) { + struct ContextExtraData *to_free = ed->next; + ed->next = to_free->next; + free(to_free); + return; + } + } +} + +static void metacube2_crc_init_table_once(void) +{ + av_assert0(av_crc_init + (metacube2_crc_table, 0, 16, 0x8fdb, + sizeof(metacube2_crc_table)) >= 0); +} + +static uint16_t metacube2_compute_crc_ff(const struct + metacube2_block_header *hdr) +{ + static const int data_len = sizeof(hdr->size) + sizeof(hdr->flags); + const uint8_t *data = (uint8_t *) & hdr->size; + uint16_t crc; + + pthread_once(&metacube2_crc_once_control, + metacube2_crc_init_table_once); + + // Metacube2 specifies a CRC start of 0x1234, but its pycrc-derived CRC + // includes a finalization step that is done somewhat differently in av_crc(). + // 0x1234 alone sent through that finalization becomes 0x394a, and then we + // need a byte-swap of the CRC value (both on input and output) to account for + // differing conventions. + crc = av_crc(metacube2_crc_table, 0x4a39, data, data_len); + return av_bswap16(crc); +} + +static int write_packet(void *opaque, uint8_t * buf, int buf_size, + enum AVIODataMarkerType type, int64_t time) +{ + if (buf_size < 0) { + return AVERROR(EINVAL); + } + + struct ContextExtraData *ed = (struct ContextExtraData *) opaque; + + if (ed->in_header) { + if (ed->buffered_header_bytes == 0) { + ed->header_first_time = time; + } + + size_t new_buffered_header_bytes = + ed->buffered_header_bytes + buf_size; + if (new_buffered_header_bytes < ed->buffered_header_bytes) { + return AVERROR(ENOMEM); + } + ed->buffered_header = + (uint8_t *) realloc(ed->buffered_header, + new_buffered_header_bytes); + if (ed->buffered_header == NULL) { + return AVERROR(ENOMEM); + } + + memcpy(ed->buffered_header + ed->buffered_header_bytes, + buf, buf_size); + ed->buffered_header_bytes = new_buffered_header_bytes; + return buf_size; + } + // Find block size if we add a Metacube2 header in front. + unsigned new_buf_size = + (unsigned) buf_size + sizeof(struct metacube2_block_header); + if (new_buf_size < (unsigned) buf_size + || new_buf_size > (unsigned) INT_MAX) { + // Overflow. + return -1; + } + // Fill in the header. + struct metacube2_block_header hdr; + int flags = 0; + if (type == AVIO_DATA_MARKER_SYNC_POINT) + ed->seen_sync_point = 1; + else if (type == AVIO_DATA_MARKER_HEADER) + // NOTE: If there are multiple blocks marked METACUBE_FLAGS_HEADER, + // only the last one will count. This may become a problem if the + // mux flushes halfway through the stream header; if so, we would + // need to keep track of and concatenate the different parts. + flags |= METACUBE_FLAGS_HEADER; + else if (ed->seen_sync_point) + flags |= METACUBE_FLAGS_NOT_SUITABLE_FOR_STREAM_START; + + memcpy(hdr.sync, METACUBE2_SYNC, sizeof(hdr.sync)); + AV_WB32(&hdr.size, buf_size); + AV_WB16(&hdr.flags, flags); + AV_WB16(&hdr.csum, metacube2_compute_crc_ff(&hdr)); + + int ret; + ed->ctx->opaque = ed->old_opaque; + if (new_buf_size < ed->ctx->max_packet_size) { + // Combine the two packets. (This is what we normally want.) + // So we allocate a new block, with a Metacube2 header in front. + uint8_t *buf_with_hdr = (uint8_t *) malloc(new_buf_size); + if (buf_with_hdr == NULL) { + return AVERROR(ENOMEM); + } + memcpy(buf_with_hdr, &hdr, sizeof(hdr)); + memcpy(buf_with_hdr + sizeof(hdr), buf, buf_size); + if (ed->old_write_data_type) { + ret = + ed->old_write_data_type(ed->old_opaque, + buf_with_hdr, + new_buf_size, type, + time); + } else { + ret = + ed->ctx->write_packet(ed->old_opaque, + buf_with_hdr, + new_buf_size); + } + free(buf_with_hdr); + + if (ret >= 0 + && ret >= sizeof(struct metacube2_block_header)) { + ret -= sizeof(struct metacube2_block_header); + } + } else { + // Send separately. This will split a header block if it's really large, + // which we don't want, but that's how things are. + if (ed->old_write_data_type) { + ret = + ed->old_write_data_type(ed->old_opaque, + (uint8_t *) & hdr, + sizeof(hdr), type, + time); + } else { + ret = + ed->ctx->write_packet(ed->old_opaque, + (uint8_t *) & hdr, + sizeof(hdr)); + } + if (ret < 0) { + return ret; + } + if (ret != sizeof(hdr)) { + return AVERROR(EIO); + } + + if (ed->old_write_data_type) { + ret = + ed->old_write_data_type(ed->old_opaque, buf, + buf_size, type, time); + } else { + ret = + ed->ctx->write_packet(ed->old_opaque, buf, + buf_size); + } + } + + ed->ctx->opaque = ed; + return ret; +} + +// Actual hooked functions below. + +int avformat_write_header(AVFormatContext * ctx, AVDictionary ** options) +{ + metacube2_crc_init_table_once(); + + struct ContextExtraData *ed = get_extra_data(ctx->pb); + ed->old_opaque = ctx->pb->opaque; + ed->old_write_data_type = ctx->pb->write_data_type; + ctx->pb->opaque = ed; + ctx->pb->write_data_type = write_packet; + ctx->pb->seek = NULL; + ctx->pb->seekable = 0; + if (ed->old_write_data_type == NULL) { + ctx->pb->ignore_boundary_point = 1; + } + + int (*original_func)(AVFormatContext * ctx, + AVDictionary ** options); + original_func = dlsym(RTLD_NEXT, "avformat_write_header"); + + ed->in_header = true; + int ret = (*original_func) (ctx, options); + ed->in_header = false; + + if (ed->buffered_header_bytes > 0) { + int hdr_ret = write_packet(ed, ed->buffered_header, + ed->buffered_header_bytes, + AVIO_DATA_MARKER_HEADER, + ed->header_first_time); + free(ed->buffered_header); + ed->buffered_header = NULL; + + if (hdr_ret >= 0 && hdr_ret < ed->buffered_header_bytes) { + hdr_ret = AVERROR(EIO); + } + ed->buffered_header_bytes = 0; + if (hdr_ret < 0) { + return hdr_ret; + } + } + + return ret; +} + +void avformat_free_context(AVFormatContext * ctx) +{ + if (ctx == NULL) { + return; + } + free_extra_data(ctx->pb); + + void (*original_func)(AVFormatContext * ctx); + original_func = dlsym(RTLD_NEXT, "avformat_free_context"); + return (*original_func) (ctx); +} + +// Hook so that we can restore opaque instead of ours being freed by the caller. +int avio_close(AVIOContext * ctx) +{ + if (ctx == NULL) { + return 0; + } + struct ContextExtraData ed = *get_extra_data(ctx); + free_extra_data(ctx); + ctx->opaque = ed.old_opaque; + + int (*original_func)(AVIOContext * ctx); + original_func = dlsym(RTLD_NEXT, "avio_close"); + return (*original_func) (ctx); +} + +// Identical to FFmpeg's definition, but we cannot hook avio_close() +// when called from FFmpeg's avio_closep(), so we need to hook this one +// as well. +int avio_closep(AVIOContext ** s) +{ + int ret = avio_close(*s); + *s = NULL; + return ret; +} + + +int avio_open2(AVIOContext ** s, const char *filename, int flags, + const AVIOInterruptCB * int_cb, AVDictionary ** options) +{ + // The options, if any, are destroyed on entry, so we can add new ones + // pretty freely. + if (options && *options) { + AVDictionaryEntry *listen = + av_dict_get(*options, "listen", NULL, + AV_DICT_MATCH_CASE); + if (listen != NULL && atoi(listen->value) != 0) { + // If -listen is set, we'll want to add a header, too. + av_dict_set(options, "headers", + "Content-encoding: metacube\r\n", + AV_DICT_APPEND); + } + } + + int (*original_func)(AVIOContext ** s, const char *filename, + int flags, const AVIOInterruptCB * int_cb, + AVDictionary ** options); + original_func = dlsym(RTLD_NEXT, "avio_open2"); + return (*original_func) (s, filename, flags, int_cb, options); +} -- 2.39.2 From babe41894e32db0db5480612c523bdc091305458 Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Sun, 19 Mar 2023 15:53:53 +0100 Subject: [PATCH 03/16] Fix a typo. --- httpinput.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/httpinput.h b/httpinput.h index 1ee1f8f..0c3b534 100644 --- a/httpinput.h +++ b/httpinput.h @@ -100,7 +100,7 @@ private: // The socket we are downloading on (or -1). int sock = -1; - // pid of the cihld process (or -1). + // pid of the child process (or -1). pid_t child_pid = -1; // Mutex protecting . -- 2.39.2 From 18321cd79681ba2bfbacc5ded79625ca2ca5e7d0 Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Tue, 29 Aug 2023 00:20:40 +0200 Subject: [PATCH 04/16] Use , which now is included in glibc, for kTLS. --- tlse/README | 7 ++--- tlse/ktls.h | 81 +---------------------------------------------------- 2 files changed, 4 insertions(+), 84 deletions(-) diff --git a/tlse/README b/tlse/README index 3fa5ca5..8e91f40 100644 --- a/tlse/README +++ b/tlse/README @@ -2,7 +2,6 @@ This is a copy of the core files of TLSe, checked out from https://github.com/eduardsui/tlse -It is patched to add and include ktls.h, since glibc does not include -kTLS definitions yet, and then irrelevant files removed. It is embedded -(as opposed to a regular link) since TLSe does not provide a static or -shared library. +It has had irrelevant files removed, and a dummy ktls.h is added to +forward to glibc's implementation. It is embedded (as opposed to +a regular link) since TLSe does not provide a static or shared library. diff --git a/tlse/ktls.h b/tlse/ktls.h index 7ed59c4..4693d2c 100644 --- a/tlse/ktls.h +++ b/tlse/ktls.h @@ -1,80 +1 @@ -/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-2-Clause) */ -/* - * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef _UAPI_LINUX_TLS_H -#define _UAPI_LINUX_TLS_H - -#include - -/* TLS socket options */ -#define TLS_TX 1 /* Set transmit parameters */ -#define TLS_RX 2 /* Set receive parameters */ - -/* Supported versions */ -#define TLS_VERSION_MINOR(ver) ((ver) & 0xFF) -#define TLS_VERSION_MAJOR(ver) (((ver) >> 8) & 0xFF) - -#define TLS_VERSION_NUMBER(id) ((((id##_VERSION_MAJOR) & 0xFF) << 8) | \ - ((id##_VERSION_MINOR) & 0xFF)) - -#define TLS_1_2_VERSION_MAJOR 0x3 -#define TLS_1_2_VERSION_MINOR 0x3 -#define TLS_1_2_VERSION TLS_VERSION_NUMBER(TLS_1_2) - -/* Supported ciphers */ -#define TLS_CIPHER_AES_GCM_128 51 -#define TLS_CIPHER_AES_GCM_128_IV_SIZE 8 -#define TLS_CIPHER_AES_GCM_128_KEY_SIZE 16 -#define TLS_CIPHER_AES_GCM_128_SALT_SIZE 4 -#define TLS_CIPHER_AES_GCM_128_TAG_SIZE 16 -#define TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE 8 - -#define TLS_SET_RECORD_TYPE 1 - -struct tls_crypto_info { - __u16 version; - __u16 cipher_type; -}; - -struct tls12_crypto_info_aes_gcm_128 { - struct tls_crypto_info info; - unsigned char iv[TLS_CIPHER_AES_GCM_128_IV_SIZE]; - unsigned char key[TLS_CIPHER_AES_GCM_128_KEY_SIZE]; - unsigned char salt[TLS_CIPHER_AES_GCM_128_SALT_SIZE]; - unsigned char rec_seq[TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE]; -}; - -#define SOL_TLS 282 -#define TCP_ULP 31 - -#endif /* _UAPI_LINUX_TLS_H */ +#include -- 2.39.2 From 16c4bfba6d41e67bbff2ba9b4b579b017e121bbd Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Wed, 6 Dec 2023 21:07:45 +0100 Subject: [PATCH 05/16] pkg-config is required. --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README b/README index 15913a1..a674f7a 100644 --- a/README +++ b/README @@ -27,7 +27,7 @@ A short list of features: HOWTO: - sudo apt install libprotobuf-dev protobuf-compiler libsystemd-dev libtomcrypt-dev + sudo apt install libprotobuf-dev protobuf-compiler libsystemd-dev libtomcrypt-dev pkg-config ./configure make -j4 -- 2.39.2 From 0bbe147ee38bb636e96e4c4211aefc17b28e1bcd Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Sat, 9 Dec 2023 15:19:34 +0100 Subject: [PATCH 06/16] Linux 3.13 is really old by now, so remove it from the README. --- README | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README b/README index a674f7a..dc6d4d3 100644 --- a/README +++ b/README @@ -11,8 +11,7 @@ A short list of features: and sending a SIGHUP; all clients will continue as if nothing had happened (unless you delete the stream they are watching, of course). Cubemap also survives the encoder dying and reconnecting. - - Support for setting max pacing rate through the fq packet scheduler - (depends on Linux 3.13 or newer). + - Support for setting max pacing rate through the fq packet scheduler. - Reflects anything VLC can reflect over HTTP, even the muxes VLC has problems reflecting itself (in particular, FLV). - Multicast support, both for sending and receiving (supports only protocols -- 2.39.2 From b8cc60e5bec90ab08ac96e8017c0de22c0fb5e41 Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Sat, 9 Dec 2023 15:26:19 +0100 Subject: [PATCH 07/16] Update README about Metacube input. --- README | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/README b/README index dc6d4d3..efcfa77 100644 --- a/README +++ b/README @@ -1,5 +1,4 @@ -Cubemap is a high-performance, high-availability video reflector, -specifically made for use with VLC. +Cubemap is a high-performance, high-availability video reflector. A short list of features: @@ -30,8 +29,17 @@ HOWTO: ./configure make -j4 -If you want to use HTTP input (you probably want to), you want VLC 2.2.0 -or newer. Then start the VLC encoder with the “metacube” flag to the http +Cubemap does not in itself understand the format of video streams; +it requires HTTP input that is already delineated into headers and blocks, +so that it knows what parts of the stream to skip to clients that +connect after the stream has been. (The exception is self-synchronizing +streams with no headers, such as MPEG-TS. These can also be sent or +received over UDP) The input format is specific to Cubemap and is called +Metacube (technically, Metacube2, but Metacube1 is long gone and nothing +uses it anymore). Currently, you have three options to create Metacube +video streams: VLC, Nageru or FFmpeg with a hack. + +For VLC, start the VLC encoder with the “metacube” flag to the http access mux, like this: cvlc [...] --sout '#std{access=http{metacube,mime=video/x-flv},mux=flv,dst=:4013/test.flv}' @@ -39,8 +47,9 @@ access mux, like this: Then look through cubemap.config.sample, copy it to cubemap.config, compile and start cubemap. -Nageru, my free video mixer, can also produce Metacube streams natively. -See the manual at https://nageru.sesse.net/doc/ for more information. +Nageru, my free video mixer, can also produce Metacube streams natively, +and so can its included transcoder/remuxer Kaeru. See the manual +at https://nageru.sesse.net/doc/ for more information. If you feel very adventurous, you can use LD_PRELOAD to load ffmpeg_metacube_hack.so into an FFmpeg-using binary. (This is -- 2.39.2 From ef7de8af43f5a8ce27c9480ebd7561ab464b4fc2 Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Sat, 9 Dec 2023 15:37:49 +0100 Subject: [PATCH 08/16] Document pipes in the README. --- README | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README b/README index efcfa77..7ae25df 100644 --- a/README +++ b/README @@ -12,7 +12,10 @@ A short list of features: Cubemap also survives the encoder dying and reconnecting. - Support for setting max pacing rate through the fq packet scheduler. - Reflects anything VLC can reflect over HTTP, even the muxes VLC - has problems reflecting itself (in particular, FLV). + has problems reflecting itself (in particular, FLV). Can also + call out to external programs to receive or fetch streams in arbitrary + ways (e.g. through SRT), although of course they do not necessarily + have the same intrinsic high-availability reconfiguration support. - Multicast support, both for sending and receiving (supports only protocols that can go over UDP, e.g. MPEG-TS). Supports both ASM and SSM. - TLS output support, through the TLSe library (requires libtomcrypt) -- 2.39.2 From 3464053ffc7a676961ab825f6fc6aa0dcbd9f50f Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Sat, 9 Dec 2023 16:28:26 +0100 Subject: [PATCH 09/16] Better HOWTO in the README. --- README | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README b/README index 7ae25df..46a943a 100644 --- a/README +++ b/README @@ -28,7 +28,9 @@ A short list of features: HOWTO: - sudo apt install libprotobuf-dev protobuf-compiler libsystemd-dev libtomcrypt-dev pkg-config + sudo apt install libprotobuf-dev protobuf-compiler libsystemd-dev libtomcrypt-dev pkg-config autoconf + sudo apt install libavformat-dev libavutil-dev # Optional + autoreconf -I . ./configure make -j4 -- 2.39.2 From 8546e7ca65486e7896df77b08a5f3bab756222b9 Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Sat, 9 Dec 2023 16:29:12 +0100 Subject: [PATCH 10/16] Remove obsolete comment. --- version.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/version.h b/version.h index 257e4b4..074eba2 100644 --- a/version.h +++ b/version.h @@ -1,8 +1,6 @@ #ifndef _VERSION_H #define _VERSION_H -// Version number. Don't expect this to change all that often. - #define SERVER_VERSION "1.5.0-pre" #define SERVER_IDENTIFICATION "Cubemap/" SERVER_VERSION -- 2.39.2 From bb469bd80e10f3bfc8985602b23306629ccf4acb Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Sat, 9 Dec 2023 16:32:27 +0100 Subject: [PATCH 11/16] Release Cubemap 1.5.0. --- NEWS | 23 ++++++++++++++++++++++- version.h | 2 +- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index a154c1e..209f90c 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,24 @@ +Cubemap 1.5.0, 2023-12-09 + + * Support input from pipes (subprocesses). + + With a program that can output Metacube on stdout, such as a suitably + patched FFmpeg binary, this gives Cubemap transcoding/remuxing capabilities. + + Of course, one cannot upgrade such a binary by SIGHUP-ing it, but it will + survive a Cubemap reload/restart. + + * Add a LD_PRELOAD-able library to force Metacube output from FFmpeg. + + This hooks just the right amount of functions to add Metacube output + to arbitrary FFmpeg programs, but is obviously very brittle. + (Native FFmpeg support would be better, but a patch did not + go through when I tried a while back.) It is only lightly tested. + Documentation in the README and cubemap.config.sample. + + * Various bugfixes. + + Cubemap 1.4.3, 2019-11-17 * Keep the HLS backlog even if the stream header changes. Useful @@ -6,7 +27,7 @@ Cubemap 1.4.3, 2019-11-17 discontinuities). * Various bugfixes. - + Cubemap 1.4.2, 2018-12-21 diff --git a/version.h b/version.h index 074eba2..97fbe85 100644 --- a/version.h +++ b/version.h @@ -1,7 +1,7 @@ #ifndef _VERSION_H #define _VERSION_H -#define SERVER_VERSION "1.5.0-pre" +#define SERVER_VERSION "1.5.0" #define SERVER_IDENTIFICATION "Cubemap/" SERVER_VERSION #endif // !defined(_VERSION_H) -- 2.39.2 From 2190c25b2e2dcb56b69f9e648d99203b01018063 Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Sun, 17 Dec 2023 21:00:59 +0100 Subject: [PATCH 12/16] Use systemd.pc to place systemd system units. Patch by Chris Hofstaedtler. Useful in particular for merged-/usr systems. --- Makefile.in | 5 +++-- configure.ac | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile.in b/Makefile.in index 5f9f053..5f7fbf4 100644 --- a/Makefile.in +++ b/Makefile.in @@ -8,6 +8,7 @@ CXXFLAGS=-Wall @CXXFLAGS@ @protobuf_CFLAGS@ @libsystemd_CFLAGS@ @libtomcrypt_CFL CFLAGS=-Wall @CFLAGS@ LDFLAGS=@LDFLAGS@ -pthread LIBS=@LIBS@ @protobuf_LIBS@ @libsystemd_LIBS@ @libtomcrypt_LIBS@ +SYSTEMDSYSTEMUNITDIR=@SYSTEMDSYSTEMUNITDIR@ OBJS=main.o client.o server.o stream.o udpstream.o serverpool.o input.o input_stats.o httpinput.o udpinput.o parse.o config.o acceptor.o stats.o accesslog.o thread.o util.o log.o metacube2.o sa_compare.o timespec.o state.pb.o tlse/tlse.o @@ -49,7 +50,7 @@ install: $(DESTDIR)$(LOCALSTATEDIR)/lib/cubemap \ $(DESTDIR)$(LOCALSTATEDIR)/log/cubemap \ $(DESTDIR)$(PREFIX)/share/munin/plugins \ - $(DESTDIR)/lib/systemd/system + $(DESTDIR)$(SYSTEMDSYSTEMUNITDIR) $(INSTALL) -m 755 -o root -g root cubemap $(DESTDIR)$(PREFIX)/bin/cubemap $(INSTALL) -m 755 -o root -g root munin/cubemap munin/cubemap_input $(DESTDIR)$(PREFIX)/share/munin/plugins/ ifeq (@have_ffmpeg@,yes) @@ -59,7 +60,7 @@ endif sed \ -e "s,@prefix[@],$(PREFIX),g" \ -e "s,@sysconfdir[@],$(SYSCONFDIR),g" \ - cubemap.service.in > $(DESTDIR)/lib/systemd/system/cubemap.service + cubemap.service.in > $(DESTDIR)$(SYSTEMDSYSTEMUNITDIR)/cubemap.service # Make sure we don't overwrite an existing configuration. test ! -f $(DESTDIR)$(SYSCONFDIR)/cubemap.config sed \ diff --git a/configure.ac b/configure.ac index 189e45f..9564ae9 100644 --- a/configure.ac +++ b/configure.ac @@ -9,6 +9,7 @@ PKG_PROG_PKG_CONFIG PKG_CHECK_MODULES([libsystemd], [libsystemd]) PKG_CHECK_MODULES([protobuf], [protobuf]) PKG_CHECK_MODULES([libtomcrypt], [libtomcrypt]) +PKG_CHECK_VAR(SYSTEMDSYSTEMUNITDIR, systemd, systemdsystemunitdir) have_ffmpeg=yes PKG_CHECK_MODULES([libavformat], [libavformat], [true], [have_ffmpeg=no]) -- 2.39.2 From f1f37cbb978224d706f2f251aa06072927236f6b Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Sun, 17 Dec 2023 21:03:29 +0100 Subject: [PATCH 13/16] Release Cubemap 1.5.1. --- NEWS | 6 ++++++ version.h | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 209f90c..10d5c0e 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,9 @@ +Cubemap 1.5.1, 2023-12-17 + + * Use systemd.pc to place systemd system units. + Patch by Chris Hofstaedtler. + + Cubemap 1.5.0, 2023-12-09 * Support input from pipes (subprocesses). diff --git a/version.h b/version.h index 97fbe85..45996d2 100644 --- a/version.h +++ b/version.h @@ -1,7 +1,7 @@ #ifndef _VERSION_H #define _VERSION_H -#define SERVER_VERSION "1.5.0" +#define SERVER_VERSION "1.5.1" #define SERVER_IDENTIFICATION "Cubemap/" SERVER_VERSION #endif // !defined(_VERSION_H) -- 2.39.2 From 05d5d025510b05b8ff92a55c50576c7073c64646 Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Tue, 2 Jan 2024 18:45:07 +0100 Subject: [PATCH 14/16] Bump version number. --- configure.ac | 2 +- version.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 9564ae9..1d9bc39 100644 --- a/configure.ac +++ b/configure.ac @@ -1,5 +1,5 @@ AC_CONFIG_MACRO_DIR([m4]) -AC_INIT(cubemap, 1.5.0) +AC_INIT(cubemap, 1.5.2-pre) AC_CONFIG_SRCDIR(main.cpp) diff --git a/version.h b/version.h index 45996d2..fe89b31 100644 --- a/version.h +++ b/version.h @@ -1,7 +1,7 @@ #ifndef _VERSION_H #define _VERSION_H -#define SERVER_VERSION "1.5.1" +#define SERVER_VERSION "1.5.2-pre" #define SERVER_IDENTIFICATION "Cubemap/" SERVER_VERSION #endif // !defined(_VERSION_H) -- 2.39.2 From f1ef19331f02cd8f001d6c53764fe5b2dcf49d57 Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Tue, 2 Jan 2024 18:45:25 +0100 Subject: [PATCH 15/16] Lowercase $PREFIX and similar variables. This is evidently standard enough that at least Debian tools expect $prefix (in lowercase) to exist and that other variables can refer to it, so follow suit. --- Makefile.in | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/Makefile.in b/Makefile.in index 5f7fbf4..a44f145 100644 --- a/Makefile.in +++ b/Makefile.in @@ -38,39 +38,39 @@ DEPS=$(OBJS:.o=.d) clean: $(RM) cubemap $(OBJS) $(DEPS) state.pb.h state.pb.cc ffmpeg_metacube_hack.so -PREFIX=@prefix@ -SYSCONFDIR=@sysconfdir@ -LOCALSTATEDIR=@localstatedir@ -LIBDIR=@libdir@ +prefix=@prefix@ +sysconfdir=@sysconfdir@ +localstatedir=@localstatedir@ +libdir=@libdir@ install: $(INSTALL) -m 755 -o root -g root -d \ - $(DESTDIR)$(PREFIX)/bin \ - $(DESTDIR)$(PREFIX)/share/man/man1 \ - $(DESTDIR)$(SYSCONFDIR) \ - $(DESTDIR)$(LOCALSTATEDIR)/lib/cubemap \ - $(DESTDIR)$(LOCALSTATEDIR)/log/cubemap \ - $(DESTDIR)$(PREFIX)/share/munin/plugins \ + $(DESTDIR)$(prefix)/bin \ + $(DESTDIR)$(prefix)/share/man/man1 \ + $(DESTDIR)$(sysconfdir) \ + $(DESTDIR)$(localstatedir)/lib/cubemap \ + $(DESTDIR)$(localstatedir)/log/cubemap \ + $(DESTDIR)$(prefix)/share/munin/plugins \ $(DESTDIR)$(SYSTEMDSYSTEMUNITDIR) - $(INSTALL) -m 755 -o root -g root cubemap $(DESTDIR)$(PREFIX)/bin/cubemap - $(INSTALL) -m 755 -o root -g root munin/cubemap munin/cubemap_input $(DESTDIR)$(PREFIX)/share/munin/plugins/ + $(INSTALL) -m 755 -o root -g root cubemap $(DESTDIR)$(prefix)/bin/cubemap + $(INSTALL) -m 755 -o root -g root munin/cubemap munin/cubemap_input $(DESTDIR)$(prefix)/share/munin/plugins/ ifeq (@have_ffmpeg@,yes) - $(INSTALL) -m 755 -o root -g root ffmpeg_metacube_hack.so $(DESTDIR)$(LIBDIR)/ + $(INSTALL) -m 755 -o root -g root ffmpeg_metacube_hack.so $(DESTDIR)$(libdir)/ endif - gzip -c cubemap.1 > $(DESTDIR)$(PREFIX)/share/man/man1/cubemap.1.gz + gzip -c cubemap.1 > $(DESTDIR)$(prefix)/share/man/man1/cubemap.1.gz sed \ - -e "s,@prefix[@],$(PREFIX),g" \ - -e "s,@sysconfdir[@],$(SYSCONFDIR),g" \ + -e "s,@prefix[@],$(prefix),g" \ + -e "s,@sysconfdir[@],$(sysconfdir),g" \ cubemap.service.in > $(DESTDIR)$(SYSTEMDSYSTEMUNITDIR)/cubemap.service # Make sure we don't overwrite an existing configuration. - test ! -f $(DESTDIR)$(SYSCONFDIR)/cubemap.config + test ! -f $(DESTDIR)$(sysconfdir)/cubemap.config sed \ - -e "s,cubemap\.stats,$(LOCALSTATEDIR)/lib/cubemap/\0,g" \ - -e "s,cubemap-input\.stats,$(LOCALSTATEDIR)/lib/cubemap/\0,g" \ - -e "s,access\.log,$(LOCALSTATEDIR)/log/cubemap/\0,g" \ - -e "s,cubemap\.log,$(LOCALSTATEDIR)/log/cubemap/\0,g" \ + -e "s,cubemap\.stats,$(localstatedir)/lib/cubemap/\0,g" \ + -e "s,cubemap-input\.stats,$(localstatedir)/lib/cubemap/\0,g" \ + -e "s,access\.log,$(localstatedir)/log/cubemap/\0,g" \ + -e "s,cubemap\.log,$(localstatedir)/log/cubemap/\0,g" \ -e 's,^stream,#\0,g' \ -e 's,^udpstream,#\0,g' \ - cubemap.config.sample > $(DESTDIR)$(SYSCONFDIR)/cubemap.config + cubemap.config.sample > $(DESTDIR)$(sysconfdir)/cubemap.config .PHONY: clean install .SUFFIXES: -- 2.39.2 From 3968945f8c317d892d6f1e3c58b791c8f2eb7d94 Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Tue, 2 Jan 2024 18:47:33 +0100 Subject: [PATCH 16/16] Create $(libdir) on make install. --- Makefile.in | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile.in b/Makefile.in index a44f145..c910c25 100644 --- a/Makefile.in +++ b/Makefile.in @@ -54,6 +54,7 @@ install: $(INSTALL) -m 755 -o root -g root cubemap $(DESTDIR)$(prefix)/bin/cubemap $(INSTALL) -m 755 -o root -g root munin/cubemap munin/cubemap_input $(DESTDIR)$(prefix)/share/munin/plugins/ ifeq (@have_ffmpeg@,yes) + $(INSTALL) -m 755 -o root -g root -d $(DESTDIR)$(libdir) $(INSTALL) -m 755 -o root -g root ffmpeg_metacube_hack.so $(DESTDIR)$(libdir)/ endif gzip -c cubemap.1 > $(DESTDIR)$(prefix)/share/man/man1/cubemap.1.gz -- 2.39.2