From f9ba261c26e2bb56d28654a4fdf6180c6eb218df Mon Sep 17 00:00:00 2001 From: Zach van Rijn Date: Thu, 4 May 2023 09:57:57 -0500 Subject: {user => legacy}/mistserver: 32-bit headache. see #1008. --- user/mistserver/APKBUILD | 72 - user/mistserver/add-dtls-srtp-cmake-option.patch | 15 - user/mistserver/fix-cmake-test-format.patch | 37 - .../gizahnl-mbedtls-dev-from-origin-master.patch | 13180 ------------------- user/mistserver/link-execinfo.patch | 11 - user/mistserver/mistserver.confd | 7 - user/mistserver/mistserver.initd | 29 - user/mistserver/mistserver.pre-install | 7 - 8 files changed, 13358 deletions(-) delete mode 100644 user/mistserver/APKBUILD delete mode 100644 user/mistserver/add-dtls-srtp-cmake-option.patch delete mode 100644 user/mistserver/fix-cmake-test-format.patch delete mode 100644 user/mistserver/gizahnl-mbedtls-dev-from-origin-master.patch delete mode 100644 user/mistserver/link-execinfo.patch delete mode 100644 user/mistserver/mistserver.confd delete mode 100755 user/mistserver/mistserver.initd delete mode 100644 user/mistserver/mistserver.pre-install (limited to 'user') diff --git a/user/mistserver/APKBUILD b/user/mistserver/APKBUILD deleted file mode 100644 index 093d88f4e..000000000 --- a/user/mistserver/APKBUILD +++ /dev/null @@ -1,72 +0,0 @@ -# Contributor: Síle Ekaterin Liszka -# Maintainer: Síle Ekaterin Liszka -pkgname=mistserver -pkgver=3.1 -pkgrel=0 -pkgdesc="Multimedia streaming services" -url="https://mistserver.org" -arch="all" -options="" -license="Zlib AND AGPL-3.0-only" -depends="" -install="$pkgname.pre-install" -makedepends="cmake libexecinfo-dev mbedtls-dev libsrtp-dev" -pkgusers="mistserver" -pkggroups="mistserver" -subpackages="$pkgname-dev $pkgname-openrc" -# ATTENTION MAINTAINERS: Generate a new aggregate patch from: -# (assuming rebased on upstream correctly) -# git clone https://github.com/gizahNL/mistserver.git -# cd mistserver -# git format-patch -M origin/master --stdout > gizahnl-mbedtls-dev-from-origin-master.patch -source="mistserver-$pkgver.tar.gz::https://github.com/DDVTECH/mistserver/archive/refs/tags/$pkgver.tar.gz - mistserver.confd - mistserver.initd - link-execinfo.patch - - add-dtls-srtp-cmake-option.patch - gizahnl-mbedtls-dev-from-origin-master.patch - fix-cmake-test-format.patch - " - -build() { - if [ "$CBUILD" != "$CHOST" ]; then - CMAKE_CROSSOPTS="-DCMAKE_SYSTEM_NAME=Linux -DCMAKE_HOST_SYSTEM_NAME=Linux" - fi - cmake \ - -DCMAKE_INSTALL_PREFIX=/usr \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -DBUILD_SHARED_LIBS=True \ - -DCMAKE_BUILD_TYPE=RelWithDebugInfo \ - -DCMAKE_CXX_FLAGS="$CXXFLAGS -fPIC" \ - -DCMAKE_C_FLAGS="$CFLAGS -fPIC" \ - -DUSE_MBEDTLS_SSL_DTLS_SRTP=True \ - ${CMAKE_CROSSOPTS} \ - . - make -j1 # do not increase this (race conditions) -} - -check() { - # FIXME!!! - #CTEST_OUTPUT_ON_FAILURE=TRUE ctest - - # temporary sanity check - ./MistSession -v | grep Built -} - -package() { - make DESTDIR="$pkgdir" install - - # OpenRC - - install -Dm755 "$srcdir"/mistserver.initd "$pkgdir"/etc/init.d/mistserver - install -Dm644 "$srcdir"/mistserver.confd "$pkgdir"/etc/conf.d/mistserver -} - -sha512sums="efcac86cf031c5cc13dd274a4d63292122f1ef3d46faea0457e075898cda01bdea29f011699b595e07c8ed984886a33da2a04395a67698d6b2b405325f1b9715 mistserver-3.1.tar.gz -7288adab6589f2facc1cb794057b1c5d9ec94e12e60d6afc8f6f25c54a8e908cc9841b83b5a6e608fa799fd6aa11767e92a963004182d45f7be9ccd3b65097e7 mistserver.confd -e0c7df42f4d486983ece1ea50ab8f3006ebab5386881c14c4b2ff1246b6dd38ace935dc54f8f8a7687edb7ca5975b8c26abd6e99957b8c892862732263d49eb9 mistserver.initd -a27bac965078f7eafb339ae7be9e50519d5728ae4f5d725905d5eecbb3fdf048df3e150cfa881be4bab754ca674a11271343156d5d97758d2ca65bef5bff55a6 link-execinfo.patch -f90737722ac4a2ecff64386a9287ce0ddd48e7b176239f3de26725cadace52667ab44febe536738d8e0dba1fee2047e8f65caa8a2f282c7c6e9dbcc4f8daa23a add-dtls-srtp-cmake-option.patch -48e835a09b8096f78e94f25429768debf65ab9f4b3152c45b69eb072ac5d3abc0036daae894e6b85c57ad7ae7993bf2940c730fbb02dcafecd2ed9716b86dfb9 gizahnl-mbedtls-dev-from-origin-master.patch -de8bc5279426c7fab58074a4e73001c590dd386c0d7cd1747e6fff74278aa901662e7720bf97ea645eb28fc1a6c3e4830afdcaf5e4a770c996c639936dd3e28b fix-cmake-test-format.patch" diff --git a/user/mistserver/add-dtls-srtp-cmake-option.patch b/user/mistserver/add-dtls-srtp-cmake-option.patch deleted file mode 100644 index 8704b5a08..000000000 --- a/user/mistserver/add-dtls-srtp-cmake-option.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 6af16808..a12aa8cf 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -91,6 +91,10 @@ else() - message("SSL/TLS support is turned OFF") - endif() - -+if (USE_MBEDTLS_SSL_DTLS_SRTP) -+ add_definitions(-DHAVE_UPSTREAM_MBEDTLS_SRTP=1) -+endif() -+ - if (DEFINED DATASIZE ) - add_definitions(-DSHM_DATASIZE=${DATASIZE}) - endif() diff --git a/user/mistserver/fix-cmake-test-format.patch b/user/mistserver/fix-cmake-test-format.patch deleted file mode 100644 index 07161a2c5..000000000 --- a/user/mistserver/fix-cmake-test-format.patch +++ /dev/null @@ -1,37 +0,0 @@ -I don't know if this is correct. - -https://gitlab.kitware.com/cmake/cmake/-/issues/19109 - ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -931,24 +931,24 @@ - ######################################## - add_executable(urltest test/url.cpp ${BINARY_DIR}/mist/.headers) - target_link_libraries(urltest mist) --add_test(URLTest COMMAND urltest) -+add_test(NAME URLTest COMMAND urltest) - add_executable(logtest test/log.cpp ${BINARY_DIR}/mist/.headers) - target_link_libraries(logtest mist) --add_test(LOGTest COMMAND logtest) -+add_test(NAME LOGTest COMMAND logtest) - add_executable(downloadertest test/downloader.cpp ${BINARY_DIR}/mist/.headers) - target_link_libraries(downloadertest mist) --add_test(DownloaderTest COMMAND downloadertest) -+add_test(NAME DownloaderTest COMMAND downloadertest) - add_executable(urireadertest test/urireader.cpp ${BINARY_DIR}/mist/.headers) - target_link_libraries(urireadertest mist) --add_test(URIReaderTest COMMAND urireadertest) -+add_test(NAME URIReaderTest COMMAND urireadertest) - add_executable(jsontest test/json.cpp ${BINARY_DIR}/mist/.headers) - target_link_libraries(jsontest mist) --add_test(JSONTest COMMAND jsontest) -+add_test(NAME JSONTest COMMAND jsontest) - add_executable(resolvetest test/resolve.cpp ${BINARY_DIR}/mist/.headers) - target_link_libraries(resolvetest mist) - add_executable(bitwritertest test/bitwriter.cpp ${BINARY_DIR}/mist/.headers) - target_link_libraries(bitwritertest mist) --add_test(BitWriterTest COMMAND bitwritertest) -+add_test(NAME BitWriterTest COMMAND bitwritertest) - add_executable(streamstatustest test/status.cpp ${BINARY_DIR}/mist/.headers) - target_link_libraries(streamstatustest mist) - add_executable(websockettest test/websocket.cpp ${BINARY_DIR}/mist/.headers) diff --git a/user/mistserver/gizahnl-mbedtls-dev-from-origin-master.patch b/user/mistserver/gizahnl-mbedtls-dev-from-origin-master.patch deleted file mode 100644 index 29a6914e2..000000000 --- a/user/mistserver/gizahnl-mbedtls-dev-from-origin-master.patch +++ /dev/null @@ -1,13180 +0,0 @@ -From 0430a644dfcd0fcc0ea6494bdb8d431073b8361d Mon Sep 17 00:00:00 2001 -From: Matthew James -Date: Fri, 22 Jul 2022 00:44:28 +0800 -Subject: [PATCH 01/38] Enable Parameters on TSSRT listener - ---- - lib/socket_srt.cpp | 7 +++---- - lib/socket_srt.h | 2 +- - src/input/input_tssrt.cpp | 4 +++- - src/output/output_tssrt.cpp | 3 ++- - 4 files changed, 9 insertions(+), 7 deletions(-) - -diff --git a/lib/socket_srt.cpp b/lib/socket_srt.cpp -index a01117db..3e03bce4 100644 ---- a/lib/socket_srt.cpp -+++ b/lib/socket_srt.cpp -@@ -462,12 +462,11 @@ namespace Socket{ - - SRTServer::SRTServer(int fromSock){conn = SRTConnection(fromSock);} - -- SRTServer::SRTServer(int port, std::string hostname, bool nonblock, const std::string &_direction){ -+ SRTServer::SRTServer(int port, std::string hostname, std::map _params, bool nonblock, const std::string &_direction){ - // We always create a server as listening -- std::map listenMode; -- listenMode["mode"] = "listener"; -+ _params["mode"] = "listener"; - if (hostname == ""){hostname = "0.0.0.0";} -- conn.connect(hostname, port, _direction, listenMode); -+ conn.connect(hostname, port, _direction, _params); - conn.setBlocking(true); - if (!conn){ - ERROR_MSG("Unable to create socket"); -diff --git a/lib/socket_srt.h b/lib/socket_srt.h -index 5101d38d..07e947f8 100644 ---- a/lib/socket_srt.h -+++ b/lib/socket_srt.h -@@ -97,7 +97,7 @@ namespace Socket{ - public: - SRTServer(); - SRTServer(int existingSock); -- SRTServer(int port, std::string hostname, bool nonblock = false, const std::string &_direction = "input"); -+ SRTServer(int port, std::string hostname, std::map params, bool nonblock = false, const std::string &_direction = "input"); - SRTConnection accept(bool nonblock = false, const std::string &direction = "input"); - void setBlocking(bool blocking); - bool connected() const; -diff --git a/src/input/input_tssrt.cpp b/src/input/input_tssrt.cpp -index 4a7adcef..d12f9cba 100644 ---- a/src/input/input_tssrt.cpp -+++ b/src/input/input_tssrt.cpp -@@ -137,7 +137,9 @@ namespace Mist{ - HTTP::URL u(source); - INFO_MSG("Parsed url: %s", u.getUrl().c_str()); - if (Socket::interpretSRTMode(u) == "listener"){ -- sSock = Socket::SRTServer(u.getPort(), u.host, false); -+ std::map arguments; -+ HTTP::parseVars(u.args, arguments); -+ sSock = Socket::SRTServer(u.getPort(), u.host, arguments, false); - struct sigaction new_action; - struct sigaction cur_action; - new_action.sa_sigaction = signal_handler; -diff --git a/src/output/output_tssrt.cpp b/src/output/output_tssrt.cpp -index aa06189e..fb07b11e 100644 ---- a/src/output/output_tssrt.cpp -+++ b/src/output/output_tssrt.cpp -@@ -434,7 +434,8 @@ int main(int argc, char *argv[]){ - sigaction(SIGUSR1, &new_action, NULL); - } - if (conf.getInteger("port") && conf.getString("interface").size()){ -- server_socket = Socket::SRTServer(conf.getInteger("port"), conf.getString("interface"), false, "output"); -+ std::map arguments; -+ server_socket = Socket::SRTServer(conf.getInteger("port"), conf.getString("interface"), arguments, false, "output"); - } - if (!server_socket.connected()){ - DEVEL_MSG("Failure to open socket"); --- -2.25.1 - - -From 508506c241b2980d1aed607cd269ea60e5265e41 Mon Sep 17 00:00:00 2001 -From: Thulinma -Date: Tue, 5 Jan 2021 15:14:04 +0100 -Subject: [PATCH 02/38] Fixes to UDP socket behaviour - ---- - lib/socket.cpp | 25 ++++++++++++++++++++----- - 1 file changed, 20 insertions(+), 5 deletions(-) - -diff --git a/lib/socket.cpp b/lib/socket.cpp -index 22f5e32b..865c3be7 100644 ---- a/lib/socket.cpp -+++ b/lib/socket.cpp -@@ -1713,15 +1713,19 @@ void Socket::UDPConnection::SetDestination(std::string destIp, uint32_t port){ - memset(&hints, 0, sizeof(struct addrinfo)); - hints.ai_family = family; - hints.ai_socktype = SOCK_DGRAM; -- hints.ai_flags = AI_ADDRCONFIG | AI_V4MAPPED; -+ hints.ai_flags = AI_ADDRCONFIG | AI_ALL; - hints.ai_protocol = IPPROTO_UDP; - hints.ai_canonname = NULL; - hints.ai_addr = NULL; - hints.ai_next = NULL; - int s = getaddrinfo(destIp.c_str(), ss.str().c_str(), &hints, &result); - if (s != 0){ -- FAIL_MSG("Could not connect UDP socket to %s:%i! Error: %s", destIp.c_str(), port, gai_strmagic(s)); -- return; -+ hints.ai_family = AF_UNSPEC; -+ s = getaddrinfo(destIp.c_str(), ss.str().c_str(), &hints, &result); -+ if (s != 0){ -+ FAIL_MSG("Could not connect UDP socket to %s:%i! Error: %s", destIp.c_str(), port, gai_strmagic(s)); -+ return; -+ } - } - - for (rp = result; rp != NULL; rp = rp->ai_next){ -@@ -1745,7 +1749,12 @@ void Socket::UDPConnection::SetDestination(std::string destIp, uint32_t port){ - bind(boundPort, boundAddr, boundMulti); - } - } -- HIGH_MSG("Set UDP destination: %s:%d (%s)", destIp.c_str(), port, addrFam(family)); -+ { -+ std::string trueDest; -+ uint32_t truePort; -+ GetDestination(trueDest, truePort); -+ HIGH_MSG("Set UDP destination: %s:%d => %s:%d (%s)", destIp.c_str(), port, trueDest.c_str(), truePort, addrFam(family)); -+ } - freeaddrinfo(result); - return; - //\todo Possibly detect and handle failure -@@ -1881,6 +1890,12 @@ uint16_t Socket::UDPConnection::bind(int port, std::string iface, const std::str - for (rp = addr_result; rp != NULL; rp = rp->ai_next){ - sock = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol); - if (sock == -1){continue;} -+ if (rp->ai_family == AF_INET6){ -+ const int optval = 0; -+ if (setsockopt(sock, SOL_SOCKET, IPV6_V6ONLY, &optval, sizeof(optval)) < 0){ -+ WARN_MSG("Could not set IPv6 UDP socket to be dual-stack! %s", strerror(errno)); -+ } -+ } - checkRecvBuf(); - char human_addr[INET6_ADDRSTRLEN]; - char human_port[16]; -@@ -1909,7 +1924,7 @@ uint16_t Socket::UDPConnection::bind(int port, std::string iface, const std::str - if (multicast){ - const int optval = 1; - if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval)) < 0){ -- WARN_MSG("Could not set multicast UDP socket re-use!"); -+ WARN_MSG("Could not set multicast UDP socket re-use! %s", strerror(errno)); - } - } - if (::bind(sock, rp->ai_addr, rp->ai_addrlen) == 0){ --- -2.25.1 - - -From ed9910d587b5504d38e4f7ef2777f2ea7bd9b760 Mon Sep 17 00:00:00 2001 -From: Thulinma -Date: Wed, 3 Aug 2022 14:58:05 +0200 -Subject: [PATCH 03/38] Fix for list limit in HLS/CMAF outputs when using - non-live - ---- - src/output/output_cmaf.cpp | 2 +- - src/output/output_hls.cpp | 2 +- - 2 files changed, 2 insertions(+), 2 deletions(-) - -diff --git a/src/output/output_cmaf.cpp b/src/output/output_cmaf.cpp -index 756fceb8..4582a8b6 100644 ---- a/src/output/output_cmaf.cpp -+++ b/src/output/output_cmaf.cpp -@@ -284,7 +284,7 @@ namespace Mist{ - requestTid, - M.biggestFragment(timingTid) / 1000, - atol(H.GetVar("iMsn").c_str()), -- config->getInteger("listlimit"), -+ M.getLive()? config->getInteger("listlimit") : 0, - urlPrefix, - systemBoot, - bootMsOffset, -diff --git a/src/output/output_hls.cpp b/src/output/output_hls.cpp -index 93410f54..19d083e6 100644 ---- a/src/output/output_hls.cpp -+++ b/src/output/output_hls.cpp -@@ -173,7 +173,7 @@ namespace Mist{ - requestTid, - M.biggestFragment(timingTid) / 1000, - atol(H.GetVar("iMsn").c_str()), -- config->getInteger("listlimit"), -+ M.getLive()? config->getInteger("listlimit") : 0, - urlPrefix, - systemBoot, - bootMsOffset, --- -2.25.1 - - -From c37aac8898484903da8c2095d29125bf69888c0b Mon Sep 17 00:00:00 2001 -From: Thulinma -Date: Thu, 4 Aug 2022 09:14:51 +0200 -Subject: [PATCH 04/38] Remove "Rate=..." debug message from RTMP push output - ---- - src/output/output_rtmp.cpp | 1 - - 1 file changed, 1 deletion(-) - -diff --git a/src/output/output_rtmp.cpp b/src/output/output_rtmp.cpp -index 69d5f91e..16a400d0 100644 ---- a/src/output/output_rtmp.cpp -+++ b/src/output/output_rtmp.cpp -@@ -618,7 +618,6 @@ namespace Mist{ - - if (type == "audio"){ - uint32_t rate = M.getRate(thisIdx); -- WARN_MSG("Rate=%i", rate); - rtmpheader[7] = 0x08; - if (codec == "AAC"){ - dataheader[0] += 0xA0; --- -2.25.1 - - -From 4084768a3eb72026e901118e05fd17115dd785b3 Mon Sep 17 00:00:00 2001 -From: Thulinma -Date: Tue, 5 Jul 2022 15:21:04 +0200 -Subject: [PATCH 05/38] Improved incoming push accept timing - ---- - src/output/output.cpp | 7 ++++--- - 1 file changed, 4 insertions(+), 3 deletions(-) - -diff --git a/src/output/output.cpp b/src/output/output.cpp -index 5c785bb0..1fa86eee 100644 ---- a/src/output/output.cpp -+++ b/src/output/output.cpp -@@ -2050,7 +2050,7 @@ namespace Mist{ - INFO_MSG("Waiting for stream reset before attempting push input accept (%" PRIu64 " <= %" PRIu64 "+500)", twoTime, oneTime); - while (streamStatus != STRMSTAT_OFF && keepGoing()){ - userSelect.clear(); -- Util::wait(1000); -+ Util::wait(250); - streamStatus = Util::getStreamStatus(streamName); - } - reconnect(); -@@ -2059,14 +2059,15 @@ namespace Mist{ - while (((streamStatus != STRMSTAT_WAIT && streamStatus != STRMSTAT_READY) || !meta) && keepGoing()){ - INFO_MSG("Waiting for %s buffer to be ready... (%u)", streamName.c_str(), streamStatus); - disconnect(); -- Util::wait(1000); - streamStatus = Util::getStreamStatus(streamName); - if (streamStatus == STRMSTAT_OFF || streamStatus == STRMSTAT_WAIT || streamStatus == STRMSTAT_READY){ - INFO_MSG("Reconnecting to %s buffer... (%u)", streamName.c_str(), streamStatus); -- Util::wait(500); - reconnect(); - streamStatus = Util::getStreamStatus(streamName); - } -+ if (((streamStatus != STRMSTAT_WAIT && streamStatus != STRMSTAT_READY) || !meta) && keepGoing()){ -+ Util::wait(100); -+ } - } - if (streamStatus == STRMSTAT_READY || streamStatus == STRMSTAT_WAIT){reconnect();} - if (!meta){ --- -2.25.1 - - -From 14bc94ece51affb7cb6d52a491b97921ed420587 Mon Sep 17 00:00:00 2001 -From: Thulinma -Date: Tue, 12 Jul 2022 15:47:29 +0200 -Subject: [PATCH 06/38] Prevent eternal sleep when waiting for RelAccX - structure to become ready - ---- - lib/util.cpp | 10 +++++++++- - 1 file changed, 9 insertions(+), 1 deletion(-) - -diff --git a/lib/util.cpp b/lib/util.cpp -index d522a151..8dd75727 100644 ---- a/lib/util.cpp -+++ b/lib/util.cpp -@@ -446,7 +446,15 @@ namespace Util{ - hdrOffset = (uint16_t*)(p+26); - hdrEndPos = (uint64_t*)(p+28); - if (waitReady){ -- while (!isReady()){Util::sleep(50);} -+ uint64_t maxWait = Util::bootMS() + 10000; -+ while (!isReady()){ -+ if (Util::bootMS() > maxWait){ -+ FAIL_MSG("Waiting for RelAccX structure to be ready timed out, aborting"); -+ p = 0; -+ return; -+ } -+ Util::sleep(50); -+ } - } - if (isReady()){ - uint16_t offset = RAXHDR_FIELDOFFSET; --- -2.25.1 - - -From 5bdd4a416e964ae0cfcc56d99c062691d1b3931d Mon Sep 17 00:00:00 2001 -From: Thulinma -Date: Thu, 14 Jul 2022 10:45:53 +0200 -Subject: [PATCH 07/38] Fix load balancer CPU usage - ---- - src/input/input_balancer.cpp | 3 +++ - 1 file changed, 3 insertions(+) - -diff --git a/src/input/input_balancer.cpp b/src/input/input_balancer.cpp -index 448e6eff..36db5709 100644 ---- a/src/input/input_balancer.cpp -+++ b/src/input/input_balancer.cpp -@@ -140,6 +140,9 @@ namespace Mist{ - startTime = 0; // note success - break; // break out of while loop - } -+ }else{ -+ //Prevent 100% CPU usage if the response is slow -+ Util::sleep(25); - } - } - if (startTime){ --- -2.25.1 - - -From fffe98804cb9c126ec91b7a7b67f2c917561ce02 Mon Sep 17 00:00:00 2001 -From: Thulinma -Date: Thu, 30 Jun 2022 14:04:34 +0200 -Subject: [PATCH 08/38] Fixed TS SRT input not closing the connection when - stopping for internal reasons rather than external reasons - ---- - src/input/input_tssrt.cpp | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/src/input/input_tssrt.cpp b/src/input/input_tssrt.cpp -index d12f9cba..6cd614e7 100644 ---- a/src/input/input_tssrt.cpp -+++ b/src/input/input_tssrt.cpp -@@ -241,6 +241,7 @@ namespace Mist{ - } - // If we are here: we have a proper connection (either accepted or pull input) and should start parsing it as such - Input::streamMainLoop(); -+ srtConn.close(); - } - - bool inputTSSRT::needsLock(){return false;} --- -2.25.1 - - -From 267a74f0f6d44b928b99312b356a3080579fdc1c Mon Sep 17 00:00:00 2001 -From: Thulinma -Date: Thu, 4 Aug 2022 16:43:04 +0200 -Subject: [PATCH 09/38] Fix track selector logic when multiple selections are - considered - ---- - lib/stream.cpp | 33 +++++++++++++++++++++++++++++++-- - 1 file changed, 31 insertions(+), 2 deletions(-) - -diff --git a/lib/stream.cpp b/lib/stream.cpp -index 6036a8fd..715015d6 100644 ---- a/lib/stream.cpp -+++ b/lib/stream.cpp -@@ -1249,7 +1249,7 @@ std::set Util::wouldSelect(const DTSC::Meta &M, const std::map result; - - /*LTS-START*/ -- bool noSelAudio = false, noSelVideo = false, noSelSub = false; -+ bool noSelAudio = false, noSelVideo = false, noSelSub = false, noSelMeta = false; - // Then, select the tracks we've been asked to select. - if (targetParams.count("audio") && targetParams.at("audio").size()){ - if (targetParams.at("audio") != "-1" && targetParams.at("audio") != "none"){ -@@ -1270,6 +1270,11 @@ std::set Util::wouldSelect(const DTSC::Meta &M, const std::map tracks = Util::findTracks(M, capa, "meta", targetParams.at("meta"), UA); -+ result.insert(tracks.begin(), tracks.end()); -+ noSelMeta = true; -+ } - /*LTS-END*/ - - std::set validTracks = M.getValidTracks(); -@@ -1291,6 +1296,7 @@ std::set Util::wouldSelect(const DTSC::Meta &M, const std::map Util::wouldSelect(const DTSC::Meta &M, const std::map Util::wouldSelect(const DTSC::Meta &M, const std::map 0){ - jsonForEachConst((*it), itb){ - if ((*itb).size() > 0){ -@@ -1360,6 +1368,26 @@ std::set Util::wouldSelect(const DTSC::Meta &M, const std::map::iterator itd = validTracks.begin(); itd != validTracks.end(); itd++){ -+ if ((!byType && M.getCodec(*itd) == strRef.substr(shift)) || -+ (byType && M.getType(*itd) == strRef.substr(shift)) || strRef.substr(shift) == "*"){ -+ // user-agent-check -+ bool problems = false; -+ if (capa.isMember("exceptions") && capa["exceptions"].isObject() && -+ capa["exceptions"].size()){ -+ jsonForEachConst(capa["exceptions"], ex){ -+ if (ex.key() == "codec:" + strRef.substr(shift)){ -+ problems = !Util::checkException(*ex, UA); -+ break; -+ } -+ } -+ } -+ if (problems){break;} -+ extraCounter++; - if (!multiSel){break;} - } - } -@@ -1367,8 +1395,9 @@ std::set Util::wouldSelect(const DTSC::Meta &M, const std::map bestSoFarCount){ -+ if (selCounter > bestSoFarCount || (selCounter == bestSoFarCount && extraCounter > bestSoFarCountExtra)){ - bestSoFarCount = selCounter; -+ bestSoFarCountExtra = extraCounter; - bestSoFar = index; - HIGH_MSG("Matched %u: %s", selCounter, (*it).toString().c_str()); - } --- -2.25.1 - - -From 3734c90544a68d03a1a69438f00f215c517fec01 Mon Sep 17 00:00:00 2001 -From: Phencys -Date: Sun, 20 Sep 2020 20:31:17 +0200 -Subject: [PATCH 10/38] Added support for raw passthrough of MPEG2-TS data - ---- - src/input/input_ts.cpp | 73 ++++++++++++++++++++++++++++++++--- - src/input/input_ts.h | 5 +++ - src/input/input_tsrist.cpp | 54 +++++++++++++++++++++++++- - src/input/input_tsrist.h | 6 +++ - src/input/input_tssrt.cpp | 35 +++++++++++++++++ - src/input/input_tssrt.h | 4 ++ - src/io.cpp | 7 ++++ - src/io.h | 1 + - src/output/output_httpts.cpp | 1 + - src/output/output_ts.cpp | 1 + - src/output/output_ts_base.cpp | 5 +++ - src/output/output_tsrist.cpp | 1 + - src/output/output_tssrt.cpp | 1 + - 13 files changed, 187 insertions(+), 7 deletions(-) - -diff --git a/src/input/input_ts.cpp b/src/input/input_ts.cpp -index 98300411..cfe3f7cd 100644 ---- a/src/input/input_ts.cpp -+++ b/src/input/input_ts.cpp -@@ -134,6 +134,13 @@ void parseThread(void *mistIn){ - } - } - } -+ -+ //On shutdown, make sure to clean up stream buffer -+ if (idx != INVALID_TRACK_ID){ -+ tthread::lock_guard guard(threadClaimMutex); -+ input->liveFinalize(idx); -+ } -+ - std::string reason = "unknown reason"; - if (!(Util::bootSecs() - threadTimer[tid] < THREAD_TIMEOUT)){reason = "thread timeout";} - if (!cfgPointer->is_active){reason = "input shutting down";} -@@ -155,6 +162,9 @@ namespace Mist{ - /// Constructor of TS Input - /// \arg cfg Util::Config that contains all current configurations. - inputTS::inputTS(Util::Config *cfg) : Input(cfg){ -+ rawMode = false; -+ rawIdx = INVALID_TRACK_ID; -+ lastRawPacket = 0; - capa["name"] = "TS"; - capa["desc"] = - "This input allows you to stream MPEG2-TS data from static files (/*.ts), streamed files " -@@ -188,6 +198,7 @@ namespace Mist{ - capa["codecs"][0u][1u].append("AC3"); - capa["codecs"][0u][1u].append("MP2"); - capa["codecs"][0u][1u].append("opus"); -+ capa["codecs"][1u][0u].append("rawts"); - inFile = NULL; - inputProcess = 0; - isFinished = false; -@@ -232,6 +243,16 @@ namespace Mist{ - "Alternative stream to load for playback when there is no active broadcast"; - capa["optional"]["fallback_stream"]["type"] = "str"; - capa["optional"]["fallback_stream"]["default"] = ""; -+ -+ capa["optional"]["raw"]["name"] = "Raw input mode"; -+ capa["optional"]["raw"]["help"] = "Enable raw MPEG-TS passthrough mode"; -+ capa["optional"]["raw"]["option"] = "--raw"; -+ -+ JSON::Value option; -+ option["long"] = "raw"; -+ option["short"] = "R"; -+ option["help"] = "Enable raw MPEG-TS passthrough mode"; -+ config->addOption("raw", option); - } - - inputTS::~inputTS(){ -@@ -257,6 +278,10 @@ namespace Mist{ - /// Live Setup of TS Input - bool inputTS::preRun(){ - INFO_MSG("Prerun: %s", config->getString("input").c_str()); -+ -+ rawMode = config->getBool("raw"); -+ if (rawMode){INFO_MSG("Entering raw mode");} -+ - // streamed standard input - if (config->getString("input") == "-"){ - standAlone = false; -@@ -520,9 +545,28 @@ namespace Mist{ - } - if (tcpCon.Received().available(188) && tcpCon.Received().get()[0] == 0x47){ - std::string newData = tcpCon.Received().remove(188); -- tsBuf.FromPointer(newData.data()); -- liveStream.add(tsBuf); -- if (!liveStream.isDataTrack(tsBuf.getPID())){liveStream.parse(tsBuf.getPID());} -+ if (rawMode){ -+ keepAlive(); -+ rawBuffer.append(newData); -+ if (rawBuffer.size() >= 1316 && (lastRawPacket == 0 || lastRawPacket != Util::bootMS())){ -+ if (rawIdx == INVALID_TRACK_ID){ -+ rawIdx = meta.addTrack(); -+ meta.setType(rawIdx, "meta"); -+ meta.setCodec(rawIdx, "rawts"); -+ meta.setID(rawIdx, 1); -+ userSelect[rawIdx].reload(streamName, rawIdx, COMM_STATUS_SOURCE); -+ } -+ uint64_t packetTime = Util::bootMS(); -+ thisPacket.genericFill(packetTime, 0, 1, rawBuffer, rawBuffer.size(), 0, 0); -+ bufferLivePacket(thisPacket); -+ lastRawPacket = packetTime; -+ rawBuffer.truncate(0); -+ } -+ }else { -+ tsBuf.FromPointer(newData.data()); -+ liveStream.add(tsBuf); -+ if (!liveStream.isDataTrack(tsBuf.getPID())){liveStream.parse(tsBuf.getPID());} -+ } - } - } - noDataSince = Util::bootSecs(); -@@ -543,7 +587,26 @@ namespace Mist{ - gettingData = true; - INFO_MSG("Now receiving UDP data..."); - } -- assembler.assemble(liveStream, udpCon.data, udpCon.data.size()); -+ if (rawMode){ -+ keepAlive(); -+ rawBuffer.append(udpCon.data, udpCon.data.size()); -+ if (rawBuffer.size() >= 1316 && (lastRawPacket == 0 || lastRawPacket != Util::bootMS())){ -+ if (rawIdx == INVALID_TRACK_ID){ -+ rawIdx = meta.addTrack(); -+ meta.setType(rawIdx, "meta"); -+ meta.setCodec(rawIdx, "rawts"); -+ meta.setID(rawIdx, 1); -+ userSelect[rawIdx].reload(streamName, rawIdx, COMM_STATUS_SOURCE); -+ } -+ uint64_t packetTime = Util::bootMS(); -+ thisPacket.genericFill(packetTime, 0, 1, rawBuffer, rawBuffer.size(), 0, 0); -+ bufferLivePacket(thisPacket); -+ lastRawPacket = packetTime; -+ rawBuffer.truncate(0); -+ } -+ }else{ -+ assembler.assemble(liveStream, udpCon.data, udpCon.data.size()); -+ } - } - if (!received){ - Util::sleep(100); -@@ -578,7 +641,7 @@ namespace Mist{ - } - - std::set activeTracks = liveStream.getActiveTracks(); -- { -+ if (!rawMode){ - tthread::lock_guard guard(threadClaimMutex); - if (hasStarted && !threadTimer.size()){ - if (!isAlwaysOn()){ -diff --git a/src/input/input_ts.h b/src/input/input_ts.h -index e810ddb4..3116725d 100644 ---- a/src/input/input_ts.h -+++ b/src/input/input_ts.h -@@ -41,6 +41,11 @@ namespace Mist{ - pid_t inputProcess; - size_t tmpIdx; - bool isFinished; -+ -+ bool rawMode; -+ Util::ResizeablePointer rawBuffer; -+ size_t rawIdx; -+ uint64_t lastRawPacket; - }; - }// namespace Mist - -diff --git a/src/input/input_tsrist.cpp b/src/input/input_tsrist.cpp -index 18b05c9d..5b95aa9f 100644 ---- a/src/input/input_tsrist.cpp -+++ b/src/input/input_tsrist.cpp -@@ -66,6 +66,10 @@ namespace Mist{ - /// Constructor of TS Input - /// \arg cfg Util::Config that contains all current configurations. - inputTSRIST::inputTSRIST(Util::Config *cfg) : Input(cfg){ -+ rawMode = false; -+ rawIdx = INVALID_TRACK_ID; -+ lastRawPacket = 0; -+ hasRaw = false; - connPtr = this; - cnfPtr = config; - -@@ -96,6 +100,7 @@ namespace Mist{ - capa["codecs"][0u][1u].append("AC3"); - capa["codecs"][0u][1u].append("MP2"); - capa["codecs"][0u][1u].append("opus"); -+ capa["codecs"][1u][0u].append("rawts"); - - JSON::Value option; - option["arg"] = "integer"; -@@ -132,6 +137,15 @@ namespace Mist{ - capa["optional"]["profile"]["type"] = "select"; - capa["optional"]["profile"]["option"] = "--profile"; - -+ capa["optional"]["raw"]["name"] = "Raw input mode"; -+ capa["optional"]["raw"]["help"] = "Enable raw MPEG-TS passthrough mode"; -+ capa["optional"]["raw"]["option"] = "--raw"; -+ -+ option["long"] = "raw"; -+ option["short"] = "R"; -+ option["help"] = "Enable raw MPEG-TS passthrough mode"; -+ config->addOption("raw", option); -+ - lastTimeStamp = 0; - timeStampOffset = 0; - receiver_ctx = 0; -@@ -146,6 +160,9 @@ namespace Mist{ - - /// Live Setup of SRT Input. Runs only if we are the "main" thread - bool inputTSRIST::preRun(){ -+ rawMode = config->getBool("raw"); -+ if (rawMode){INFO_MSG("Entering raw mode");} -+ - std::string source = config->getString("input"); - standAlone = false; - HTTP::URL u(source); -@@ -161,6 +178,20 @@ namespace Mist{ - // Retrieve the next packet to be played from the srt connection. - void inputTSRIST::getNext(size_t idx){ - thisPacket.null(); -+ if (rawMode){ -+ //Set to false so the other thread knows its safe to fill -+ hasRaw = false; -+ while (!hasRaw && config->is_active){ -+ Util::sleep(50); -+ if (!bufferActive()){ -+ Util::logExitReason("Buffer shut down"); -+ return; -+ } -+ } -+ //if hasRaw, thisPacket has been filled by the other thread -+ return; -+ } -+ - while (!thisPacket && config->is_active){ - if (tsStream.hasPacket()){ - tsStream.getEarliestPacket(thisPacket); -@@ -228,8 +259,27 @@ namespace Mist{ - } - - void inputTSRIST::addData(const char * ptr, size_t len){ -- for (size_t o = 0; o <= len-188; o += 188){ -- tsStream.parse((char*)ptr+o, 0); -+ for (size_t o = 0; o+188 <= len; o += 188){ -+ if (rawMode){ -+ rawBuffer.append(ptr+o, 188); -+ if (!hasRaw && rawBuffer.size() >= 1316 && (lastRawPacket == 0 || lastRawPacket != Util::bootMS())){ -+ if (rawIdx == INVALID_TRACK_ID){ -+ rawIdx = meta.addTrack(); -+ meta.setType(rawIdx, "meta"); -+ meta.setCodec(rawIdx, "rawts"); -+ meta.setID(rawIdx, 1); -+ userSelect[rawIdx].reload(streamName, rawIdx, COMM_STATUS_SOURCE); -+ } -+ thisTime = Util::bootMS(); -+ thisIdx = rawIdx; -+ thisPacket.genericFill(thisTime, 0, 1, rawBuffer, rawBuffer.size(), 0, 0); -+ lastRawPacket = thisTime; -+ rawBuffer.truncate(0); -+ hasRaw = true; -+ } -+ }else{ -+ tsStream.parse((char*)ptr+o, 0); -+ } - } - } - -diff --git a/src/input/input_tsrist.h b/src/input/input_tsrist.h -index 3bc29e1c..731f9b04 100644 ---- a/src/input/input_tsrist.h -+++ b/src/input/input_tsrist.h -@@ -33,6 +33,12 @@ namespace Mist{ - virtual void connStats(Comms::Statistics &statComm); - - struct rist_ctx *receiver_ctx; -+ -+ bool rawMode; -+ Util::ResizeablePointer rawBuffer; -+ size_t rawIdx; -+ uint64_t lastRawPacket; -+ bool hasRaw; - }; - }// namespace Mist - -diff --git a/src/input/input_tssrt.cpp b/src/input/input_tssrt.cpp -index 6cd614e7..da69be29 100644 ---- a/src/input/input_tssrt.cpp -+++ b/src/input/input_tssrt.cpp -@@ -25,6 +25,7 @@ - Util::Config *cfgPointer = NULL; - std::string baseStreamName; - Socket::SRTServer sSock; -+bool rawMode = false; - - void (*oldSignal)(int, siginfo_t *,void *) = 0; - -@@ -49,6 +50,8 @@ namespace Mist{ - /// Constructor of TS Input - /// \arg cfg Util::Config that contains all current configurations. - inputTSSRT::inputTSSRT(Util::Config *cfg, SRTSOCKET s) : Input(cfg){ -+ rawIdx = INVALID_TRACK_ID; -+ lastRawPacket = 0; - capa["name"] = "TSSRT"; - capa["desc"] = "This input allows for processing MPEG2-TS-based SRT streams. Use mode=listener " - "for push input."; -@@ -66,6 +69,7 @@ namespace Mist{ - capa["codecs"][0u][1u].append("AC3"); - capa["codecs"][0u][1u].append("MP2"); - capa["codecs"][0u][1u].append("opus"); -+ capa["codecs"][1u][0u].append("rawts"); - - JSON::Value option; - option["arg"] = "integer"; -@@ -103,7 +107,16 @@ namespace Mist{ - capa["optional"]["acceptable"]["select"][2u][0u] = 2; - capa["optional"]["acceptable"]["select"][2u][1u] = "Disallow non-matching streamid"; - -+ capa["optional"]["raw"]["name"] = "Raw input mode"; -+ capa["optional"]["raw"]["help"] = "Enable raw MPEG-TS passthrough mode"; -+ capa["optional"]["raw"]["option"] = "--raw"; - -+ option.null(); -+ option["long"] = "raw"; -+ option["short"] = "R"; -+ option["help"] = "Enable raw MPEG-TS passthrough mode"; -+ config->addOption("raw", option); -+ - // Setup if we are called form with a thread for push-based input. - if (s != -1){ - srtConn = Socket::SRTConnection(s); -@@ -131,6 +144,8 @@ namespace Mist{ - - /// Live Setup of SRT Input. Runs only if we are the "main" thread - bool inputTSSRT::preRun(){ -+ rawMode = config->getBool("raw"); -+ if (rawMode){INFO_MSG("Entering raw mode");} - if (srtConn.getSocket() == -1){ - std::string source = config->getString("input"); - standAlone = false; -@@ -183,6 +198,26 @@ namespace Mist{ - - size_t recvSize = srtConn.RecvNow(); - if (recvSize){ -+ if (rawMode){ -+ keepAlive(); -+ rawBuffer.append(srtConn.recvbuf, recvSize); -+ if (rawBuffer.size() >= 1316 && (lastRawPacket == 0 || lastRawPacket != Util::bootMS())){ -+ if (rawIdx == INVALID_TRACK_ID){ -+ rawIdx = meta.addTrack(); -+ meta.setType(rawIdx, "meta"); -+ meta.setCodec(rawIdx, "rawts"); -+ meta.setID(rawIdx, 1); -+ userSelect[rawIdx].reload(streamName, rawIdx, COMM_STATUS_SOURCE); -+ } -+ uint64_t packetTime = Util::bootMS(); -+ thisPacket.genericFill(packetTime, 0, 1, rawBuffer, rawBuffer.size(), 0, 0); -+ bufferLivePacket(thisPacket); -+ lastRawPacket = packetTime; -+ rawBuffer.truncate(0); -+ return; -+ } -+ continue; -+ } - if (assembler.assemble(tsStream, srtConn.recvbuf, recvSize, true)){hasPacket = tsStream.hasPacket();} - }else if (srtConn){ - // This should not happen as the SRT socket is read blocking and won't return until there is -diff --git a/src/input/input_tssrt.h b/src/input/input_tssrt.h -index 40fa05c1..4f337b48 100644 ---- a/src/input/input_tssrt.h -+++ b/src/input/input_tssrt.h -@@ -41,6 +41,10 @@ namespace Mist{ - Socket::SRTConnection srtConn; - bool singularFlag; - virtual void connStats(Comms::Statistics &statComm); -+ -+ Util::ResizeablePointer rawBuffer; -+ size_t rawIdx; -+ uint64_t lastRawPacket; - }; - }// namespace Mist - -diff --git a/src/io.cpp b/src/io.cpp -index 6a5844c4..fea32193 100644 ---- a/src/io.cpp -+++ b/src/io.cpp -@@ -292,6 +292,13 @@ namespace Mist{ - tPages.setInt("avail", pageOffset + packDataLen, pageIdx); - } - -+ /// Wraps up the buffering of a shared memory data page -+ /// \param idx The track index of the page to finalize -+ void InOutBase::liveFinalize(size_t idx){ -+ if (!livePage.count(idx)){return;} -+ bufferFinalize(idx, livePage[idx]); -+ } -+ - /// Wraps up the buffering of a shared memory data page - /// \param idx The track index of the page to finalize - void InOutBase::bufferFinalize(size_t idx, IPC::sharedPage & page){ -diff --git a/src/io.h b/src/io.h -index d839952a..e6545929 100644 ---- a/src/io.h -+++ b/src/io.h -@@ -21,6 +21,7 @@ namespace Mist{ - - bool bufferStart(size_t idx, uint32_t pageNumber, IPC::sharedPage & page, DTSC::Meta & aMeta); - void bufferFinalize(size_t idx, IPC::sharedPage & page); -+ void liveFinalize(size_t idx); - bool isCurrentLivePage(size_t idx, uint32_t pageNumber); - void bufferRemove(size_t idx, uint32_t pageNumber); - void bufferLivePacket(const DTSC::Packet &packet); -diff --git a/src/output/output_httpts.cpp b/src/output/output_httpts.cpp -index 6aab2b9b..9daa9dc3 100644 ---- a/src/output/output_httpts.cpp -+++ b/src/output/output_httpts.cpp -@@ -135,6 +135,7 @@ namespace Mist{ - capa["codecs"][0u][1u].append("+AC3"); - capa["codecs"][0u][1u].append("+MP2"); - capa["codecs"][0u][1u].append("+opus"); -+ capa["codecs"][1u][0u].append("rawts"); - capa["methods"][0u]["handler"] = "http"; - capa["methods"][0u]["type"] = "html5/video/mpeg"; - capa["methods"][0u]["hrn"] = "TS HTTP progressive"; -diff --git a/src/output/output_ts.cpp b/src/output/output_ts.cpp -index d2addc24..561bf9aa 100644 ---- a/src/output/output_ts.cpp -+++ b/src/output/output_ts.cpp -@@ -179,6 +179,7 @@ namespace Mist{ - capa["codecs"][0u][1u].append("+AC3"); - capa["codecs"][0u][1u].append("+MP2"); - capa["codecs"][0u][1u].append("+opus"); -+ capa["codecs"][1u][0u].append("rawts"); - cfg->addConnectorOptions(8888, capa); - config = cfg; - capa["push_urls"].append("tsudp://*"); -diff --git a/src/output/output_ts_base.cpp b/src/output/output_ts_base.cpp -index 17b2a8ce..c4163e7f 100644 ---- a/src/output/output_ts_base.cpp -+++ b/src/output/output_ts_base.cpp -@@ -75,6 +75,11 @@ namespace Mist{ - size_t dataLen = 0; - thisPacket.getString("data", dataPointer, dataLen); // data - -+ if (codec == "rawts"){ -+ for (size_t i = 0; i+188 <= dataLen; i+=188){sendTS(dataPointer+i, 188);} -+ return; -+ } -+ - packTime *= 90; - std::string bs; - // prepare bufferstring -diff --git a/src/output/output_tsrist.cpp b/src/output/output_tsrist.cpp -index ee57d8ee..c4e03f80 100644 ---- a/src/output/output_tsrist.cpp -+++ b/src/output/output_tsrist.cpp -@@ -202,6 +202,7 @@ namespace Mist{ - capa["codecs"][0u][1u].append("+AC3"); - capa["codecs"][0u][1u].append("+MP2"); - capa["codecs"][0u][1u].append("+opus"); -+ capa["codecs"][1u][0u].append("rawts"); - - capa["optional"]["profile"]["name"] = "RIST profile"; - capa["optional"]["profile"]["help"] = "RIST profile to use"; -diff --git a/src/output/output_tssrt.cpp b/src/output/output_tssrt.cpp -index fb07b11e..db07dc91 100644 ---- a/src/output/output_tssrt.cpp -+++ b/src/output/output_tssrt.cpp -@@ -200,6 +200,7 @@ namespace Mist{ - capa["codecs"][0u][1u].append("AC3"); - capa["codecs"][0u][1u].append("MP2"); - capa["codecs"][0u][1u].append("opus"); -+ capa["codecs"][1u][0u].append("rawts"); - cfg->addConnectorOptions(8889, capa); - config = cfg; - capa["push_urls"].append("srt://*"); --- -2.25.1 - - -From 6c117b63cfe85a204b48dfddddb355f416b39364 Mon Sep 17 00:00:00 2001 -From: Alex Kordic -Date: Thu, 4 Aug 2022 14:49:57 +0200 -Subject: [PATCH 11/38] Add s3 protocol to `URIReader` - ---- - lib/timing.cpp | 12 +++++ - lib/timing.h | 1 + - lib/urireader.cpp | 114 +++++++++++++++++++++++++++++++++++++++++++++- - 3 files changed, 125 insertions(+), 2 deletions(-) - -diff --git a/lib/timing.cpp b/lib/timing.cpp -index 036f0b29..f339f4fe 100644 ---- a/lib/timing.cpp -+++ b/lib/timing.cpp -@@ -114,3 +114,15 @@ std::string Util::getUTCString(uint64_t epoch){ - ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec); - return std::string(result); - } -+ -+std::string Util::getDateString(uint64_t epoch){ -+ char buffer[80]; -+ time_t rawtime = epoch; -+ if (!epoch) { -+ time(&rawtime); -+ } -+ struct tm * timeinfo; -+ timeinfo = localtime(&rawtime); -+ strftime(buffer, sizeof(buffer), "%a, %d %b %Y %H:%M:%S %z", timeinfo); -+ return std::string(buffer); -+} -diff --git a/lib/timing.h b/lib/timing.h -index 107ccee5..b2a4d612 100644 ---- a/lib/timing.h -+++ b/lib/timing.h -@@ -17,4 +17,5 @@ namespace Util{ - uint64_t getNTP(); - uint64_t epoch(); ///< Gets the amount of seconds since 01/01/1970. - std::string getUTCString(uint64_t epoch = 0); -+ std::string getDateString(uint64_t epoch = 0); - }// namespace Util -diff --git a/lib/urireader.cpp b/lib/urireader.cpp -index 80f204c8..ad85b052 100644 ---- a/lib/urireader.cpp -+++ b/lib/urireader.cpp -@@ -3,11 +3,86 @@ - #include "timing.h" - #include "urireader.h" - #include "util.h" -+#include "encode.h" - #include - #include -+#include -+#include - - namespace HTTP{ - -+ // When another protocol needs this, rename struct to HeaderOverride or similar -+ struct HTTPHeadThenGet { -+ bool continueOperation; -+ std::string date, headAuthorization, getAuthorization; -+ -+ HTTPHeadThenGet() : continueOperation(false) {} -+ -+ void prepareHeadHeaders(HTTP::Downloader& downloader) { -+ if(!continueOperation) return; -+ downloader.setHeader("Date", date); -+ downloader.setHeader("Authorization", headAuthorization); -+ } -+ -+ void prepareGetHeaders(HTTP::Downloader& downloader) { -+ if(!continueOperation) return; -+ // .setHeader() overwrites existing header value -+ downloader.setHeader("Date", date); -+ downloader.setHeader("Authorization", getAuthorization); -+ } -+ }; -+ -+#ifndef NOSSL -+ inline bool s3CalculateSignature(std::string& signature, const std::string method, const std::string date, const std::string& requestPath, const std::string& accessKey, const std::string& secret) { -+ std::string toSign = method + "\n\n\n" + date + "\n" + requestPath; -+ unsigned char signatureBytes[MBEDTLS_MD_MAX_SIZE]; -+ const int sha1Size = 20; -+ mbedtls_md_context_t md_ctx = {0}; -+ // TODO: When we use MBEDTLS_MD_SHA512 ? Consult documentation/code -+ const mbedtls_md_info_t *md_info = mbedtls_md_info_from_type(MBEDTLS_MD_SHA1); -+ if (!md_info){ FAIL_MSG("error s3 MBEDTLS_MD_SHA1 unavailable"); return false; } -+ int status = mbedtls_md_setup(&md_ctx, md_info, 1); -+ if(status != 0) { FAIL_MSG("error s3 mbedtls_md_setup error %d", status); return false; } -+ status = mbedtls_md_hmac_starts(&md_ctx, (const unsigned char *)secret.c_str(), secret.size()); -+ if(status != 0) { FAIL_MSG("error s3 mbedtls_md_hmac_starts error %d", status); return false; } -+ status = mbedtls_md_hmac_update(&md_ctx, (const unsigned char *)toSign.c_str(), toSign.size()); -+ if(status != 0) { FAIL_MSG("error s3 mbedtls_md_hmac_update error %d", status); return false; } -+ status = mbedtls_md_hmac_finish(&md_ctx, signatureBytes); -+ if(status != 0) { FAIL_MSG("error s3 mbedtls_md_hmac_finish error %d", status); return false; } -+ std::string base64encoded = Encodings::Base64::encode(std::string((const char*)signatureBytes, sha1Size)); -+ signature = "AWS " + accessKey + ":" + base64encoded; -+ return true; -+ } -+ // Input url == s3+https://s3_key:secret@storage.googleapis.com/alexk-dms-upload-test/testvideo.ts -+ // Transform to: -+ // url=https://storage.googleapis.com/alexk-dms-upload-test/testvideo.ts -+ // header Date: ${Util::getDateString(()} -+ // header Authorization: AWS ${url.user}:${signature} -+ inline HTTPHeadThenGet s3TransformToHttp(HTTP::URL& url) { -+ HTTPHeadThenGet result; -+ result.date = Util::getDateString(); -+ // remove "s3+" prefix -+ url.protocol = url.protocol.erase(0, 3); -+ // Use user and pass to create signature and remove from HTTP request -+ std::string accessKey(url.user), secret(url.pass); -+ url.user = ""; -+ url.pass = ""; -+ std::string requestPath = "/" + Encodings::URL::encode(url.path, "/:=@[]#?&"); -+ if(url.args.size()) requestPath += "?" + url.args; -+ // Calculate Authorization data -+ if(!s3CalculateSignature(result.headAuthorization, "HEAD", result.date, requestPath, accessKey, secret)) { -+ result.continueOperation = false; -+ return result; -+ } -+ if(!s3CalculateSignature(result.getAuthorization, "GET", result.date, requestPath, accessKey, secret)) { -+ result.continueOperation = false; -+ return result; -+ } -+ result.continueOperation = true; -+ return result; -+ } -+#endif // ifndef NOSSL -+ - void URIReader::init(){ - handle = -1; - mapped = 0; -@@ -97,12 +172,45 @@ namespace HTTP{ - } - } - -+ // prepare for s3 and http -+ HTTPHeadThenGet httpHeaderOverride; -+ -+#ifndef NOSSL -+ // In case of s3 URI we prepare HTTP request with AWS authorization and rely on HTTP logic below -+ if (myURI.protocol == "s3+https" || myURI.protocol == "s3+http"){ -+ // Check fallback to global credentials in env vars -+ bool haveCredentials = myURI.user.size() && myURI.pass.size(); -+ if(!haveCredentials) { -+ // Use environment variables -+ char * envValue = std::getenv("S3_ACCESS_KEY_ID"); -+ if(envValue == NULL) { -+ FAIL_MSG("error s3 uri without credentials. Consider setting S3_ACCESS_KEY_ID env var"); -+ return false; -+ } -+ myURI.user = envValue; -+ envValue = std::getenv("S3_SECRET_ACCESS_KEY"); -+ if(envValue == NULL) { -+ FAIL_MSG("error s3 uri without credentials. Consider setting S3_SECRET_ACCESS_KEY env var"); -+ return false; -+ } -+ myURI.pass = envValue; -+ } -+ // Transform s3 url to HTTP request: -+ httpHeaderOverride = s3TransformToHttp(myURI); -+ bool errorInSignatureCalculation = !httpHeaderOverride.continueOperation; -+ if(errorInSignatureCalculation) return false; -+ // Do not return, continue to HTTP case -+ } -+#endif // ifndef NOSSL -+ - // HTTP, stream or regular download? - if (myURI.protocol == "http" || myURI.protocol == "https"){ -- stateType = HTTP::HTTP; -+ stateType = HTTP; -+ downer.clearHeaders(); - -+ // One set of headers specified for HEAD request -+ httpHeaderOverride.prepareHeadHeaders(downer); - // Send HEAD request to determine range request is supported, and get total length -- downer.clearHeaders(); - if (userAgentOverride.size()){downer.setHeader("User-Agent", userAgentOverride);} - if (!downer.head(myURI) || !downer.isOk()){ - FAIL_MSG("Error getting URI info for '%s': %" PRIu32 " %s", myURI.getUrl().c_str(), -@@ -120,6 +228,8 @@ namespace HTTP{ - myURI = downer.lastURL(); - } - -+ // Other set of headers specified for GET request -+ httpHeaderOverride.prepareGetHeaders(downer); - // streaming mode when size is unknown - if (!supportRangeRequest){ - MEDIUM_MSG("URI get without range request: %s, totalsize: %zu", myURI.getUrl().c_str(), totalSize); --- -2.25.1 - - -From df4076a06eeb6fbc0fc20d9ace0091eb476420a2 Mon Sep 17 00:00:00 2001 -From: Thulinma -Date: Wed, 17 Aug 2022 14:57:29 +0200 -Subject: [PATCH 12/38] Added ResizeablePointer::shift operator to shift data - forward in buffer - ---- - lib/util.cpp | 11 +++++++++++ - lib/util.h | 1 + - 2 files changed, 12 insertions(+) - -diff --git a/lib/util.cpp b/lib/util.cpp -index 8dd75727..d4ff380f 100644 ---- a/lib/util.cpp -+++ b/lib/util.cpp -@@ -191,6 +191,17 @@ namespace Util{ - maxSize = 0; - } - -+ void ResizeablePointer::shift(size_t byteCount){ -+ //Shifting the entire buffer is easy, we do nothing and set size to zero -+ if (byteCount >= currSize){ -+ currSize = 0; -+ return; -+ } -+ //Shifting partial needs a memmove and size change -+ memmove(ptr, ((char*)ptr)+byteCount, currSize-byteCount); -+ currSize -= byteCount; -+ } -+ - bool ResizeablePointer::assign(const void *p, uint32_t l){ - if (!allocate(l)){return false;} - memcpy(ptr, p, l); -diff --git a/lib/util.h b/lib/util.h -index 84b0d56c..d28a7f76 100644 ---- a/lib/util.h -+++ b/lib/util.h -@@ -45,6 +45,7 @@ namespace Util{ - bool append(const void *p, uint32_t l); - bool append(const std::string &str); - bool allocate(uint32_t l); -+ void shift(size_t byteCount); - uint32_t rsize(); - void truncate(const size_t newLen); - inline operator char *(){return (char *)ptr;} --- -2.25.1 - - -From 44c28097332b0a137c4674bcfd8d243b593b3cee Mon Sep 17 00:00:00 2001 -From: Thulinma -Date: Thu, 18 Aug 2022 02:44:11 +0200 -Subject: [PATCH 13/38] Decreased verbosity of harmless data offset warnings, - increased verbosity of harmful ones - ---- - src/input/input.cpp | 16 +++++++++++----- - 1 file changed, 11 insertions(+), 5 deletions(-) - -diff --git a/src/input/input.cpp b/src/input/input.cpp -index 942a55f8..0a062392 100644 ---- a/src/input/input.cpp -+++ b/src/input/input.cpp -@@ -1382,10 +1382,12 @@ namespace Mist{ - size_t currPos = tPages.getInt("avail", pageIdx); - if (currPos){ - size_t keySize = keys.getSize(keyNum); -- if (currPos-prevPos != keySize){ -- INFO_MSG("Key %" PRIu32 " was %zu bytes but should've been %zu bytes! (differs %d)", keyNum, currPos-prevPos, keySize, (int)(currPos-prevPos-keySize)); -- }else{ -+ if (currPos-prevPos == keySize){ - VERYHIGH_MSG("Key %" PRIu32 " was %zu bytes", keyNum, currPos-prevPos); -+ }else if (currPos-prevPos > keySize){ -+ FAIL_MSG("Key %" PRIu32 " was %zu bytes but should've been %zu bytes! (differs %d)", keyNum, currPos-prevPos, keySize, (int)(currPos-prevPos-keySize)); -+ }else{ -+ MEDIUM_MSG("Key %" PRIu32 " was %zu bytes but should've been %zu bytes! (differs %d)", keyNum, currPos-prevPos, keySize, (int)(currPos-prevPos-keySize)); - } - ++keyNum; - prevPos = currPos; -@@ -1412,8 +1414,12 @@ namespace Mist{ - size_t currPos = tPages.getInt("avail", pageIdx); - if (currPos){ - size_t keySize = keys.getSize(keyNum); -- if (currPos-prevPos != keySize){ -- INFO_MSG("Key %" PRIu32 " was %zu bytes but should've been %zu bytes! (differs %d)", keyNum, currPos-prevPos, keySize, (int)(currPos-prevPos-keySize)); -+ if (currPos-prevPos == keySize){ -+ VERYHIGH_MSG("Key %" PRIu32 " was %zu bytes", keyNum, currPos-prevPos); -+ }else if (currPos-prevPos > keySize){ -+ FAIL_MSG("Key %" PRIu32 " was %zu bytes but should've been %zu bytes! (differs %d)", keyNum, currPos-prevPos, keySize, (int)(currPos-prevPos-keySize)); -+ }else{ -+ MEDIUM_MSG("Key %" PRIu32 " was %zu bytes but should've been %zu bytes! (differs %d)", keyNum, currPos-prevPos, keySize, (int)(currPos-prevPos-keySize)); - } - ++keyNum; - prevPos = currPos; --- -2.25.1 - - -From 747438746c068f9217f804fab4d58bdf57156a43 Mon Sep 17 00:00:00 2001 -From: Thulinma -Date: Thu, 18 Aug 2022 15:45:46 +0200 -Subject: [PATCH 14/38] Change VoD data page logic to use wallclock seconds - rather than loop iterations for timeouts - ---- - src/input/input.cpp | 30 ++++++++++++++++++++++-------- - src/input/input.h | 2 +- - 2 files changed, 23 insertions(+), 9 deletions(-) - -diff --git a/src/input/input.cpp b/src/input/input.cpp -index 0a062392..a86fe3cd 100644 ---- a/src/input/input.cpp -+++ b/src/input/input.cpp -@@ -1022,28 +1022,42 @@ namespace Mist{ - } - - void Input::removeUnused(){ -+ uint64_t cTime = Util::bootSecs(); - std::set validTracks = M.getValidTracks(); -+ std::map > checkedPages; - for (std::set::iterator it = validTracks.begin(); it != validTracks.end(); ++it){ - Util::RelAccX &tPages = meta.pages(*it); - for (size_t i = tPages.getDeleted(); i < tPages.getEndPos(); i++){ - uint64_t pageNum = tPages.getInt("firstkey", i); -+ checkedPages[*it].insert(pageNum); - if (pageCounter[*it].count(pageNum)){ - // If the page is still being written to, reset the counter rather than potentially unloading it - if (isCurrentLivePage(*it, pageNum)){ -- pageCounter[*it][pageNum] = DEFAULT_PAGE_TIMEOUT; -+ pageCounter[*it][pageNum] = cTime; - continue; - } -- --pageCounter[*it][pageNum]; -- if (!pageCounter[*it][pageNum]){ -+ if (cTime > pageCounter[*it][pageNum] + DEFAULT_PAGE_TIMEOUT){ - pageCounter[*it].erase(pageNum); - bufferRemove(*it, pageNum); - } -+ }else{ -+ pageCounter[*it][pageNum] = cTime; - } -- else{ -- pageCounter[*it][pageNum] = DEFAULT_PAGE_TIMEOUT; -+ } -+ } -+ //Check pages we buffered but forgot about -+ for (std::map >::iterator it = pageCounter.begin(); -+ it != pageCounter.end(); it++){ -+ for (std::map::iterator it2 = it->second.begin(); it2 != it->second.end(); it2++){ -+ if (!checkedPages.count(it->first) || !checkedPages[it->first].count(it2->first)){ -+ INFO_MSG("Deleting forgotten page %zu:%" PRIu32, it->first, it2->first); -+ bufferRemove(it->first, it2->first); -+ it->second.erase(it2); -+ it2 = it->second.begin(); - } - } - } -+ - } - - std::string formatGUID(const std::string &val){ -@@ -1290,8 +1304,8 @@ namespace Mist{ - } - uint32_t pageNumber = tPages.getInt("firstkey", pageIdx); - if (isBuffered(idx, pageNumber, meta)){ -- // Mark the page for removal after 15 seconds of no one watching it -- pageCounter[idx][pageNumber] = DEFAULT_PAGE_TIMEOUT; -+ // Mark the page as still actively requested -+ pageCounter[idx][pageNumber] = Util::bootSecs(); - DONTEVEN_MSG("Track %zu, key %" PRIu32 " is already buffered in page %" PRIu32 - ". Cancelling bufferFrame", - idx, keyNum, pageNumber); -@@ -1432,7 +1446,7 @@ namespace Mist{ - idx, pageNumber, tPages.getInt("firsttime", pageIdx), thisTime, bufferTimer); - INFO_MSG(" (%" PRIu32 "/%" PRIu64 " parts, %" PRIu64 " bytes)", packCounter, - tPages.getInt("parts", pageIdx), byteCounter); -- pageCounter[idx][pageNumber] = DEFAULT_PAGE_TIMEOUT; -+ pageCounter[idx][pageNumber] = Util::bootSecs(); - return true; - } - -diff --git a/src/input/input.h b/src/input/input.h -index 28880337..8d7e8891 100644 ---- a/src/input/input.h -+++ b/src/input/input.h -@@ -87,7 +87,7 @@ namespace Mist{ - - IPC::sharedPage streamStatus; - -- std::map > pageCounter; -+ std::map > pageCounter; - - static Input *singleton; - --- -2.25.1 - - -From b210b4f5afc98b3cb4ea46e607267e12cf868831 Mon Sep 17 00:00:00 2001 -From: Thulinma -Date: Wed, 17 Aug 2022 17:06:25 +0200 -Subject: [PATCH 15/38] Fixed seek-related bugs in URIReader for HTTP sources - ---- - lib/urireader.cpp | 40 ++++++++++++++++++++++------------------ - 1 file changed, 22 insertions(+), 18 deletions(-) - -diff --git a/lib/urireader.cpp b/lib/urireader.cpp -index ad85b052..d4dd0e31 100644 ---- a/lib/urireader.cpp -+++ b/lib/urireader.cpp -@@ -255,20 +255,30 @@ namespace HTTP{ - - // seek to pos, return true if succeeded. - bool URIReader::seek(const uint64_t pos){ -- if (isSeekable()){ -- allData.truncate(0); -- bufPos = 0; -- if (stateType == HTTP::File){ -- curPos = pos; -- return true; -- }else if (stateType == HTTP::HTTP && supportRangeRequest){ -- INFO_MSG("SEEK: RangeRequest to %" PRIu64, pos); -- if (!downer.getRangeNonBlocking(myURI.getUrl(), pos, 0)){ -- FAIL_MSG("error loading request"); -- } -- } -+ //Seeking in a non-seekable source? No-op, always fails. -+ if (!isSeekable()){return false;} -+ -+ //Reset internal buffers -+ allData.truncate(0); -+ bufPos = 0; -+ -+ //Files always succeed because we use memmap -+ if (stateType == HTTP::File){ -+ curPos = pos; -+ return true; - } - -+ //HTTP-based needs to do a range request -+ if (stateType == HTTP::HTTP && supportRangeRequest){ -+ downer.getSocket().close(); -+ downer.getSocket().Received().clear(); -+ if (!downer.getRangeNonBlocking(myURI.getUrl(), pos, 0)){ -+ FAIL_MSG("Error making range request"); -+ return false; -+ } -+ curPos = pos; -+ return true; -+ } - return false; - } - -@@ -323,19 +333,13 @@ namespace HTTP{ - - }else if (stateType == HTTP::HTTP){ - downer.continueNonBlocking(cb); -- if (curPos == downer.const_data().size()){ -- Util::sleep(50); -- } -- curPos = downer.const_data().size(); - }else{// streaming mode - int s; -- static int totaal = 0; - if ((downer.getSocket() && downer.getSocket().spool())){// || downer.getSocket().Received().size() > 0){ - s = downer.getSocket().Received().bytes(wantedLen); - std::string buf = downer.getSocket().Received().remove(s); - - cb.dataCallback(buf.data(), s); -- totaal += s; - }else{ - Util::sleep(50); - } --- -2.25.1 - - -From 01a2ff54edb0cb544495bff57d8cc2a61440db2a Mon Sep 17 00:00:00 2001 -From: Thulinma -Date: Wed, 17 Aug 2022 14:57:48 +0200 -Subject: [PATCH 16/38] Converted MP4 input to use URIReader - ---- - src/input/input_mp4.cpp | 518 ++++++++++++++++++++++------------------ - src/input/input_mp4.h | 15 +- - 2 files changed, 298 insertions(+), 235 deletions(-) - -diff --git a/src/input/input_mp4.cpp b/src/input/input_mp4.cpp -index 2e1c7923..f85911fb 100644 ---- a/src/input/input_mp4.cpp -+++ b/src/input/input_mp4.cpp -@@ -159,11 +159,14 @@ namespace Mist{ - } - - inputMP4::inputMP4(Util::Config *cfg) : Input(cfg){ -- malSize = 4; // initialise data read buffer to 0; -- data = (char *)malloc(malSize); - capa["name"] = "MP4"; - capa["desc"] = "This input allows streaming of MP4 files as Video on Demand."; -- capa["source_match"] = "/*.mp4"; -+ capa["source_match"].append("/*.mp4"); -+ capa["source_match"].append("http://*.mp4"); -+ capa["source_match"].append("https://*.mp4"); -+ capa["source_match"].append("s3+http://*.mp4"); -+ capa["source_match"].append("s3+https://*.mp4"); -+ capa["source_match"].append("mp4:*"); - capa["source_file"] = "$source"; - capa["priority"] = 9; - capa["codecs"][0u][0u].append("HEVC"); -@@ -173,10 +176,9 @@ namespace Mist{ - capa["codecs"][0u][1u].append("AAC"); - capa["codecs"][0u][1u].append("AC3"); - capa["codecs"][0u][1u].append("MP3"); -+ readPos = 0; - } - -- inputMP4::~inputMP4(){free(data);} -- - bool inputMP4::checkArguments(){ - if (config->getString("input") == "-"){ - std::cerr << "Input from stdin not yet supported" << std::endl; -@@ -199,255 +201,292 @@ namespace Mist{ - - bool inputMP4::preRun(){ - // open File -- inFile = fopen(config->getString("input").c_str(), "r"); -+ std::string inUrl = config->getString("input"); -+ if (inUrl.size() > 4 && inUrl.substr(0, 4) == "mp4:"){inUrl.erase(0, 4);} -+ inFile.open(inUrl); - if (!inFile){return false;} -+ if (!inFile.isSeekable()){ -+ FAIL_MSG("MP4 input only supports seekable data sources, for now, and this source is not seekable: %s", config->getString("input").c_str()); -+ return false; -+ } - return true; - } - -+ void inputMP4::dataCallback(const char *ptr, size_t size){readBuffer.append(ptr, size);} -+ - bool inputMP4::readHeader(){ - if (!inFile){ - INFO_MSG("inFile failed!"); - return false; - } -+ bool hasMoov = false; -+ readBuffer.truncate(0); -+ readPos = 0; - - // first we get the necessary header parts - size_t tNumber = 0; -- while (!feof(inFile)){ -- std::string boxType = MP4::readBoxType(inFile); -- if (boxType == "erro"){break;} -+ activityCounter = Util::bootSecs(); -+ while (inFile && keepRunning()){ -+ //Read box header if needed -+ while (readBuffer.size() < 16 && inFile && keepRunning()){inFile.readSome(16, *this);} -+ //Failed? Abort. -+ if (readBuffer.size() < 16){ -+ FAIL_MSG("Could not read box header from input!"); -+ break; -+ } -+ //Box type is always on bytes 5-8 from the start of a box -+ std::string boxType = std::string(readBuffer+4, 4); -+ uint64_t boxSize = MP4::calcBoxSize(readBuffer); - if (boxType == "moov"){ -- MP4::MOOV moovBox; -- moovBox.read(inFile); -- // for all box in moov -+ while (readBuffer.size() < boxSize && inFile && keepRunning()){inFile.readSome(boxSize-readBuffer.size(), *this);} -+ if (readBuffer.size() < boxSize){ -+ FAIL_MSG("Could not read entire MOOV box into memory"); -+ break; -+ } -+ MP4::Box moovBox(readBuffer, false); - -- std::deque trak = moovBox.getChildren(); -+ // for all box in moov -+ std::deque trak = ((MP4::MOOV*)&moovBox)->getChildren(); - for (std::deque::iterator trakIt = trak.begin(); trakIt != trak.end(); trakIt++){ - trackHeaders.push_back(mp4TrackHeader()); - trackHeaders.rbegin()->read(*trakIt); - } -- continue; -+ hasMoov = true; -+ break; - } -- if (!MP4::skipBox(inFile)){// moving on to next box -- FAIL_MSG("Error in skipping box, exiting"); -- return false; -+ activityCounter = Util::bootSecs(); -+ //Skip to next box -+ if (readBuffer.size() > boxSize){ -+ readBuffer.shift(boxSize); -+ readPos += boxSize; -+ }else{ -+ readBuffer.truncate(0); -+ if (!inFile.seek(readPos + boxSize)){ -+ FAIL_MSG("Seek to %" PRIu64 " failed! Aborting load", readPos+boxSize); -+ } -+ readPos = inFile.getPos(); - } - } -- fseeko(inFile, 0, SEEK_SET); - - // See whether a separate header file exists. -- if (readExistingHeader()){return true;} -- HIGH_MSG("Not read existing header"); -+ if (readExistingHeader()){ -+ bps = 0; -+ std::set tracks = M.getValidTracks(); -+ for (std::set::iterator it = tracks.begin(); it != tracks.end(); it++){bps += M.getBps(*it);} -+ return true; -+ } -+ INFO_MSG("Not read existing header"); - - meta.reInit(isSingular() ? streamName : ""); -+ if (!hasMoov){ -+ FAIL_MSG("No MOOV box found; aborting header creation!"); -+ return false; -+ } - - tNumber = 0; - // Create header file from MP4 data -- while (!feof(inFile)){ -- std::string boxType = MP4::readBoxType(inFile); -- if (boxType == "erro"){break;} -- if (boxType == "moov"){ -- MP4::MOOV moovBox; -- moovBox.read(inFile); -+ MP4::Box moovBox(readBuffer, false); - -- std::deque trak = moovBox.getChildren(); -- HIGH_MSG("Obtained %zu trak Boxes", trak.size()); -+ std::deque trak = ((MP4::MOOV*)&moovBox)->getChildren(); -+ HIGH_MSG("Obtained %zu trak Boxes", trak.size()); - -- for (std::deque::iterator trakIt = trak.begin(); trakIt != trak.end(); trakIt++){ -- MP4::MDIA mdiaBox = trakIt->getChild(); -+ for (std::deque::iterator trakIt = trak.begin(); trakIt != trak.end(); trakIt++){ -+ MP4::MDIA mdiaBox = trakIt->getChild(); - -- std::string hdlrType = mdiaBox.getChild().getHandlerType(); -- if (hdlrType != "vide" && hdlrType != "soun" && hdlrType != "sbtl"){ -- INFO_MSG("Unsupported handler: %s", hdlrType.c_str()); -- continue; -- } -+ std::string hdlrType = mdiaBox.getChild().getHandlerType(); -+ if (hdlrType != "vide" && hdlrType != "soun" && hdlrType != "sbtl"){ -+ INFO_MSG("Unsupported handler: %s", hdlrType.c_str()); -+ continue; -+ } - -- tNumber = meta.addTrack(); -+ tNumber = meta.addTrack(); - -- MP4::TKHD tkhdBox = trakIt->getChild(); -- if (tkhdBox.getWidth() > 0){ -- meta.setWidth(tNumber, tkhdBox.getWidth()); -- meta.setHeight(tNumber, tkhdBox.getHeight()); -- } -- meta.setID(tNumber, tkhdBox.getTrackID()); -- -- MP4::MDHD mdhdBox = mdiaBox.getChild(); -- uint64_t timescale = mdhdBox.getTimeScale(); -- meta.setLang(tNumber, mdhdBox.getLanguage()); -- -- MP4::STBL stblBox = mdiaBox.getChild().getChild(); -- -- MP4::STSD stsdBox = stblBox.getChild(); -- MP4::Box sEntryBox = stsdBox.getEntry(0); -- std::string sType = sEntryBox.getType(); -- HIGH_MSG("Found track %zu of type %s", tNumber, sType.c_str()); -- -- if (sType == "avc1" || sType == "h264" || sType == "mp4v"){ -- MP4::VisualSampleEntry &vEntryBox = (MP4::VisualSampleEntry &)sEntryBox; -- meta.setType(tNumber, "video"); -- meta.setCodec(tNumber, "H264"); -- if (!meta.getWidth(tNumber)){ -- meta.setWidth(tNumber, vEntryBox.getWidth()); -- meta.setHeight(tNumber, vEntryBox.getHeight()); -- } -- MP4::Box initBox = vEntryBox.getCLAP(); -- if (initBox.isType("avcC")){ -- meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); -- } -- initBox = vEntryBox.getPASP(); -- if (initBox.isType("avcC")){ -- meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); -- } -- /// this is a hacky way around invalid FLV data (since it gets ignored nearly -- /// everywhere, but we do need correct data... -- if (!meta.getWidth(tNumber)){ -- h264::sequenceParameterSet sps; -- sps.fromDTSCInit(meta.getInit(tNumber)); -- h264::SPSMeta spsChar = sps.getCharacteristics(); -- meta.setWidth(tNumber, spsChar.width); -- meta.setHeight(tNumber, spsChar.height); -- } -- } -- if (sType == "hev1" || sType == "hvc1"){ -- MP4::VisualSampleEntry &vEntryBox = (MP4::VisualSampleEntry &)sEntryBox; -- meta.setType(tNumber, "video"); -- meta.setCodec(tNumber, "HEVC"); -- if (!meta.getWidth(tNumber)){ -- meta.setWidth(tNumber, vEntryBox.getWidth()); -- meta.setHeight(tNumber, vEntryBox.getHeight()); -- } -- MP4::Box initBox = vEntryBox.getCLAP(); -- if (initBox.isType("hvcC")){ -- meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); -- } -- initBox = vEntryBox.getPASP(); -- if (initBox.isType("hvcC")){ -- meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); -- } -- } -- if (sType == "mp4a" || sType == "aac " || sType == "ac-3"){ -- MP4::AudioSampleEntry &aEntryBox = (MP4::AudioSampleEntry &)sEntryBox; -- meta.setType(tNumber, "audio"); -- meta.setChannels(tNumber, aEntryBox.getChannelCount()); -- meta.setRate(tNumber, aEntryBox.getSampleRate()); -- -- if (sType == "ac-3"){ -- meta.setCodec(tNumber, "AC3"); -- }else{ -- MP4::ESDS esdsBox = (MP4::ESDS &)(aEntryBox.getCodecBox()); -- meta.setCodec(tNumber, esdsBox.getCodec()); -- meta.setInit(tNumber, esdsBox.getInitData()); -- } -- meta.setSize(tNumber, 16); ///\todo this might be nice to calculate from mp4 file; -- } -- if (sType == "tx3g"){// plain text subtitles -- meta.setType(tNumber, "meta"); -- meta.setCodec(tNumber, "subtitle"); -+ MP4::TKHD tkhdBox = trakIt->getChild(); -+ if (tkhdBox.getWidth() > 0){ -+ meta.setWidth(tNumber, tkhdBox.getWidth()); -+ meta.setHeight(tNumber, tkhdBox.getHeight()); -+ } -+ meta.setID(tNumber, tkhdBox.getTrackID()); -+ -+ MP4::MDHD mdhdBox = mdiaBox.getChild(); -+ uint64_t timescale = mdhdBox.getTimeScale(); -+ meta.setLang(tNumber, mdhdBox.getLanguage()); -+ -+ MP4::STBL stblBox = mdiaBox.getChild().getChild(); -+ -+ MP4::STSD stsdBox = stblBox.getChild(); -+ MP4::Box sEntryBox = stsdBox.getEntry(0); -+ std::string sType = sEntryBox.getType(); -+ HIGH_MSG("Found track %zu of type %s", tNumber, sType.c_str()); -+ -+ if (sType == "avc1" || sType == "h264" || sType == "mp4v"){ -+ MP4::VisualSampleEntry &vEntryBox = (MP4::VisualSampleEntry &)sEntryBox; -+ meta.setType(tNumber, "video"); -+ meta.setCodec(tNumber, "H264"); -+ if (!meta.getWidth(tNumber)){ -+ meta.setWidth(tNumber, vEntryBox.getWidth()); -+ meta.setHeight(tNumber, vEntryBox.getHeight()); -+ } -+ MP4::Box initBox = vEntryBox.getCLAP(); -+ if (initBox.isType("avcC")){ -+ meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); -+ } -+ initBox = vEntryBox.getPASP(); -+ if (initBox.isType("avcC")){ -+ meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); -+ } -+ /// this is a hacky way around invalid FLV data (since it gets ignored nearly -+ /// everywhere, but we do need correct data... -+ if (!meta.getWidth(tNumber)){ -+ h264::sequenceParameterSet sps; -+ sps.fromDTSCInit(meta.getInit(tNumber)); -+ h264::SPSMeta spsChar = sps.getCharacteristics(); -+ meta.setWidth(tNumber, spsChar.width); -+ meta.setHeight(tNumber, spsChar.height); -+ } -+ } -+ if (sType == "hev1" || sType == "hvc1"){ -+ MP4::VisualSampleEntry &vEntryBox = (MP4::VisualSampleEntry &)sEntryBox; -+ meta.setType(tNumber, "video"); -+ meta.setCodec(tNumber, "HEVC"); -+ if (!meta.getWidth(tNumber)){ -+ meta.setWidth(tNumber, vEntryBox.getWidth()); -+ meta.setHeight(tNumber, vEntryBox.getHeight()); -+ } -+ MP4::Box initBox = vEntryBox.getCLAP(); -+ if (initBox.isType("hvcC")){ -+ meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); -+ } -+ initBox = vEntryBox.getPASP(); -+ if (initBox.isType("hvcC")){ -+ meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); -+ } -+ } -+ if (sType == "mp4a" || sType == "aac " || sType == "ac-3"){ -+ MP4::AudioSampleEntry &aEntryBox = (MP4::AudioSampleEntry &)sEntryBox; -+ meta.setType(tNumber, "audio"); -+ meta.setChannels(tNumber, aEntryBox.getChannelCount()); -+ meta.setRate(tNumber, aEntryBox.getSampleRate()); -+ -+ if (sType == "ac-3"){ -+ meta.setCodec(tNumber, "AC3"); -+ }else{ -+ MP4::ESDS esdsBox = (MP4::ESDS &)(aEntryBox.getCodecBox()); -+ meta.setCodec(tNumber, esdsBox.getCodec()); -+ meta.setInit(tNumber, esdsBox.getInitData()); -+ } -+ meta.setSize(tNumber, 16); ///\todo this might be nice to calculate from mp4 file; -+ } -+ if (sType == "tx3g"){// plain text subtitles -+ meta.setType(tNumber, "meta"); -+ meta.setCodec(tNumber, "subtitle"); -+ } -+ -+ MP4::STSS stssBox = stblBox.getChild(); -+ MP4::STTS sttsBox = stblBox.getChild(); -+ MP4::STSZ stszBox = stblBox.getChild(); -+ MP4::STCO stcoBox = stblBox.getChild(); -+ MP4::CO64 co64Box = stblBox.getChild(); -+ MP4::STSC stscBox = stblBox.getChild(); -+ MP4::CTTS cttsBox = stblBox.getChild(); // optional ctts box -+ -+ bool stco64 = co64Box.isType("co64"); -+ bool hasCTTS = cttsBox.isType("ctts"); -+ -+ uint64_t totaldur = 0; ///\todo note: set this to begin time -+ mp4PartBpos BsetPart; -+ -+ uint64_t entryNo = 0; -+ uint64_t sampleNo = 0; -+ -+ uint64_t stssIndex = 0; -+ uint64_t stcoIndex = 0; -+ uint64_t stscIndex = 0; -+ uint64_t cttsIndex = 0; // current ctts Index we are reading -+ uint64_t cttsEntryRead = 0; // current part of ctts we are reading -+ -+ uint64_t stssCount = stssBox.getEntryCount(); -+ uint64_t stscCount = stscBox.getEntryCount(); -+ uint64_t stszCount = stszBox.getSampleCount(); -+ uint64_t stcoCount = (stco64 ? co64Box.getEntryCount() : stcoBox.getEntryCount()); -+ -+ MP4::STTSEntry sttsEntry = sttsBox.getSTTSEntry(0); -+ -+ uint32_t fromSTCOinSTSC = 0; -+ uint64_t tmpOffset = (stco64 ? co64Box.getChunkOffset(0) : stcoBox.getChunkOffset(0)); -+ -+ uint64_t nextFirstChunk = (stscCount > 1 ? stscBox.getSTSCEntry(1).firstChunk - 1 : stcoCount); -+ -+ for (uint64_t stszIndex = 0; stszIndex < stszCount; ++stszIndex){ -+ if (stcoIndex >= nextFirstChunk){ -+ ++stscIndex; -+ nextFirstChunk = -+ (stscIndex + 1 < stscCount ? stscBox.getSTSCEntry(stscIndex + 1).firstChunk - 1 : stcoCount); -+ } -+ BsetPart.keyframe = (meta.getType(tNumber) == "video" && stssIndex < stssCount && -+ stszIndex + 1 == stssBox.getSampleNumber(stssIndex)); -+ if (BsetPart.keyframe){++stssIndex;} -+ // in bpos set -+ BsetPart.stcoNr = stcoIndex; -+ // bpos = chunkoffset[samplenr] in stco -+ BsetPart.bpos = tmpOffset; -+ ++fromSTCOinSTSC; -+ if (fromSTCOinSTSC < stscBox.getSTSCEntry(stscIndex).samplesPerChunk){// as long as we are still in this chunk -+ tmpOffset += stszBox.getEntrySize(stszIndex); -+ }else{ -+ ++stcoIndex; -+ fromSTCOinSTSC = 0; -+ tmpOffset = (stco64 ? co64Box.getChunkOffset(stcoIndex) : stcoBox.getChunkOffset(stcoIndex)); -+ } -+ BsetPart.time = (totaldur * 1000) / timescale; -+ totaldur += sttsEntry.sampleDelta; -+ sampleNo++; -+ if (sampleNo >= sttsEntry.sampleCount){ -+ ++entryNo; -+ sampleNo = 0; -+ if (entryNo < sttsBox.getEntryCount()){sttsEntry = sttsBox.getSTTSEntry(entryNo);} -+ } -+ -+ if (hasCTTS){ -+ MP4::CTTSEntry cttsEntry = cttsBox.getCTTSEntry(cttsIndex); -+ cttsEntryRead++; -+ if (cttsEntryRead >= cttsEntry.sampleCount){ -+ ++cttsIndex; -+ cttsEntryRead = 0; - } -+ BsetPart.timeOffset = (cttsEntry.sampleOffset * 1000) / timescale; -+ }else{ -+ BsetPart.timeOffset = 0; -+ } - -- MP4::STSS stssBox = stblBox.getChild(); -- MP4::STTS sttsBox = stblBox.getChild(); -- MP4::STSZ stszBox = stblBox.getChild(); -- MP4::STCO stcoBox = stblBox.getChild(); -- MP4::CO64 co64Box = stblBox.getChild(); -- MP4::STSC stscBox = stblBox.getChild(); -- MP4::CTTS cttsBox = stblBox.getChild(); // optional ctts box -- -- bool stco64 = co64Box.isType("co64"); -- bool hasCTTS = cttsBox.isType("ctts"); -- -- uint64_t totaldur = 0; ///\todo note: set this to begin time -- mp4PartBpos BsetPart; -- -- uint64_t entryNo = 0; -- uint64_t sampleNo = 0; -- -- uint64_t stssIndex = 0; -- uint64_t stcoIndex = 0; -- uint64_t stscIndex = 0; -- uint64_t cttsIndex = 0; // current ctts Index we are reading -- uint64_t cttsEntryRead = 0; // current part of ctts we are reading -- -- uint64_t stssCount = stssBox.getEntryCount(); -- uint64_t stscCount = stscBox.getEntryCount(); -- uint64_t stszCount = stszBox.getSampleCount(); -- uint64_t stcoCount = (stco64 ? co64Box.getEntryCount() : stcoBox.getEntryCount()); -- -- MP4::STTSEntry sttsEntry = sttsBox.getSTTSEntry(0); -- -- uint32_t fromSTCOinSTSC = 0; -- uint64_t tmpOffset = (stco64 ? co64Box.getChunkOffset(0) : stcoBox.getChunkOffset(0)); -- -- uint64_t nextFirstChunk = (stscCount > 1 ? stscBox.getSTSCEntry(1).firstChunk - 1 : stcoCount); -- -- for (uint64_t stszIndex = 0; stszIndex < stszCount; ++stszIndex){ -- if (stcoIndex >= nextFirstChunk){ -- ++stscIndex; -- nextFirstChunk = -- (stscIndex + 1 < stscCount ? stscBox.getSTSCEntry(stscIndex + 1).firstChunk - 1 : stcoCount); -- } -- BsetPart.keyframe = (meta.getType(tNumber) == "video" && stssIndex < stssCount && -- stszIndex + 1 == stssBox.getSampleNumber(stssIndex)); -- if (BsetPart.keyframe){++stssIndex;} -- // in bpos set -- BsetPart.stcoNr = stcoIndex; -- // bpos = chunkoffset[samplenr] in stco -- BsetPart.bpos = tmpOffset; -- ++fromSTCOinSTSC; -- if (fromSTCOinSTSC < stscBox.getSTSCEntry(stscIndex).samplesPerChunk){// as long as we are still in this chunk -- tmpOffset += stszBox.getEntrySize(stszIndex); -- }else{ -- ++stcoIndex; -- fromSTCOinSTSC = 0; -- tmpOffset = (stco64 ? co64Box.getChunkOffset(stcoIndex) : stcoBox.getChunkOffset(stcoIndex)); -- } -- BsetPart.time = (totaldur * 1000) / timescale; -- totaldur += sttsEntry.sampleDelta; -- sampleNo++; -- if (sampleNo >= sttsEntry.sampleCount){ -- ++entryNo; -- sampleNo = 0; -- if (entryNo < sttsBox.getEntryCount()){sttsEntry = sttsBox.getSTTSEntry(entryNo);} -- } -- -- if (hasCTTS){ -- MP4::CTTSEntry cttsEntry = cttsBox.getCTTSEntry(cttsIndex); -- cttsEntryRead++; -- if (cttsEntryRead >= cttsEntry.sampleCount){ -- ++cttsIndex; -- cttsEntryRead = 0; -- } -- BsetPart.timeOffset = (cttsEntry.sampleOffset * 1000) / timescale; -- }else{ -- BsetPart.timeOffset = 0; -- } -- -- if (sType == "tx3g"){ -- if (stszBox.getEntrySize(stszIndex) <= 2 && false){ -- FAIL_MSG("size <=2"); -- }else{ -- long long packSendSize = 0; -- packSendSize = 24 + (BsetPart.timeOffset ? 17 : 0) + (BsetPart.bpos ? 15 : 0) + 19 + -- stszBox.getEntrySize(stszIndex) + 11 - 2 + 19; -- meta.update(BsetPart.time, BsetPart.timeOffset, tNumber, -- stszBox.getEntrySize(stszIndex) - 2, BsetPart.bpos, true, packSendSize); -- } -- }else{ -- meta.update(BsetPart.time, BsetPart.timeOffset, tNumber, -- stszBox.getEntrySize(stszIndex), BsetPart.bpos, BsetPart.keyframe); -- } -+ if (sType == "tx3g"){ -+ if (stszBox.getEntrySize(stszIndex) <= 2 && false){ -+ FAIL_MSG("size <=2"); -+ }else{ -+ long long packSendSize = 0; -+ packSendSize = 24 + (BsetPart.timeOffset ? 17 : 0) + (BsetPart.bpos ? 15 : 0) + 19 + -+ stszBox.getEntrySize(stszIndex) + 11 - 2 + 19; -+ meta.update(BsetPart.time, BsetPart.timeOffset, tNumber, -+ stszBox.getEntrySize(stszIndex) - 2, BsetPart.bpos, true, packSendSize); - } -+ }else{ -+ meta.update(BsetPart.time, BsetPart.timeOffset, tNumber, -+ stszBox.getEntrySize(stszIndex), BsetPart.bpos, BsetPart.keyframe); - } -- continue; -- } -- if (!MP4::skipBox(inFile)){// moving on to next box -- FAIL_MSG("Error in Skipping box, exiting"); -- return false; - } - } -- clearerr(inFile); - - // outputting dtsh file -- M.toFile(config->getString("input") + ".dtsh"); -+ std::string inUrl = config->getString("input"); -+ if (inUrl.size() > 4 && inUrl.substr(0, 4) == "mp4:"){inUrl.erase(0, 4);} -+ if (inUrl != "-" && HTTP::URL(inUrl).isLocalPath()){ -+ M.toFile(inUrl + ".dtsh"); -+ }else{ -+ INFO_MSG("Skipping header write, as the source is not a local file"); -+ } -+ bps = 0; -+ std::set tracks = M.getValidTracks(); -+ for (std::set::iterator it = tracks.begin(); it != tracks.end(); it++){bps += M.getBps(*it);} - return true; - } - -@@ -474,23 +513,48 @@ namespace Mist{ - ++nextKeyNum; - } - } -- if (fseeko(inFile, curPart.bpos, SEEK_SET)){ -- FAIL_MSG("seek unsuccessful @bpos %" PRIu64 ": %s", curPart.bpos, strerror(errno)); -- thisPacket.null(); -- return; -+ if (curPart.bpos < readPos || curPart.bpos > readPos + readBuffer.size() + 512*1024 + bps){ -+ INFO_MSG("Buffer contains %" PRIu64 "-%" PRIu64 ", but we need %" PRIu64 "; seeking!", readPos, readPos + readBuffer.size(), curPart.bpos); -+ readBuffer.truncate(0); -+ if (!inFile.seek(curPart.bpos)){ -+ FAIL_MSG("seek unsuccessful @bpos %" PRIu64 ": %s", curPart.bpos, strerror(errno)); -+ thisPacket.null(); -+ return; -+ } -+ readPos = inFile.getPos(); -+ }else{ -+ //If we have more than 5MiB buffered and are more than 5MiB into the buffer, shift the first 4MiB off the buffer. -+ //This prevents infinite growth of the read buffer for large files -+ if (readBuffer.size() >= 5*1024*1024 && curPart.bpos > readPos + 5*1024*1024 + bps){ -+ readBuffer.shift(4*1024*1024); -+ readPos += 4*1024*1024; -+ } - } -- if (curPart.size > malSize){ -- data = (char *)realloc(data, curPart.size); -- malSize = curPart.size; -+ -+ while (readPos+readBuffer.size() < curPart.bpos+curPart.size && inFile && keepRunning()){ -+ inFile.readSome((curPart.bpos+curPart.size) - (readPos+readBuffer.size()), *this); - } -- if (fread(data, curPart.size, 1, inFile) != 1){ -- FAIL_MSG("read unsuccessful at %ld", ftell(inFile)); -- thisPacket.null(); -- return; -+ if (readPos+readBuffer.size() < curPart.bpos+curPart.size){ -+ FAIL_MSG("Read unsuccessful at %" PRIu64 ", seeking to retry...", readPos+readBuffer.size()); -+ readBuffer.truncate(0); -+ if (!inFile.seek(curPart.bpos)){ -+ FAIL_MSG("seek unsuccessful @bpos %" PRIu64 ": %s", curPart.bpos, strerror(errno)); -+ thisPacket.null(); -+ return; -+ } -+ readPos = inFile.getPos(); -+ while (readPos+readBuffer.size() < curPart.bpos+curPart.size && inFile && keepRunning()){ -+ inFile.readSome((curPart.bpos+curPart.size) - (readPos+readBuffer.size()), *this); -+ } -+ if (readPos+readBuffer.size() < curPart.bpos+curPart.size){ -+ FAIL_MSG("Read retry unsuccessful at %" PRIu64 ", aborting", readPos+readBuffer.size()); -+ thisPacket.null(); -+ return; -+ } - } - - if (M.getCodec(curPart.trackID) == "subtitle"){ -- unsigned int txtLen = Bit::btohs(data); -+ unsigned int txtLen = Bit::btohs(readBuffer + (curPart.bpos-readPos)); - if (!txtLen && false){ - curPart.index++; - return getNext(idx); -@@ -499,14 +563,14 @@ namespace Mist{ - thisPack.null(); - thisPack["trackid"] = curPart.trackID; - thisPack["bpos"] = curPart.bpos; //(long long)fileSource.tellg(); -- thisPack["data"] = std::string(data + 2, txtLen); -+ thisPack["data"] = std::string(readBuffer + (curPart.bpos-readPos) + 2, txtLen); - thisPack["time"] = curPart.time; - if (curPart.duration){thisPack["duration"] = curPart.duration;} - thisPack["keyframe"] = true; - std::string tmpStr = thisPack.toNetPacked(); - thisPacket.reInit(tmpStr.data(), tmpStr.size()); - }else{ -- thisPacket.genericFill(curPart.time, curPart.offset, curPart.trackID, data, curPart.size, 0, isKeyframe); -+ thisPacket.genericFill(curPart.time, curPart.offset, curPart.trackID, readBuffer + (curPart.bpos-readPos), curPart.size, 0, isKeyframe); - } - thisTime = curPart.time; - thisIdx = curPart.trackID; -diff --git a/src/input/input_mp4.h b/src/input/input_mp4.h -index b5987d3b..4ab9f36e 100644 ---- a/src/input/input_mp4.h -+++ b/src/input/input_mp4.h -@@ -1,5 +1,6 @@ - #include "input.h" - #include -+#include - #include - #include - namespace Mist{ -@@ -70,10 +71,10 @@ namespace Mist{ - bool stco64; - }; - -- class inputMP4 : public Input{ -+ class inputMP4 : public Input, public Util::DataCallback { - public: - inputMP4(Util::Config *cfg); -- ~inputMP4(); -+ void dataCallback(const char *ptr, size_t size); - - protected: - // Private Functions -@@ -85,7 +86,10 @@ namespace Mist{ - void seek(uint64_t seekTime, size_t idx = INVALID_TRACK_ID); - void handleSeek(uint64_t seekTime, size_t idx); - -- FILE *inFile; -+ HTTP::URIReader inFile; -+ Util::ResizeablePointer readBuffer; -+ uint64_t readPos; -+ uint64_t bps; - - mp4TrackHeader &headerData(size_t trackID); - -@@ -94,11 +98,6 @@ namespace Mist{ - - // remember last seeked keyframe; - std::map nextKeyframe; -- -- // these next two variables keep a buffer for reading from filepointer inFile; -- uint64_t malSize; -- char *data; ///\todo rename this variable to a more sensible name, it is a temporary piece of -- /// memory to read from files - }; - }// namespace Mist - --- -2.25.1 - - -From 2740d65a0f5715c42eb6de24cbf610ba6271b85a Mon Sep 17 00:00:00 2001 -From: Thulinma -Date: Wed, 17 Aug 2022 10:54:31 +0200 -Subject: [PATCH 17/38] Improved RTP timestamp logging, fixed bug related to - firstTime value - ---- - lib/rtp.cpp | 7 ++++--- - 1 file changed, 4 insertions(+), 3 deletions(-) - -diff --git a/lib/rtp.cpp b/lib/rtp.cpp -index 3ed9b7c6..5a5ed829 100644 ---- a/lib/rtp.cpp -+++ b/lib/rtp.cpp -@@ -877,7 +877,7 @@ namespace RTP{ - if (!firstTime){ - milliSync = Util::bootMS(); - firstTime = pTime + 1; -- INFO_MSG("RTP timestamp rollover expected in " PRETTY_PRINT_TIME, -+ INFO_MSG("RTP timestamp rollover for %" PRIu64 " (%s) expected in " PRETTY_PRINT_TIME, trackId, codec.c_str(), - PRETTY_ARG_TIME((0xFFFFFFFFul - firstTime) / multiplier / 1000)); - }else{ - if (recentWrap){ -@@ -886,14 +886,15 @@ namespace RTP{ - }else{ - if (prevTime > pTime && pTime < 0x40000000lu && prevTime > 0x80000000lu){ - ++wrapArounds; -+ INFO_MSG("RTP timestamp rollover %" PRIu32 " for %" PRIu64 " (%s) happened; next should be in " PRETTY_PRINT_TIME, wrapArounds, trackId, codec.c_str(), PRETTY_ARG_TIME((0xFFFFFFFFul) / multiplier / 1000)); - recentWrap = true; - } - } - } - // When there are B-frames, the firstTime can be higher than the current time - // causing msTime to become negative and thus overflow -- if (firstTime > pTime + 1){ -- WARN_MSG("firstTime was higher than current packet time. Readjusting firsTime..."); -+ if (!wrapArounds && firstTime > pTime + 1){ -+ WARN_MSG("firstTime was higher than current packet time. Readjusting firstTime..."); - firstTime = pTime + 1; - } - prevTime = pkt.getTimeStamp(); --- -2.25.1 - - -From 2870ae1cedd202755bbe9d0e6808086d40656157 Mon Sep 17 00:00:00 2001 -From: Thulinma -Date: Wed, 24 Aug 2022 15:02:19 +0200 -Subject: [PATCH 18/38] =?UTF-8?q?SRT=20RAW=20fixup=20=F0=9F=A4=A6?= -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - ---- - src/input/input_tssrt.cpp | 1 - - 1 file changed, 1 deletion(-) - -diff --git a/src/input/input_tssrt.cpp b/src/input/input_tssrt.cpp -index da69be29..4219ebbc 100644 ---- a/src/input/input_tssrt.cpp -+++ b/src/input/input_tssrt.cpp -@@ -211,7 +211,6 @@ namespace Mist{ - } - uint64_t packetTime = Util::bootMS(); - thisPacket.genericFill(packetTime, 0, 1, rawBuffer, rawBuffer.size(), 0, 0); -- bufferLivePacket(thisPacket); - lastRawPacket = packetTime; - rawBuffer.truncate(0); - return; --- -2.25.1 - - -From a9ddc377895f5feb898c162de0f6b0363cf94276 Mon Sep 17 00:00:00 2001 -From: Thulinma -Date: Thu, 25 Aug 2022 10:26:38 +0200 -Subject: [PATCH 19/38] =?UTF-8?q?Why,=20Haivision,=20why=3F=20=F0=9F=98=AD?= -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - ---- - lib/socket_srt.cpp | 4 ++++ - 1 file changed, 4 insertions(+) - -diff --git a/lib/socket_srt.cpp b/lib/socket_srt.cpp -index 3e03bce4..6d1d5695 100644 ---- a/lib/socket_srt.cpp -+++ b/lib/socket_srt.cpp -@@ -392,6 +392,10 @@ namespace Socket{ - } - } - params["payloadsize"] = asString(chunkTransmitSize); -+ //This line forces the transmission type to live if unset. -+ //Live is actually the default, but not explicitly setting the option means -+ //that all other defaults do not get applied either, which is bad. -+ if (!params.count("transtype")){params["transtype"] = "live";} - } - - int SRTConnection::preConfigureSocket(){ --- -2.25.1 - - -From 8aceff951d3e320786bf33c9ff558a506df372c6 Mon Sep 17 00:00:00 2001 -From: Marco van Dijk -Date: Mon, 29 Aug 2022 14:36:00 +0200 -Subject: [PATCH 20/38] Fix connections getting closed on interrupted system - call - ---- - lib/socket.cpp | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/lib/socket.cpp b/lib/socket.cpp -index 865c3be7..6d63af00 100644 ---- a/lib/socket.cpp -+++ b/lib/socket.cpp -@@ -1032,6 +1032,7 @@ unsigned int Socket::Connection::iwrite(const void *buffer, int len){ - case MBEDTLS_ERR_SSL_WANT_WRITE: return 0; break; - case MBEDTLS_ERR_SSL_WANT_READ: return 0; break; - case EWOULDBLOCK: return 0; break; -+ case EINTR: return 0; break; - default: - Error = true; - lastErr = strerror(errno); -@@ -1071,6 +1072,7 @@ unsigned int Socket::Connection::iwrite(const void *buffer, int len){ - if (r < 0){ - switch (errno){ - case EWOULDBLOCK: return 0; break; -+ case EINTR: return 0; break; - default: - Error = true; - lastErr = strerror(errno); --- -2.25.1 - - -From 54a46146c2527004a96b78fe8b7341eab4de6f62 Mon Sep 17 00:00:00 2001 -From: Thulinma -Date: Thu, 11 Aug 2022 13:17:13 +0200 -Subject: [PATCH 21/38] Fix TS inputs taking more and more memory over time in - some cases - ---- - lib/ts_stream.cpp | 12 +++++++++--- - 1 file changed, 9 insertions(+), 3 deletions(-) - -diff --git a/lib/ts_stream.cpp b/lib/ts_stream.cpp -index 4ab20c59..8da72493 100644 ---- a/lib/ts_stream.cpp -+++ b/lib/ts_stream.cpp -@@ -190,7 +190,8 @@ namespace TS{ - uint32_t tid = newPack.getPID(); - bool unitStart = newPack.getUnitStart(); - static uint32_t wantPrev = 0; -- bool wantTrack = ((wantPrev == tid) || (tid == 0 || newPack.isPMT(pmtTracks) || pidToCodec.count(tid))); -+ bool isData = pidToCodec.count(tid); -+ bool wantTrack = ((wantPrev == tid) || (tid == 0 || newPack.isPMT(pmtTracks) || isData)); - if (!wantTrack){return;} - if (psCacheTid != tid || !psCache){ - psCache = &(pesStreams[tid]); -@@ -199,7 +200,7 @@ namespace TS{ - if (unitStart || !psCache->empty()){ - wantPrev = tid; - psCache->push_back(newPack); -- if (unitStart){ -+ if (unitStart && isData){ - pesPositions[tid].push_back(bytePos); - ++(seenUnitStart[tid]); - } -@@ -210,7 +211,7 @@ namespace TS{ - if (tid == 0){return false;} - { - tthread::lock_guard guard(tMutex); -- return !pmtTracks.count(tid); -+ return pidToCodec.count(tid); - } - } - -@@ -280,6 +281,11 @@ namespace TS{ - } - - if (!pidToCodec.count(tid)){ -+ pesStreams.erase(tid); -+ pesPositions.erase(tid); -+ seenUnitStart.erase(tid); -+ psCacheTid = 0; -+ psCache = 0; - return; // skip unknown codecs - } - --- -2.25.1 - - -From c10d5a7ec1fdc71cc4af2329a9030b422735fd34 Mon Sep 17 00:00:00 2001 -From: Marco van Dijk -Date: Wed, 16 Mar 2022 13:45:37 +0100 -Subject: [PATCH 22/38] onFail on a WS connection, send the error back using - the websocket request handler - ---- - src/output/output_http_internal.cpp | 20 ++++++++++++++++++-- - src/output/output_http_internal.h | 1 + - 2 files changed, 19 insertions(+), 2 deletions(-) - -diff --git a/src/output/output_http_internal.cpp b/src/output/output_http_internal.cpp -index c8ca2e6c..97ca455f 100644 ---- a/src/output/output_http_internal.cpp -+++ b/src/output/output_http_internal.cpp -@@ -36,6 +36,7 @@ namespace Mist{ - - OutHTTP::OutHTTP(Socket::Connection &conn) : HTTPOutput(conn){ - stayConnected = false; -+ thisError = ""; - // If this connection is a socket and not already connected to stdio, connect it to stdio. - if (myConn.getPureSocket() != -1 && myConn.getSocket() != STDIN_FILENO && myConn.getSocket() != STDOUT_FILENO){ - std::string host = getConnectedHost(); -@@ -63,6 +64,11 @@ namespace Mist{ - bool OutHTTP::listenMode(){return !(config->getString("ip").size());} - - void OutHTTP::onFail(const std::string &msg, bool critical){ -+ // If we are connected through WS, the websockethandler should return the error message -+ if (stayConnected){ -+ thisError = msg; -+ return; -+ } - if (responded){ - HTTPOutput::onFail(msg, critical); - return; -@@ -78,7 +84,6 @@ namespace Mist{ - return; - } - if (H.url.size() >= 3 && H.url.substr(H.url.size() - 3) == ".js"){ -- if (websocketHandler()){return;} - JSON::Value json_resp; - json_resp["error"] = "Could not retrieve stream. Sorry."; - json_resp["error_guru"] = msg; -@@ -1149,12 +1154,23 @@ namespace Mist{ - if (meta){meta.reloadReplacedPagesIfNeeded();} - if (newState != prevState || (newState == STRMSTAT_READY && M.getValidTracks() != prevTracks)){ - if (newState == STRMSTAT_READY){ -+ thisError = ""; - reconnect(); - prevTracks = M.getValidTracks(); - }else{ - disconnect(); - } -- JSON::Value resp = getStatusJSON(reqHost, useragent); -+ JSON::Value resp; -+ // Check if we have an error message set -+ if (thisError == ""){ -+ resp = getStatusJSON(reqHost, useragent); -+ }else{ -+ resp["error"] = "Could not retrieve stream. Sorry."; -+ resp["error_guru"] = thisError; -+ if (config->getString("nostreamtext") != ""){ -+ resp["on_error"] = config->getString("nostreamtext"); -+ } -+ } - if (currStreamName != streamName){ - currStreamName = streamName; - snprintf(pageName, NAME_BUFFER_SIZE, SHM_STREAM_STATE, streamName.c_str()); -diff --git a/src/output/output_http_internal.h b/src/output/output_http_internal.h -index 0f939e78..774eb186 100644 ---- a/src/output/output_http_internal.h -+++ b/src/output/output_http_internal.h -@@ -21,6 +21,7 @@ namespace Mist{ - private: - std::string origStreamName; - std::string mistPath; -+ std::string thisError; - }; - }// namespace Mist - --- -2.25.1 - - -From f418fed81c0c46fd338e238b558085f408456ed9 Mon Sep 17 00:00:00 2001 -From: Thulinma -Date: Thu, 24 Mar 2022 15:20:52 +0100 -Subject: [PATCH 23/38] Fix HLS output not setting "responded" value - appropriately - ---- - src/output/output_hls.cpp | 8 ++++++++ - 1 file changed, 8 insertions(+) - -diff --git a/src/output/output_hls.cpp b/src/output/output_hls.cpp -index 19d083e6..ad810aa7 100644 ---- a/src/output/output_hls.cpp -+++ b/src/output/output_hls.cpp -@@ -232,6 +232,7 @@ namespace Mist{ - H.setCORSHeaders(); - if (H.method == "OPTIONS" || H.method == "HEAD"){ - H.SendResponse("200", "OK", myConn); -+ responded = true; - return; - } - H.SetBody(""); - H.SendResponse("200", "OK", myConn); -+ responded = true; - return; - }// crossdomain.xml - -@@ -262,6 +264,7 @@ namespace Mist{ - } - H.SetBody(""); - H.SendResponse("200", "OK", myConn); -+ responded = true; - return; - } - -@@ -311,6 +314,7 @@ namespace Mist{ - targetTime = HLS::getPartTargetTime(M, idx, mTrack, startTime, msn, part); - if (!targetTime){ - H.SendResponse("404", "Partial fragment does not exist", myConn); -+ responded = true; - return; - } - startTime += part * HLS::partDurationMaxMs; -@@ -372,6 +376,7 @@ namespace Mist{ - "served.\n"); - myConn.SendNow(H.BuildResponse("404", "Fragment out of range")); - WARN_MSG("Fragment @ %" PRIu64 " too old", startTime); -+ responded = true; - return; - } - -@@ -389,10 +394,12 @@ namespace Mist{ - } - if (H.method == "OPTIONS" || H.method == "HEAD"){ - H.SendResponse("200", "OK", myConn); -+ responded = true; - return; - } - - H.StartResponse(H, myConn, VLCworkaround || config->getBool("nonchunked")); -+ responded = true; - // we assume whole fragments - but timestamps may be altered at will - contPAT = fragmentIndex; // PAT continuity counter - contPMT = fragmentIndex; // PMT continuity counter -@@ -419,6 +426,7 @@ namespace Mist{ - // Strip /hls// from url - std::string url = H.url.substr(H.url.find('/', 5) + 1); - sendHlsManifest(url); -+ responded = true; - } - } - --- -2.25.1 - - -From f3c003481d572c140643e9bbfb292942d8e2ed38 Mon Sep 17 00:00:00 2001 -From: Thulinma -Date: Mon, 26 Sep 2022 12:44:30 +0200 -Subject: [PATCH 24/38] Made FLV memory-based loader functions use const - pointers - ---- - lib/flv_tag.cpp | 6 +++--- - lib/flv_tag.h | 6 +++--- - 2 files changed, 6 insertions(+), 6 deletions(-) - -diff --git a/lib/flv_tag.cpp b/lib/flv_tag.cpp -index 9ccb6c41..780c6c5a 100644 ---- a/lib/flv_tag.cpp -+++ b/lib/flv_tag.cpp -@@ -49,7 +49,7 @@ bool FLV::check_header(char *header){ - - /// Checks the first 3 bytes for the string "FLV". Implementing a basic FLV header check, - /// returning true if it is, false if not. --bool FLV::is_header(char *header){ -+bool FLV::is_header(const char *header){ - if (header[0] != 'F') return false; - if (header[1] != 'L') return false; - if (header[2] != 'V') return false; -@@ -624,7 +624,7 @@ bool FLV::Tag::ChunkLoader(const RTMPStream::Chunk &O){ - /// \param S The size of the data buffer. - /// \param P The current position in the data buffer. Will be updated to reflect new position. - /// \return True if count bytes are read succesfully, false otherwise. --bool FLV::Tag::MemReadUntil(char *buffer, unsigned int count, unsigned int &sofar, char *D, -+bool FLV::Tag::MemReadUntil(char *buffer, unsigned int count, unsigned int &sofar, const char *D, - unsigned int S, unsigned int &P){ - if (sofar >= count){return true;} - int r = 0; -@@ -646,7 +646,7 @@ bool FLV::Tag::MemReadUntil(char *buffer, unsigned int count, unsigned int &sofa - /// location of the data buffer. \param S The size of the data buffer. \param P The current position - /// in the data buffer. Will be updated to reflect new position. \return True if a whole tag is - /// succesfully read, false otherwise. --bool FLV::Tag::MemLoader(char *D, unsigned int S, unsigned int &P){ -+bool FLV::Tag::MemLoader(const char *D, unsigned int S, unsigned int &P){ - if (len < 15){len = 15;} - if (!checkBufferSize()){return false;} - if (done){ -diff --git a/lib/flv_tag.h b/lib/flv_tag.h -index ca27f472..10f616b9 100644 ---- a/lib/flv_tag.h -+++ b/lib/flv_tag.h -@@ -22,7 +22,7 @@ namespace FLV{ - - // functions - bool check_header(char *header); ///< Checks a FLV Header for validness. -- bool is_header(char *header); ///< Checks the first 3 bytes for the string "FLV". -+ bool is_header(const char *header); ///< Checks the first 3 bytes for the string "FLV". - - /// Helper function that can quickly skip through a file looking for a particular tag type - bool seekToTagType(FILE *f, uint8_t type); -@@ -55,7 +55,7 @@ namespace FLV{ - bool DTSCMetaInit(const DTSC::Meta &M, std::set &selTracks); - void toMeta(DTSC::Meta &meta, AMF::Object &amf_storage); - void toMeta(DTSC::Meta &meta, AMF::Object &amf_storage, size_t &reTrack, const std::map &targetParams); -- bool MemLoader(char *D, unsigned int S, unsigned int &P); -+ bool MemLoader(const char *D, unsigned int S, unsigned int &P); - bool FileLoader(FILE *f); - unsigned int getTrackID(); - char *getData(); -@@ -68,7 +68,7 @@ namespace FLV{ - void setLen(); - bool checkBufferSize(); - // loader helper functions -- bool MemReadUntil(char *buffer, unsigned int count, unsigned int &sofar, char *D, -+ bool MemReadUntil(char *buffer, unsigned int count, unsigned int &sofar, const char *D, - unsigned int S, unsigned int &P); - bool FileReadUntil(char *buffer, unsigned int count, unsigned int &sofar, FILE *f); - }; --- -2.25.1 - - -From 3e85da2afd1b3d36290763eebf84dd66ad08636c Mon Sep 17 00:00:00 2001 -From: Thulinma -Date: Mon, 19 Sep 2022 17:26:15 +0200 -Subject: [PATCH 25/38] Match libav's RTMP receive window rollover behaviour - ---- - lib/rtmpchunks.cpp | 5 +++++ - src/output/output_rtmp.cpp | 2 +- - 2 files changed, 6 insertions(+), 1 deletion(-) - -diff --git a/lib/rtmpchunks.cpp b/lib/rtmpchunks.cpp -index e7c151f0..5f9d33f0 100644 ---- a/lib/rtmpchunks.cpp -+++ b/lib/rtmpchunks.cpp -@@ -475,6 +475,11 @@ bool RTMPStream::Chunk::Parse(Socket::Buffer &buffer){ - } - lastrecv[cs_id] = *this; - RTMPStream::rec_cnt += i + real_len; -+ if (RTMPStream::rec_cnt >= 0xf0000000){ -+ INFO_MSG("Resetting receive window due to impending rollover"); -+ RTMPStream::rec_cnt -= 0xf0000000; -+ RTMPStream::rec_window_at = 0; -+ } - if (len_left == 0){ - return true; - }else{ -diff --git a/src/output/output_rtmp.cpp b/src/output/output_rtmp.cpp -index 16a400d0..23194f4a 100644 ---- a/src/output/output_rtmp.cpp -+++ b/src/output/output_rtmp.cpp -@@ -1599,7 +1599,7 @@ namespace Mist{ - while (next.Parse(inputBuffer)){ - - // send ACK if we received a whole window -- if ((RTMPStream::rec_cnt - RTMPStream::rec_window_at > RTMPStream::rec_window_size) || Util::bootSecs() > lastAck+15){ -+ if ((RTMPStream::rec_cnt - RTMPStream::rec_window_at > RTMPStream::rec_window_size / 4) || Util::bootSecs() > lastAck+15){ - lastAck = Util::bootSecs(); - RTMPStream::rec_window_at = RTMPStream::rec_cnt; - myConn.SendNow(RTMPStream::SendCTL(3, RTMPStream::rec_cnt)); // send ack (msg 3) --- -2.25.1 - - -From 074e7570284a094c12b00b88aa5cdf4a8f7fe5e4 Mon Sep 17 00:00:00 2001 -From: Ramkoemar -Date: Mon, 18 Oct 2021 14:29:13 +0200 -Subject: [PATCH 26/38] Sessions rework - ---- - CMakeLists.txt | 11 + - lib/comms.cpp | 428 +++++--- - lib/comms.h | 50 +- - lib/defines.h | 7 +- - src/controller/controller.cpp | 1 - - src/controller/controller_api.cpp | 1 + - src/controller/controller_statistics.cpp | 1185 ++++++++-------------- - src/controller/controller_statistics.h | 61 +- - src/controller/controller_storage.cpp | 22 +- - src/input/input.cpp | 12 +- - src/input/input.h | 2 +- - src/input/input_rtsp.cpp | 5 +- - src/input/input_sdp.cpp | 5 +- - src/input/input_ts.cpp | 5 +- - src/input/input_tssrt.cpp | 2 +- - src/input/input_tssrt.h | 2 +- - src/output/output.cpp | 125 +-- - src/output/output.h | 9 +- - src/output/output_cmaf.cpp | 2 +- - src/output/output_cmaf.h | 3 +- - src/output/output_http.cpp | 28 +- - src/output/output_tssrt.cpp | 2 +- - src/output/output_tssrt.h | 2 +- - src/output/output_webrtc.cpp | 2 +- - src/output/output_webrtc.h | 2 +- - src/process/process_exec.cpp | 4 +- - src/session.cpp | 367 +++++++ - 27 files changed, 1192 insertions(+), 1153 deletions(-) - create mode 100644 src/session.cpp - -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 0e9bd644..6af16808 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -592,6 +592,17 @@ makeOutput(RTSP rtsp)#LTS - makeOutput(WAV wav)#LTS - makeOutput(SDP sdp http) - -+add_executable(MistSession -+ ${BINARY_DIR}/mist/.headers -+ src/session.cpp -+) -+install( -+ TARGETS MistSession -+ DESTINATION bin -+) -+target_link_libraries(MistSession mist) -+ -+ - add_executable(MistProcFFMPEG - ${BINARY_DIR}/mist/.headers - src/process/process_ffmpeg.cpp -diff --git a/lib/comms.cpp b/lib/comms.cpp -index 108c686a..85f1e6d7 100644 ---- a/lib/comms.cpp -+++ b/lib/comms.cpp -@@ -7,6 +7,7 @@ - #include "timing.h" - #include - #include -+#include "config.h" - - namespace Comms{ - Comms::Comms(){ -@@ -141,22 +142,197 @@ namespace Comms{ - } - } - -- Statistics::Statistics() : Comms(){sem.open(SEM_STATISTICS, O_CREAT | O_RDWR, ACCESSPERMS, 1);} -+ Sessions::Sessions() : Connections(){sem.open(SEM_STATISTICS, O_CREAT | O_RDWR, ACCESSPERMS, 1);} - -- void Statistics::unload(){ -- if (index != INVALID_RECORD_INDEX){ -- setStatus(COMM_STATUS_DISCONNECT | getStatus()); -+ void Sessions::reload(bool _master, bool reIssue){ -+ Comms::reload(COMMS_STATISTICS, COMMS_STATISTICS_INITSIZE, _master, reIssue); -+ } -+ -+ std::string Sessions::getSessId() const{return sessId.string(index);} -+ std::string Sessions::getSessId(size_t idx) const{return (master ? sessId.string(idx) : 0);} -+ void Sessions::setSessId(std::string _sid){sessId.set(_sid, index);} -+ void Sessions::setSessId(std::string _sid, size_t idx){ -+ if (!master){return;} -+ sessId.set(_sid, idx); -+ } -+ -+ bool Sessions::sessIdExists(std::string _sid){ -+ for (size_t i = 0; i < recordCount(); i++){ -+ if (getStatus(i) == COMM_STATUS_INVALID || (getStatus(i) & COMM_STATUS_DISCONNECT)){continue;} -+ if (getSessId(i) == _sid){ -+ if (Util::Procs::isRunning(getPid(i))){ -+ return true; -+ } -+ } - } -- index = INVALID_RECORD_INDEX; -+ return false; - } - -- void Statistics::reload(bool _master, bool reIssue){ -- Comms::reload(COMMS_STATISTICS, COMMS_STATISTICS_INITSIZE, _master, reIssue); -+ void Sessions::addFields(){ -+ Connections::addFields(); -+ dataAccX.addField("sessid", RAX_STRING, 80); - } - -- void Statistics::addFields(){ -+ void Sessions::nullFields(){ -+ Connections::nullFields(); -+ setSessId(""); -+ } -+ -+ void Sessions::fieldAccess(){ -+ Connections::fieldAccess(); -+ sessId = dataAccX.getFieldAccX("sessid"); -+ } -+ -+ Users::Users() : Comms(){} -+ -+ Users::Users(const Users &rhs) : Comms(){ -+ if (rhs){ -+ reload(rhs.streamName, (size_t)rhs.getTrack()); -+ if (*this){ -+ setKeyNum(rhs.getKeyNum()); -+ setTrack(rhs.getTrack()); -+ } -+ } -+ } -+ -+ void Users::reload(const std::string &_streamName, bool _master, bool reIssue){ -+ streamName = _streamName; -+ -+ char semName[NAME_BUFFER_SIZE]; -+ snprintf(semName, NAME_BUFFER_SIZE, SEM_USERS, streamName.c_str()); -+ sem.open(semName, O_CREAT | O_RDWR, ACCESSPERMS, 1); -+ -+ char userPageName[NAME_BUFFER_SIZE]; -+ snprintf(userPageName, NAME_BUFFER_SIZE, COMMS_USERS, streamName.c_str()); -+ -+ Comms::reload(userPageName, COMMS_USERS_INITSIZE, _master, reIssue); -+ } -+ -+ void Users::addFields(){ -+ Comms::addFields(); -+ dataAccX.addField("track", RAX_64UINT); -+ dataAccX.addField("keynum", RAX_64UINT); -+ } -+ -+ void Users::nullFields(){ -+ Comms::nullFields(); -+ setTrack(0); -+ setKeyNum(0); -+ } -+ -+ void Users::fieldAccess(){ -+ Comms::fieldAccess(); -+ track = dataAccX.getFieldAccX("track"); -+ keyNum = dataAccX.getFieldAccX("keynum"); -+ } -+ -+ void Users::reload(const std::string &_streamName, size_t idx, uint8_t initialState){ -+ reload(_streamName); -+ if (dataPage){ -+ setTrack(idx); -+ setKeyNum(0); -+ setStatus(initialState); -+ } -+ } -+ -+ uint32_t Users::getTrack() const{return track.uint(index);} -+ uint32_t Users::getTrack(size_t idx) const{return (master ? track.uint(idx) : 0);} -+ void Users::setTrack(uint32_t _track){track.set(_track, index);} -+ void Users::setTrack(uint32_t _track, size_t idx){ -+ if (!master){return;} -+ track.set(_track, idx); -+ } -+ -+ size_t Users::getKeyNum() const{return keyNum.uint(index);} -+ size_t Users::getKeyNum(size_t idx) const{return (master ? keyNum.uint(idx) : 0);} -+ void Users::setKeyNum(size_t _keyNum){keyNum.set(_keyNum, index);} -+ void Users::setKeyNum(size_t _keyNum, size_t idx){ -+ if (!master){return;} -+ keyNum.set(_keyNum, idx); -+ } -+ -+ /// \brief Claims a spot on the connections page for the input/output which calls this function -+ /// Starts the MistSession binary for each session, which handles the statistics -+ /// and the USER_NEW and USER_END triggers -+ /// \param streamName: Name of the stream the input is providing or an output is making available to viewers -+ /// \param ip: IP address of the viewer which wants to access streamName. For inputs this value can be set to any value -+ /// \param sid: Session ID given by the player or randomly generated -+ /// \param protocol: Protocol currently in use for this connection -+ /// \param sessionMode: Determines how a viewer session is defined: -+ // If set to 0, all connections with the same viewer IP and stream name are bundled. -+ // If set to 1, all connections with the same viewer IP and player ID are bundled. -+ // If set to 2, all connections with the same player ID and stream name are bundled. -+ // If set to 3, all connections with the same viewer IP, player ID and stream name are bundled. -+ /// \param _master: If True, we are reading from this page. If False, we are writing (to our entry) on this page -+ /// \param reIssue: If True, claim a new entry on this page -+ void Connections::reload(std::string streamName, std::string ip, std::string sid, std::string protocol, std::string reqUrl, uint64_t sessionMode, bool _master, bool reIssue){ -+ if (sessionMode == 0xFFFFFFFFFFFFFFFFull){ -+ FAIL_MSG("The session mode was not initialised properly. Assuming default behaviour of bundling by viewer IP, stream name and player id"); -+ sessionMode = SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID; -+ } -+ // Generate a unique session ID for each viewer, input or output -+ sessionId = generateSession(streamName, ip, sid, protocol, sessionMode); -+ if (protocol.size() >= 6 && protocol.substr(0, 6) == "INPUT:"){ -+ sessionId = "I" + sessionId; -+ }else if (protocol.size() >= 7 && protocol.substr(0, 7) == "OUTPUT:"){ -+ sessionId = "O" + sessionId; -+ } -+ char userPageName[NAME_BUFFER_SIZE]; -+ snprintf(userPageName, NAME_BUFFER_SIZE, COMMS_SESSIONS, sessionId.c_str()); -+ // Check if the page exists, if not, spawn new session process -+ if (!_master){ -+ dataPage.init(userPageName, 0, false, false); -+ if (!dataPage){ -+ pid_t thisPid; -+ std::deque args; -+ args.push_back(Util::getMyPath() + "MistSession"); -+ args.push_back(sessionId); -+ args.push_back("--sessionmode"); -+ args.push_back(JSON::Value(sessionMode).asString()); -+ args.push_back("--streamname"); -+ args.push_back(streamName); -+ args.push_back("--ip"); -+ args.push_back(ip); -+ args.push_back("--sid"); -+ args.push_back(sid); -+ args.push_back("--protocol"); -+ args.push_back(protocol); -+ args.push_back("--requrl"); -+ args.push_back(reqUrl); -+ int err = fileno(stderr); -+ thisPid = Util::Procs::StartPiped(args, 0, 0, &err); -+ Util::Procs::forget(thisPid); -+ HIGH_MSG("Spawned new session executeable (pid %u) for sessionId '%s', corresponding to host %s and stream %s", thisPid, sessionId.c_str(), ip.c_str(), streamName.c_str()); -+ } -+ } -+ // Open SEM_SESSION -+ if(!sem){ -+ char semName[NAME_BUFFER_SIZE]; -+ snprintf(semName, NAME_BUFFER_SIZE, SEM_SESSION, sessionId.c_str()); -+ sem.open(semName, O_RDWR, ACCESSPERMS, 1); -+ } -+ Comms::reload(userPageName, COMMS_SESSIONS_INITSIZE, _master, reIssue); -+ VERYHIGH_MSG("Reloading connection. Claimed record %lu", index); -+ } -+ -+ /// \brief Marks the data page as closed, so that we longer write any new data to is -+ void Connections::setExit(){ -+ if (!master){return;} -+ dataAccX.setExit(); -+ } -+ -+ bool Connections::getExit(){ -+ return dataAccX.isExit(); -+ } -+ -+ void Connections::unload(){ -+ if (index != INVALID_RECORD_INDEX){ -+ setStatus(COMM_STATUS_DISCONNECT | getStatus()); -+ } -+ index = INVALID_RECORD_INDEX; -+ } -+ void Connections::addFields(){ - Comms::addFields(); -- dataAccX.addField("sync", RAX_UINT); - dataAccX.addField("now", RAX_64UINT); - dataAccX.addField("time", RAX_64UINT); - dataAccX.addField("lastsecond", RAX_64UINT); -@@ -165,15 +341,15 @@ namespace Comms{ - dataAccX.addField("host", RAX_RAW, 16); - dataAccX.addField("stream", RAX_STRING, 100); - dataAccX.addField("connector", RAX_STRING, 20); -- dataAccX.addField("crc", RAX_32UINT); -+ dataAccX.addField("tags", RAX_STRING, 512); - dataAccX.addField("pktcount", RAX_64UINT); - dataAccX.addField("pktloss", RAX_64UINT); - dataAccX.addField("pktretrans", RAX_64UINT); - } - -- void Statistics::nullFields(){ -+ void Connections::nullFields(){ - Comms::nullFields(); -- setCRC(0); -+ setTags(""); - setConnector(""); - setStream(""); - setHost(""); -@@ -182,15 +358,13 @@ namespace Comms{ - setLastSecond(0); - setTime(0); - setNow(0); -- setSync(0); - setPacketCount(0); - setPacketLostCount(0); - setPacketRetransmitCount(0); - } - -- void Statistics::fieldAccess(){ -+ void Connections::fieldAccess(){ - Comms::fieldAccess(); -- sync = dataAccX.getFieldAccX("sync"); - now = dataAccX.getFieldAccX("now"); - time = dataAccX.getFieldAccX("time"); - lastSecond = dataAccX.getFieldAccX("lastsecond"); -@@ -199,209 +373,159 @@ namespace Comms{ - host = dataAccX.getFieldAccX("host"); - stream = dataAccX.getFieldAccX("stream"); - connector = dataAccX.getFieldAccX("connector"); -- crc = dataAccX.getFieldAccX("crc"); -+ tags = dataAccX.getFieldAccX("tags"); - pktcount = dataAccX.getFieldAccX("pktcount"); - pktloss = dataAccX.getFieldAccX("pktloss"); - pktretrans = dataAccX.getFieldAccX("pktretrans"); - } - -- uint8_t Statistics::getSync() const{return sync.uint(index);} -- uint8_t Statistics::getSync(size_t idx) const{return (master ? sync.uint(idx) : 0);} -- void Statistics::setSync(uint8_t _sync){sync.set(_sync, index);} -- void Statistics::setSync(uint8_t _sync, size_t idx){ -- if (!master){return;} -- sync.set(_sync, idx); -- } -- -- uint64_t Statistics::getNow() const{return now.uint(index);} -- uint64_t Statistics::getNow(size_t idx) const{return (master ? now.uint(idx) : 0);} -- void Statistics::setNow(uint64_t _now){now.set(_now, index);} -- void Statistics::setNow(uint64_t _now, size_t idx){ -+ uint64_t Connections::getNow() const{return now.uint(index);} -+ uint64_t Connections::getNow(size_t idx) const{return (master ? now.uint(idx) : 0);} -+ void Connections::setNow(uint64_t _now){now.set(_now, index);} -+ void Connections::setNow(uint64_t _now, size_t idx){ - if (!master){return;} - now.set(_now, idx); - } - -- uint64_t Statistics::getTime() const{return time.uint(index);} -- uint64_t Statistics::getTime(size_t idx) const{return (master ? time.uint(idx) : 0);} -- void Statistics::setTime(uint64_t _time){time.set(_time, index);} -- void Statistics::setTime(uint64_t _time, size_t idx){ -+ uint64_t Connections::getTime() const{return time.uint(index);} -+ uint64_t Connections::getTime(size_t idx) const{return (master ? time.uint(idx) : 0);} -+ void Connections::setTime(uint64_t _time){time.set(_time, index);} -+ void Connections::setTime(uint64_t _time, size_t idx){ - if (!master){return;} - time.set(_time, idx); - } - -- uint64_t Statistics::getLastSecond() const{return lastSecond.uint(index);} -- uint64_t Statistics::getLastSecond(size_t idx) const{ -+ uint64_t Connections::getLastSecond() const{return lastSecond.uint(index);} -+ uint64_t Connections::getLastSecond(size_t idx) const{ - return (master ? lastSecond.uint(idx) : 0); - } -- void Statistics::setLastSecond(uint64_t _lastSecond){lastSecond.set(_lastSecond, index);} -- void Statistics::setLastSecond(uint64_t _lastSecond, size_t idx){ -+ void Connections::setLastSecond(uint64_t _lastSecond){lastSecond.set(_lastSecond, index);} -+ void Connections::setLastSecond(uint64_t _lastSecond, size_t idx){ - if (!master){return;} - lastSecond.set(_lastSecond, idx); - } - -- uint64_t Statistics::getDown() const{return down.uint(index);} -- uint64_t Statistics::getDown(size_t idx) const{return (master ? down.uint(idx) : 0);} -- void Statistics::setDown(uint64_t _down){down.set(_down, index);} -- void Statistics::setDown(uint64_t _down, size_t idx){ -+ uint64_t Connections::getDown() const{return down.uint(index);} -+ uint64_t Connections::getDown(size_t idx) const{return (master ? down.uint(idx) : 0);} -+ void Connections::setDown(uint64_t _down){down.set(_down, index);} -+ void Connections::setDown(uint64_t _down, size_t idx){ - if (!master){return;} - down.set(_down, idx); - } - -- uint64_t Statistics::getUp() const{return up.uint(index);} -- uint64_t Statistics::getUp(size_t idx) const{return (master ? up.uint(idx) : 0);} -- void Statistics::setUp(uint64_t _up){up.set(_up, index);} -- void Statistics::setUp(uint64_t _up, size_t idx){ -+ uint64_t Connections::getUp() const{return up.uint(index);} -+ uint64_t Connections::getUp(size_t idx) const{return (master ? up.uint(idx) : 0);} -+ void Connections::setUp(uint64_t _up){up.set(_up, index);} -+ void Connections::setUp(uint64_t _up, size_t idx){ - if (!master){return;} - up.set(_up, idx); - } - -- std::string Statistics::getHost() const{return std::string(host.ptr(index), 16);} -- std::string Statistics::getHost(size_t idx) const{ -+ std::string Connections::getHost() const{return std::string(host.ptr(index), 16);} -+ std::string Connections::getHost(size_t idx) const{ - if (!master){return std::string((size_t)16, (char)'\000');} - return std::string(host.ptr(idx), 16); - } -- void Statistics::setHost(std::string _host){host.set(_host, index);} -- void Statistics::setHost(std::string _host, size_t idx){ -+ void Connections::setHost(std::string _host){host.set(_host, index);} -+ void Connections::setHost(std::string _host, size_t idx){ - if (!master){return;} - host.set(_host, idx); - } - -- std::string Statistics::getStream() const{return stream.string(index);} -- std::string Statistics::getStream(size_t idx) const{return (master ? stream.string(idx) : "");} -- void Statistics::setStream(std::string _stream){stream.set(_stream, index);} -- void Statistics::setStream(std::string _stream, size_t idx){ -+ std::string Connections::getStream() const{return stream.string(index);} -+ std::string Connections::getStream(size_t idx) const{return (master ? stream.string(idx) : "");} -+ void Connections::setStream(std::string _stream){stream.set(_stream, index);} -+ void Connections::setStream(std::string _stream, size_t idx){ - if (!master){return;} - stream.set(_stream, idx); - } - -- std::string Statistics::getConnector() const{return connector.string(index);} -- std::string Statistics::getConnector(size_t idx) const{ -+ std::string Connections::getConnector() const{return connector.string(index);} -+ std::string Connections::getConnector(size_t idx) const{ - return (master ? connector.string(idx) : ""); - } -- void Statistics::setConnector(std::string _connector){connector.set(_connector, index);} -- void Statistics::setConnector(std::string _connector, size_t idx){ -+ void Connections::setConnector(std::string _connector){connector.set(_connector, index);} -+ void Connections::setConnector(std::string _connector, size_t idx){ - if (!master){return;} - connector.set(_connector, idx); - } - -- uint32_t Statistics::getCRC() const{return crc.uint(index);} -- uint32_t Statistics::getCRC(size_t idx) const{return (master ? crc.uint(idx) : 0);} -- void Statistics::setCRC(uint32_t _crc){crc.set(_crc, index);} -- void Statistics::setCRC(uint32_t _crc, size_t idx){ -+ bool Connections::hasConnector(size_t idx, std::string protocol){ -+ std::stringstream sstream(connector.string(idx)); -+ std::string _conn; -+ while (std::getline(sstream, _conn, ',')){ -+ if (_conn == protocol){ -+ return true; -+ } -+ } -+ return false; -+ } -+ -+ std::string Connections::getTags() const{return tags.string(index);} -+ std::string Connections::getTags(size_t idx) const{return (master ? tags.string(idx) : 0);} -+ void Connections::setTags(std::string _sid){tags.set(_sid, index);} -+ void Connections::setTags(std::string _sid, size_t idx){ - if (!master){return;} -- crc.set(_crc, idx); -+ tags.set(_sid, idx); - } - -- uint64_t Statistics::getPacketCount() const{return pktcount.uint(index);} -- uint64_t Statistics::getPacketCount(size_t idx) const{ -+ uint64_t Connections::getPacketCount() const{return pktcount.uint(index);} -+ uint64_t Connections::getPacketCount(size_t idx) const{ - return (master ? pktcount.uint(idx) : 0); - } -- void Statistics::setPacketCount(uint64_t _count){pktcount.set(_count, index);} -- void Statistics::setPacketCount(uint64_t _count, size_t idx){ -+ void Connections::setPacketCount(uint64_t _count){pktcount.set(_count, index);} -+ void Connections::setPacketCount(uint64_t _count, size_t idx){ - if (!master){return;} - pktcount.set(_count, idx); - } - -- uint64_t Statistics::getPacketLostCount() const{return pktloss.uint(index);} -- uint64_t Statistics::getPacketLostCount(size_t idx) const{ -+ uint64_t Connections::getPacketLostCount() const{return pktloss.uint(index);} -+ uint64_t Connections::getPacketLostCount(size_t idx) const{ - return (master ? pktloss.uint(idx) : 0); - } -- void Statistics::setPacketLostCount(uint64_t _lost){pktloss.set(_lost, index);} -- void Statistics::setPacketLostCount(uint64_t _lost, size_t idx){ -+ void Connections::setPacketLostCount(uint64_t _lost){pktloss.set(_lost, index);} -+ void Connections::setPacketLostCount(uint64_t _lost, size_t idx){ - if (!master){return;} - pktloss.set(_lost, idx); - } - -- uint64_t Statistics::getPacketRetransmitCount() const{return pktretrans.uint(index);} -- uint64_t Statistics::getPacketRetransmitCount(size_t idx) const{ -+ uint64_t Connections::getPacketRetransmitCount() const{return pktretrans.uint(index);} -+ uint64_t Connections::getPacketRetransmitCount(size_t idx) const{ - return (master ? pktretrans.uint(idx) : 0); - } -- void Statistics::setPacketRetransmitCount(uint64_t _retrans){pktretrans.set(_retrans, index);} -- void Statistics::setPacketRetransmitCount(uint64_t _retrans, size_t idx){ -+ void Connections::setPacketRetransmitCount(uint64_t _retrans){pktretrans.set(_retrans, index);} -+ void Connections::setPacketRetransmitCount(uint64_t _retrans, size_t idx){ - if (!master){return;} - pktretrans.set(_retrans, idx); - } - -- std::string Statistics::getSessId() const{return getSessId(index);} -- -- std::string Statistics::getSessId(size_t idx) const{ -- char res[140]; -- memset(res, 0, 140); -- std::string tmp = host.string(idx); -- memcpy(res, tmp.c_str(), (tmp.size() > 16 ? 16 : tmp.size())); -- tmp = stream.string(idx); -- memcpy(res + 16, tmp.c_str(), (tmp.size() > 100 ? 100 : tmp.size())); -- tmp = connector.string(idx); -- memcpy(res + 116, tmp.c_str(), (tmp.size() > 20 ? 20 : tmp.size())); -- Bit::htobl(res + 136, crc.uint(idx)); -- return Secure::md5(res, 140); -- } -- -- Users::Users() : Comms(){} -- -- Users::Users(const Users &rhs) : Comms(){ -- if (rhs){ -- reload(rhs.streamName, (size_t)rhs.getTrack()); -- if (*this){ -- setKeyNum(rhs.getKeyNum()); -- setTrack(rhs.getTrack()); -- } -+ /// \brief Generates a session ID which is unique per viewer -+ /// \return generated session ID as string -+ std::string Connections::generateSession(std::string streamName, std::string ip, std::string sid, std::string connector, uint64_t sessionMode){ -+ std::string concat; -+ // First bit defines whether to include stream name -+ if (sessionMode > 7){ -+ concat += streamName; -+ sessionMode -= 8; - } -- } -- -- void Users::reload(const std::string &_streamName, bool _master, bool reIssue){ -- streamName = _streamName; -- -- char semName[NAME_BUFFER_SIZE]; -- snprintf(semName, NAME_BUFFER_SIZE, SEM_USERS, streamName.c_str()); -- sem.open(semName, O_CREAT | O_RDWR, ACCESSPERMS, 1); -- -- char userPageName[NAME_BUFFER_SIZE]; -- snprintf(userPageName, NAME_BUFFER_SIZE, COMMS_USERS, streamName.c_str()); -- -- Comms::reload(userPageName, COMMS_USERS_INITSIZE, _master, reIssue); -- } -- -- void Users::addFields(){ -- Comms::addFields(); -- dataAccX.addField("track", RAX_64UINT); -- dataAccX.addField("keynum", RAX_64UINT); -- } -- -- void Users::nullFields(){ -- Comms::nullFields(); -- setTrack(0); -- setKeyNum(0); -- } -- -- void Users::fieldAccess(){ -- Comms::fieldAccess(); -- track = dataAccX.getFieldAccX("track"); -- keyNum = dataAccX.getFieldAccX("keynum"); -- } -- -- void Users::reload(const std::string &_streamName, size_t idx, uint8_t initialState){ -- reload(_streamName); -- if (dataPage){ -- setTrack(idx); -- setKeyNum(0); -- setStatus(initialState); -+ // Second bit defines whether to include viewer ip -+ if (sessionMode > 3){ -+ concat += ip; -+ sessionMode -= 4; - } -- } -- -- uint32_t Users::getTrack() const{return track.uint(index);} -- uint32_t Users::getTrack(size_t idx) const{return (master ? track.uint(idx) : 0);} -- void Users::setTrack(uint32_t _track){track.set(_track, index);} -- void Users::setTrack(uint32_t _track, size_t idx){ -- if (!master){return;} -- track.set(_track, idx); -- } -- -- size_t Users::getKeyNum() const{return keyNum.uint(index);} -- size_t Users::getKeyNum(size_t idx) const{return (master ? keyNum.uint(idx) : 0);} -- void Users::setKeyNum(size_t _keyNum){keyNum.set(_keyNum, index);} -- void Users::setKeyNum(size_t _keyNum, size_t idx){ -- if (!master){return;} -- keyNum.set(_keyNum, idx); -+ // Third bit defines whether to include player ip -+ if (sessionMode > 1){ -+ concat += sid; -+ sessionMode -= 2; -+ } -+ // Fourth bit defines whether to include protocol -+ if (sessionMode == 1){ -+ concat += connector; -+ sessionMode = 0; -+ } -+ if (sessionMode > 0){ -+ WARN_MSG("Could not resolve session mode of value %lu", sessionMode); -+ } -+ return Secure::sha256(concat.c_str(), concat.length()); - } - }// namespace Comms -diff --git a/lib/comms.h b/lib/comms.h -index 9f459422..9a5c0ea9 100644 ---- a/lib/comms.h -+++ b/lib/comms.h -@@ -64,21 +64,21 @@ namespace Comms{ - Util::FieldAccX pid; - }; - -- class Statistics : public Comms{ -+ class Connections : public Comms{ - public: -- Statistics(); -- operator bool() const{return dataPage.mapped && (master || index != INVALID_RECORD_INDEX);} -+ void reload(std::string streamName, std::string ip, std::string sid, std::string protocol, std::string reqUrl, uint64_t sessionMode, bool _master = false, bool reIssue = false); - void unload(); -- void reload(bool _master = false, bool reIssue = false); -+ operator bool() const{return dataPage.mapped && (master || index != INVALID_RECORD_INDEX);} -+ std::string generateSession(std::string streamName, std::string ip, std::string sid, std::string connector, uint64_t sessionMode); -+ std::string sessionId; -+ -+ void setExit(); -+ bool getExit(); -+ - virtual void addFields(); - virtual void nullFields(); - virtual void fieldAccess(); - -- uint8_t getSync() const; -- uint8_t getSync(size_t idx) const; -- void setSync(uint8_t _sync); -- void setSync(uint8_t _sync, size_t idx); -- - uint64_t getNow() const; - uint64_t getNow(size_t idx) const; - void setNow(uint64_t _now); -@@ -118,11 +118,12 @@ namespace Comms{ - std::string getConnector(size_t idx) const; - void setConnector(std::string _connector); - void setConnector(std::string _connector, size_t idx); -+ bool hasConnector(size_t idx, std::string protocol); - -- uint32_t getCRC() const; -- uint32_t getCRC(size_t idx) const; -- void setCRC(uint32_t _crc); -- void setCRC(uint32_t _crc, size_t idx); -+ std::string getTags() const; -+ std::string getTags(size_t idx) const; -+ void setTags(std::string _sid); -+ void setTags(std::string _sid, size_t idx); - - uint64_t getPacketCount() const; - uint64_t getPacketCount(size_t idx) const; -@@ -139,11 +140,7 @@ namespace Comms{ - void setPacketRetransmitCount(uint64_t _retransmit); - void setPacketRetransmitCount(uint64_t _retransmit, size_t idx); - -- std::string getSessId() const; -- std::string getSessId(size_t index) const; -- -- private: -- Util::FieldAccX sync; -+ protected: - Util::FieldAccX now; - Util::FieldAccX time; - Util::FieldAccX lastSecond; -@@ -152,7 +149,8 @@ namespace Comms{ - Util::FieldAccX host; - Util::FieldAccX stream; - Util::FieldAccX connector; -- Util::FieldAccX crc; -+ Util::FieldAccX sessId; -+ Util::FieldAccX tags; - Util::FieldAccX pktcount; - Util::FieldAccX pktloss; - Util::FieldAccX pktretrans; -@@ -186,4 +184,18 @@ namespace Comms{ - Util::FieldAccX track; - Util::FieldAccX keyNum; - }; -+ -+ class Sessions : public Connections{ -+ public: -+ Sessions(); -+ void reload(bool _master = false, bool reIssue = false); -+ std::string getSessId() const; -+ std::string getSessId(size_t idx) const; -+ void setSessId(std::string _sid); -+ void setSessId(std::string _sid, size_t idx); -+ bool sessIdExists(std::string _sid); -+ virtual void addFields(); -+ virtual void nullFields(); -+ virtual void fieldAccess(); -+ }; - }// namespace Comms -diff --git a/lib/defines.h b/lib/defines.h -index 10c4b9b1..203b75db 100644 ---- a/lib/defines.h -+++ b/lib/defines.h -@@ -196,11 +196,14 @@ static inline void show_stackframe(){} - #define TRACK_PAGE_RECORDSIZE 36 - - #define COMMS_STATISTICS "MstStat" --#define COMMS_STATISTICS_INITSIZE 8 * 1024 * 1024 -+#define COMMS_STATISTICS_INITSIZE 16 * 1024 * 1024 - - #define COMMS_USERS "MstUser%s" //%s stream name - #define COMMS_USERS_INITSIZE 512 * 1024 - -+#define COMMS_SESSIONS "MstSession%s" -+#define COMMS_SESSIONS_INITSIZE 8 * 1024 * 1024 -+ - #define SEM_STATISTICS "/MstStat" - #define SEM_USERS "/MstUser%s" //%s stream name - -@@ -226,7 +229,9 @@ static inline void show_stackframe(){} - #define SEM_LIVE "/MstLIVE%s" //%s stream name - #define SEM_INPUT "/MstInpt%s" //%s stream name - #define SEM_TRACKLIST "/MstTRKS%s" //%s stream name -+#define SEM_SESSION "MstSess%s" - #define SEM_SESSCACHE "/MstSessCacheLock" -+#define SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID 14 - #define SHM_CAPA "MstCapa" - #define SHM_PROTO "MstProt" - #define SHM_PROXY "MstProx" -diff --git a/src/controller/controller.cpp b/src/controller/controller.cpp -index a0ff2ee8..9e17ca6b 100644 ---- a/src/controller/controller.cpp -+++ b/src/controller/controller.cpp -@@ -306,7 +306,6 @@ int main_loop(int argc, char **argv){ - if (Controller::Storage["config"].isMember("accesslog")){ - Controller::conf.getOption("accesslog", true)[0u] = Controller::Storage["config"]["accesslog"]; - } -- Controller::maxConnsPerIP = Controller::conf.getInteger("maxconnsperip"); - Controller::Storage["config"]["prometheus"] = Controller::conf.getString("prometheus"); - Controller::Storage["config"]["accesslog"] = Controller::conf.getString("accesslog"); - Controller::normalizeTrustedProxies(Controller::Storage["config"]["trustedproxy"]); -diff --git a/src/controller/controller_api.cpp b/src/controller/controller_api.cpp -index 4d94bb11..02491065 100644 ---- a/src/controller/controller_api.cpp -+++ b/src/controller/controller_api.cpp -@@ -594,6 +594,7 @@ void Controller::handleAPICommands(JSON::Value &Request, JSON::Value &Response){ - out["prometheus"] = in["prometheus"]; - Controller::prometheus = out["prometheus"].asStringRef(); - } -+ if (in.isMember("sessionMode")){out["sessionMode"] = in["sessionMode"];} - if (in.isMember("defaultStream")){out["defaultStream"] = in["defaultStream"];} - if (in.isMember("location") && in["location"].isObject()){ - out["location"]["lat"] = in["location"]["lat"].asDouble(); -diff --git a/src/controller/controller_statistics.cpp b/src/controller/controller_statistics.cpp -index c5b7f643..6e51f4ee 100644 ---- a/src/controller/controller_statistics.cpp -+++ b/src/controller/controller_statistics.cpp -@@ -15,6 +15,7 @@ - #include - #include //for fstatvfs - #include -+#include - - #ifndef KILL_ON_EXIT - #define KILL_ON_EXIT false -@@ -30,7 +31,6 @@ - #define STAT_CLI_UP 64 - #define STAT_CLI_BPS_DOWN 128 - #define STAT_CLI_BPS_UP 256 --#define STAT_CLI_CRC 512 - #define STAT_CLI_SESSID 1024 - #define STAT_CLI_PKTCOUNT 2048 - #define STAT_CLI_PKTLOST 4096 -@@ -46,59 +46,18 @@ - #define STAT_TOT_PERCRETRANS 64 - #define STAT_TOT_ALL 0xFF - --#define COUNTABLE_BYTES 128 * 1024 -- --std::map Controller::sessions; ///< list of sessions that have statistics data available --std::map Controller::connToSession; ///< Map of socket IDs to session info. -+// Mapping of sessId -> session statistics -+std::map sessions; - - std::map Controller::triggerStats; ///< Holds prometheus stats for trigger executions - bool Controller::killOnExit = KILL_ON_EXIT; - tthread::mutex Controller::statsMutex; --unsigned int Controller::maxConnsPerIP = 0; - uint64_t Controller::statDropoff = 0; - static uint64_t cpu_use = 0; - - char noBWCountMatches[1717]; - uint64_t bwLimit = 128 * 1024 * 1024; // gigabit default limit - --/// Session cache shared memory page --IPC::sharedPage *shmSessions = 0; --/// Lock for the session cache shared memory page --IPC::semaphore *cacheLock = 0; -- --/// Convert bandwidth config into memory format --void Controller::updateBandwidthConfig(){ -- size_t offset = 0; -- bwLimit = 128 * 1024 * 1024; // gigabit default limit -- memset(noBWCountMatches, 0, 1717); -- if (Storage.isMember("bandwidth")){ -- if (Storage["bandwidth"].isMember("limit")){bwLimit = Storage["bandwidth"]["limit"].asInt();} -- if (Storage["bandwidth"].isMember("exceptions")){ -- jsonForEach(Storage["bandwidth"]["exceptions"], j){ -- std::string newbins = Socket::getBinForms(j->asStringRef()); -- if (offset + newbins.size() < 1700){ -- memcpy(noBWCountMatches + offset, newbins.data(), newbins.size()); -- offset += newbins.size(); -- } -- } -- } -- } -- //Localhost is always excepted from counts -- { -- std::string newbins = Socket::getBinForms("::1"); -- if (offset + newbins.size() < 1700){ -- memcpy(noBWCountMatches + offset, newbins.data(), newbins.size()); -- offset += newbins.size(); -- } -- } -- { -- std::string newbins = Socket::getBinForms("127.0.0.1/8"); -- if (offset + newbins.size() < 1700){ -- memcpy(noBWCountMatches + offset, newbins.data(), newbins.size()); -- offset += newbins.size(); -- } -- } --} - - // For server-wide totals. Local to this file only. - struct streamTotals{ -@@ -116,11 +75,30 @@ struct streamTotals{ - uint64_t packLoss; - uint64_t packRetrans; - }; -+ -+Comms::Sessions statComm; -+bool statCommActive = false; -+// Global server wide statistics -+static uint64_t servUpBytes = 0; -+static uint64_t servDownBytes = 0; -+static uint64_t servUpOtherBytes = 0; -+static uint64_t servDownOtherBytes = 0; -+static uint64_t servInputs = 0; -+static uint64_t servOutputs = 0; -+static uint64_t servViewers = 0; -+static uint64_t servSeconds = 0; -+static uint64_t servPackSent = 0; -+static uint64_t servPackLoss = 0; -+static uint64_t servPackRetrans = 0; -+// Total time watched for all sessions which are no longer active -+static uint64_t viewSecondsTotal = 0; -+// Mapping of streamName -> summary of stream-wide statistics - static std::map streamStats; - --static void createEmptyStatsIfNeeded(const std::string & strm){ -- if (streamStats.count(strm)){return;} -- streamTotals & sT = streamStats[strm]; -+// If sessId does not exist yet in streamStats, create and init an entry for it -+static void createEmptyStatsIfNeeded(const std::string & sessId){ -+ if (streamStats.count(sessId)){return;} -+ streamTotals & sT = streamStats[sessId]; - sT.upBytes = 0; - sT.downBytes = 0; - sT.inputs = 0; -@@ -136,68 +114,38 @@ static void createEmptyStatsIfNeeded(const std::string & strm){ - sT.packRetrans = 0; - } - -- --static uint64_t servUpBytes = 0; --static uint64_t servDownBytes = 0; --static uint64_t servUpOtherBytes = 0; --static uint64_t servDownOtherBytes = 0; --static uint64_t servInputs = 0; --static uint64_t servOutputs = 0; --static uint64_t servViewers = 0; --static uint64_t servSeconds = 0; --static uint64_t servPackSent = 0; --static uint64_t servPackLoss = 0; --static uint64_t servPackRetrans = 0; -- --Controller::sessIndex::sessIndex(){ -- crc = 0; --} -- --/// Initializes a sessIndex from a statistics object + index, converting binary format IP addresses --/// into strings. This extracts the host, stream name, connector and crc field, ignoring everything --/// else. --Controller::sessIndex::sessIndex(const Comms::Statistics &statComm, size_t id){ -- Socket::hostBytesToStr(statComm.getHost(id).data(), 16, host); -- streamName = statComm.getStream(id); -- connector = statComm.getConnector(id); -- crc = statComm.getCRC(id); -- ID = statComm.getSessId(id); --} -- --std::string Controller::sessIndex::toStr(){ -- std::stringstream s; -- s << ID << "(" << host << " " << crc << " " << streamName << " " << connector << ")"; -- return s.str(); --} -- --bool Controller::sessIndex::operator==(const Controller::sessIndex &b) const{ -- return (host == b.host && crc == b.crc && streamName == b.streamName && connector == b.connector); --} -- --bool Controller::sessIndex::operator!=(const Controller::sessIndex &b) const{ -- return !(*this == b); --} -- --bool Controller::sessIndex::operator>(const Controller::sessIndex &b) const{ -- return host > b.host || -- (host == b.host && -- (crc > b.crc || (crc == b.crc && (streamName > b.streamName || -- (streamName == b.streamName && connector > b.connector))))); --} -- --bool Controller::sessIndex::operator<(const Controller::sessIndex &b) const{ -- return host < b.host || -- (host == b.host && -- (crc < b.crc || (crc == b.crc && (streamName < b.streamName || -- (streamName == b.streamName && connector < b.connector))))); --} -- --bool Controller::sessIndex::operator<=(const Controller::sessIndex &b) const{ -- return !(*this > b); --} -- --bool Controller::sessIndex::operator>=(const Controller::sessIndex &b) const{ -- return !(*this < b); -+/// Convert bandwidth config into memory format -+void Controller::updateBandwidthConfig(){ -+ size_t offset = 0; -+ bwLimit = 128 * 1024 * 1024; // gigabit default limit -+ memset(noBWCountMatches, 0, 1717); -+ if (Storage.isMember("bandwidth")){ -+ if (Storage["bandwidth"].isMember("limit")){bwLimit = Storage["bandwidth"]["limit"].asInt();} -+ if (Storage["bandwidth"].isMember("exceptions")){ -+ jsonForEach(Storage["bandwidth"]["exceptions"], j){ -+ std::string newbins = Socket::getBinForms(j->asStringRef()); -+ if (offset + newbins.size() < 1700){ -+ memcpy(noBWCountMatches + offset, newbins.data(), newbins.size()); -+ offset += newbins.size(); -+ } -+ } -+ } -+ } -+ //Localhost is always excepted from counts -+ { -+ std::string newbins = Socket::getBinForms("::1"); -+ if (offset + newbins.size() < 1700){ -+ memcpy(noBWCountMatches + offset, newbins.data(), newbins.size()); -+ offset += newbins.size(); -+ } -+ } -+ { -+ std::string newbins = Socket::getBinForms("127.0.0.1/8"); -+ if (offset + newbins.size() < 1700){ -+ memcpy(noBWCountMatches + offset, newbins.data(), newbins.size()); -+ offset += newbins.size(); -+ } -+ } - } - - /// This function is ran whenever a stream becomes active. -@@ -211,9 +159,6 @@ void Controller::streamStopped(std::string stream){ - INFO_MSG("Stream %s became inactive", stream.c_str()); - } - --Comms::Statistics statComm; --bool statCommActive = false; -- - /// Invalidates all current sessions for the given streamname - /// Updates the session cache, afterwards. - void Controller::sessions_invalidate(const std::string &streamname){ -@@ -221,18 +166,17 @@ void Controller::sessions_invalidate(const std::string &streamname){ - FAIL_MSG("In shutdown procedure - cannot invalidate sessions."); - return; - } -- unsigned int invalidated = 0; - unsigned int sessCount = 0; -- tthread::lock_guard guard(statsMutex); -- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ -- if (it->first.streamName == streamname){ -+ // Find all matching streams in statComm -+ for (size_t i = 0; i < statComm.recordCount(); i++){ -+ if (statComm.getStatus(i) == COMM_STATUS_INVALID || (statComm.getStatus(i) & COMM_STATUS_DISCONNECT)){continue;} -+ if (statComm.getStream(i) == streamname){ - sessCount++; -- invalidated += it->second.invalidate(); -+ // Re-trigger USER_NEW trigger for this session -+ kill(statComm.getPid(i), SIGUSR1); - } - } -- Controller::writeSessionCache(); -- INFO_MSG("Invalidated %u connections in %u sessions for stream %s", invalidated, sessCount, -- streamname.c_str()); -+ INFO_MSG("Invalidated %u session(s) for stream %s", sessCount, streamname.c_str()); - } - - /// Shuts down all current sessions for the given streamname -@@ -256,18 +200,8 @@ void Controller::sessId_shutdown(const std::string &sessId){ - FAIL_MSG("In controller shutdown procedure - cannot shutdown sessions."); - return; - } -- unsigned int murdered = 0; -- unsigned int sessCount = 0; -- tthread::lock_guard guard(statsMutex); -- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ -- if (it->first.ID == sessId){ -- sessCount++; -- murdered += it->second.kill(); -- break; -- } -- } -- Controller::writeSessionCache(); -- INFO_MSG("Shut down %u connections in %u session(s) for ID %s", murdered, sessCount, sessId.c_str()); -+ killConnections(sessId); -+ INFO_MSG("Shut down session with session ID %s", sessId.c_str()); - } - - /// Tags the given session -@@ -277,8 +211,8 @@ void Controller::sessId_tag(const std::string &sessId, const std::string &tag){ - return; - } - tthread::lock_guard guard(statsMutex); -- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ -- if (it->first.ID == sessId){ -+ for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ -+ if (it->first == sessId){ - it->second.tags.insert(tag); - return; - } -@@ -295,17 +229,15 @@ void Controller::tag_shutdown(const std::string &tag){ - FAIL_MSG("In controller shutdown procedure - cannot shutdown sessions."); - return; - } -- unsigned int murdered = 0; - unsigned int sessCount = 0; - tthread::lock_guard guard(statsMutex); -- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ -+ for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ - if (it->second.tags.count(tag)){ - sessCount++; -- murdered += it->second.kill(); -+ killConnections(it->first); - } - } -- Controller::writeSessionCache(); -- INFO_MSG("Shut down %u connections in %u session(s) for tag %s", murdered, sessCount, tag.c_str()); -+ INFO_MSG("Shut down %u session(s) for tag %s", sessCount, tag.c_str()); - } - - /// Shuts down all current sessions for the given streamname -@@ -315,48 +247,23 @@ void Controller::sessions_shutdown(const std::string &streamname, const std::str - FAIL_MSG("In controller shutdown procedure - cannot shutdown sessions."); - return; - } -- unsigned int murdered = 0; - unsigned int sessCount = 0; - tthread::lock_guard guard(statsMutex); -- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ -- if ((!streamname.size() || it->first.streamName == streamname) && -- (!protocol.size() || it->first.connector == protocol)){ -+ // Find all matching streams in statComm and get their sessId -+ for (size_t i = 0; i < statComm.recordCount(); i++){ -+ if (statComm.getStatus(i) == COMM_STATUS_INVALID || (statComm.getStatus(i) & COMM_STATUS_DISCONNECT)){continue;} -+ if ((!streamname.size() || statComm.getStream(i) == streamname) && -+ (!protocol.size() || statComm.hasConnector(i, protocol))){ -+ uint32_t pid = statComm.getPid(i); - sessCount++; -- murdered += it->second.kill(); -- } -- } -- Controller::writeSessionCache(); -- INFO_MSG("Shut down %u connections in %u sessions for stream %s/%s", murdered, sessCount, -- streamname.c_str(), protocol.c_str()); --} -- --/// Writes the session cache to shared memory. --/// Assumes the config mutex, stats mutex and session cache semaphore are already locked. --/// Does nothing if the session cache could not be initialized on the first try --/// Does no error checking after first open attempt (fails silently)! --void Controller::writeSessionCache(){ -- uint32_t shmOffset = 0; -- if (shmSessions && shmSessions->mapped){ -- if (cacheLock){cacheLock->wait(16);} -- if (sessions.size()){ -- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ -- if (it->second.hasData()){ -- // store an entry in the shmSessions page, if it fits -- if (it->second.sync > 2 && shmOffset + SHM_SESSIONS_ITEM < SHM_SESSIONS_SIZE){ -- *((uint32_t *)(shmSessions->mapped + shmOffset)) = it->first.crc; -- strncpy(shmSessions->mapped + shmOffset + 4, it->first.streamName.c_str(), 100); -- strncpy(shmSessions->mapped + shmOffset + 104, it->first.connector.c_str(), 20); -- strncpy(shmSessions->mapped + shmOffset + 124, it->first.host.c_str(), 40); -- shmSessions->mapped[shmOffset + 164] = it->second.sync; -- shmOffset += SHM_SESSIONS_ITEM; -- } -- } -+ if (pid > 1){ -+ Util::Procs::Stop(pid); -+ INFO_MSG("Killing PID %" PRIu32, pid); - } - } -- // set a final shmSessions entry to all zeroes -- memset(shmSessions->mapped + shmOffset, 0, SHM_SESSIONS_ITEM); -- if (cacheLock){cacheLock->post(16);} - } -+ INFO_MSG("Shut down %u sessions for stream %s/%s", sessCount, -+ streamname.c_str(), protocol.c_str()); - } - - /// This function runs as a thread and roughly once per second retrieves -@@ -366,14 +273,6 @@ void Controller::SharedMemStats(void *config){ - HIGH_MSG("Starting stats thread"); - statComm.reload(true); - statCommActive = true; -- shmSessions = new IPC::sharedPage(SHM_SESSIONS, SHM_SESSIONS_SIZE, false, false); -- if (!shmSessions || !shmSessions->mapped){ -- if (shmSessions){delete shmSessions;} -- shmSessions = new IPC::sharedPage(SHM_SESSIONS, SHM_SESSIONS_SIZE, true); -- } -- cacheLock = new IPC::semaphore(SEM_SESSCACHE, O_CREAT | O_RDWR, ACCESSPERMS, 16); -- cacheLock->unlink(); -- cacheLock->open(SEM_SESSCACHE, O_CREAT | O_RDWR, ACCESSPERMS, 16); - std::set inactiveStreams; - Controller::initState(); - bool shiftWrites = true; -@@ -401,7 +300,6 @@ void Controller::SharedMemStats(void *config){ - } - } - { -- - tthread::lock_guard guard(Controller::configMutex); - tthread::lock_guard guard2(statsMutex); - // parse current users -@@ -429,29 +327,47 @@ void Controller::SharedMemStats(void *config){ - it->second.packRetrans = 0; - } - } -- // wipe old statistics -+ unsigned int tOut = Util::bootSecs() - STATS_DELAY; -+ unsigned int tIn = Util::bootSecs() - STATS_INPUT_DELAY; -+ if (streamStats.size()){ -+ for (std::map::iterator it = streamStats.begin(); -+ it != streamStats.end(); ++it){ -+ it->second.currViews = 0; -+ it->second.currIns = 0; -+ it->second.currOuts = 0; -+ } -+ } -+ // wipe old statistics and set session type counters - if (sessions.size()){ -- std::list mustWipe; -+ std::list mustWipe; - uint64_t cutOffPoint = Util::bootSecs() - STAT_CUTOFF; -- uint64_t disconnectPointIn = Util::bootSecs() - STATS_INPUT_DELAY; -- uint64_t disconnectPointOut = Util::bootSecs() - STATS_DELAY; -- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ -- uint64_t dPoint = it->second.getSessType() == SESS_INPUT ? disconnectPointIn : disconnectPointOut; -- if (it->second.sync == 100){ -- // Denied entries are connection-entry-wiped as soon as they become boring -- it->second.wipeOld(dPoint); -- }else{ -- // Normal entries are summarized after STAT_CUTOFF seconds -- it->second.wipeOld(cutOffPoint); -- } -+ for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ - // This part handles ending sessions, keeping them in cache for now -- if (it->second.isTracked() && !it->second.isConnected() && it->second.getEnd() < dPoint){ -- it->second.dropSession(it->first); -- } -- // This part handles wiping from the session cache -- if (!it->second.hasData()){ -- it->second.dropSession(it->first); // End the session, just in case it wasn't yet -+ if (it->second.getEnd() < cutOffPoint && it->second.newestDataPoint() < cutOffPoint){ -+ viewSecondsTotal += it->second.getConnTime(); - mustWipe.push_back(it->first); -+ // Don't count this session as a viewer -+ continue; -+ } -+ // Recount input, output and viewer type sessions -+ switch (it->second.getSessType()){ -+ case SESS_UNSET: break; -+ case SESS_VIEWER: -+ if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){ -+ streamStats[it->first].currViews++; -+ } -+ servSeconds += it->second.getConnTime(); -+ break; -+ case SESS_INPUT: -+ if (it->second.hasDataFor(tIn) && it->second.isViewerOn(tIn)){ -+ streamStats[it->first].currIns++; -+ } -+ break; -+ case SESS_OUTPUT: -+ if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){ -+ streamStats[it->first].currOuts++; -+ } -+ break; - } - } - while (mustWipe.size()){ -@@ -513,7 +429,6 @@ void Controller::SharedMemStats(void *config){ - shiftWrites = true; - } - /*LTS-START*/ -- Controller::writeSessionCache(); - Controller::checkServerLimits(); - /*LTS-END*/ - } -@@ -523,7 +438,6 @@ void Controller::SharedMemStats(void *config){ - HIGH_MSG("Stopping stats thread"); - if (Util::Config::is_restarting){ - statComm.setMaster(false); -- shmSessions->master = false; - }else{/*LTS-START*/ - if (Controller::killOnExit){ - WARN_MSG("Killing all connected clients to force full shutdown"); -@@ -532,10 +446,6 @@ void Controller::SharedMemStats(void *config){ - /*LTS-END*/ - } - Controller::deinitState(Util::Config::is_restarting); -- delete shmSessions; -- shmSessions = 0; -- delete cacheLock; -- cacheLock = 0; - } - - /// Gets a complete list of all streams currently in active state, with optional prefix matching -@@ -559,97 +469,53 @@ std::set Controller::getActiveStreams(const std::string &prefix){ - return ret; - } - --/// Forces a re-sync of the session --/// Assumes the session cache will be updated separately - may not work correctly if this is forgotten! --uint32_t Controller::statSession::invalidate(){ -- uint32_t ret = 0; -- sync = 1; -- if (curConns.size() && statCommActive){ -- for (std::map::iterator jt = curConns.begin(); jt != curConns.end(); ++jt){ -- if (statComm.getStatus(jt->first) != COMM_STATUS_INVALID){ -- statComm.setSync(2, jt->first); -- ret++; -- } -- } -- } -- return ret; --} -- --/// Kills all active connections, sets the session state to denied (sync=100). -+/// Kills all connection of a given session - /// Assumes the session cache will be updated separately - may not work correctly if this is forgotten! --uint32_t Controller::statSession::kill(){ -- uint32_t ret = 0; -- sync = 100; -- if (curConns.size() && statCommActive){ -- for (std::map::iterator jt = curConns.begin(); jt != curConns.end(); ++jt){ -- if (statComm.getStatus(jt->first) != COMM_STATUS_INVALID){ -- statComm.setSync(100, jt->first); -- uint32_t pid = statComm.getPid(jt->first); -+void Controller::killConnections(std::string sessId){ -+ if (statCommActive){ -+ // Find a matching stream in statComm with a matching sessID and kill it -+ for (size_t i = 0; i < statComm.recordCount(); i++){ -+ if (statComm.getStatus(i) == COMM_STATUS_INVALID || (statComm.getStatus(i) & COMM_STATUS_DISCONNECT)){continue;} -+ if (statComm.getSessId(i) == sessId){ -+ uint32_t pid = statComm.getPid(i); - if (pid > 1){ - Util::Procs::Stop(pid); - INFO_MSG("Killing PID %" PRIu32, pid); - } -- ret++; - } - } - } -- return ret; - } - - /// Updates the given active connection with new stats data. --void Controller::statSession::update(uint64_t index, Comms::Statistics &statComm){ -- std::string myHost; -- Socket::hostBytesToStr(statComm.getHost(index).data(), 16, myHost); -- std::string myStream = statComm.getStream(index); -- std::string myConnector = statComm.getConnector(index); -- // update the sync byte: 0 = requesting fill, 2 = requesting refill, 1 = needs checking, > 2 = -- // state known (100=denied, 10=accepted) -- if (!statComm.getSync(index)){ -- sessIndex tmpidx(statComm, index); -- // if we have a maximum connection count per IP, enforce it -- if (maxConnsPerIP && !statComm.getSync(index)){ -- unsigned int currConns = 1; -- long long shortly = Util::bootSecs(); -- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ -- -- if (&it->second != this && it->first.host == myHost && -- (it->second.hasDataFor(shortly - STATS_DELAY) || it->second.hasDataFor(shortly) || -- it->second.hasDataFor(shortly - 1) || it->second.hasDataFor(shortly - 2) || -- it->second.hasDataFor(shortly - 3) || it->second.hasDataFor(shortly - 4) || -- it->second.hasDataFor(shortly - 5)) && -- ++currConns > maxConnsPerIP){ -- break; -- } -- } -- if (currConns > maxConnsPerIP){ -- WARN_MSG("Disconnecting session from %s: exceeds max connection count of %u", myHost.c_str(), maxConnsPerIP); -- statComm.setSync(100, index); -- } -- } -- if (statComm.getSync(index) != 100){ -- // only set the sync if this is the first connection in the list -- // we also catch the case that there are no connections, which is an error-state -- if (!sessions[tmpidx].curConns.size() || sessions[tmpidx].curConns.begin()->first == index){ -- MEDIUM_MSG("Requesting sync to %u for %s, %s, %s, %" PRIu32, sync, myStream.c_str(), -- myConnector.c_str(), myHost.c_str(), statComm.getCRC(index) & 0xFFFFFFFFu); -- statComm.setSync(sync, index); -- } -- // and, always set the sync if it is > 2 -- if (sync > 2){ -- MEDIUM_MSG("Setting sync to %u for %s, %s, %s, %" PRIu32, sync, myStream.c_str(), -- myConnector.c_str(), myHost.c_str(), statComm.getCRC(index) & 0xFFFFFFFFu); -- statComm.setSync(sync, index); -- } -+void Controller::statSession::update(uint64_t index, Comms::Sessions &statComm){ -+ if (host == ""){ -+ Socket::hostBytesToStr(statComm.getHost(index).data(), 16, host); -+ } -+ if (streamName == ""){ -+ streamName = statComm.getStream(index); -+ } -+ if (curConnector == ""){ -+ curConnector = statComm.getConnector(index); -+ } -+ if (sessId == ""){ -+ sessId = statComm.getSessId(index); -+ } -+ // Export tags to session -+ if (tags.size()){ -+ std::stringstream tagStream; -+ for (std::set::iterator it = tags.begin(); it != tags.end(); ++it){ -+ tagStream << "[" << *it << "]"; - } -- }else{ -- if (sync < 2 && statComm.getSync(index) > 2){sync = statComm.getSync(index);} -+ statComm.setTags(tagStream.str(), index); - } -+ - long long prevDown = getDown(); - long long prevUp = getUp(); - uint64_t prevPktSent = getPktCount(); - uint64_t prevPktLost = getPktLost(); - uint64_t prevPktRetrans = getPktRetransmit(); -- curConns[index].update(statComm, index); -+ curData.update(statComm, index); - // store timestamp of first received data, if older - if (firstSec > statComm.getNow(index)){firstSec = statComm.getNow(index);} - uint64_t secIncr = 0; -@@ -671,7 +537,7 @@ void Controller::statSession::update(uint64_t index, Comms::Statistics &statComm - uint64_t currPktRetrans = getPktRetransmit(); - if (currUp - prevUp < 0 || currDown - prevDown < 0){ - INFO_MSG("Negative data usage! %lldu/%lldd (u%lld->%lld) in %s over %s, #%" PRIu64, currUp - prevUp, -- currDown - prevDown, prevUp, currUp, myStream.c_str(), myConnector.c_str(), index); -+ currDown - prevDown, prevUp, currUp, streamName.c_str(), curConnector.c_str(), index); - }else{ - if (!noBWCount){ - size_t bwMatchOffset = 0; -@@ -701,56 +567,39 @@ void Controller::statSession::update(uint64_t index, Comms::Statistics &statComm - servPackRetrans += currPktRetrans - prevPktRetrans; - } - } -- if (currDown + currUp >= COUNTABLE_BYTES){ -- if (sessionType == SESS_UNSET){ -- if (myConnector.size() >= 5 && myConnector.substr(0, 5) == "INPUT"){ -- ++servInputs; -- createEmptyStatsIfNeeded(myStream); -- streamStats[myStream].inputs++; -- streamStats[myStream].currIns++; -- sessionType = SESS_INPUT; -- }else if (myConnector.size() >= 6 && myConnector.substr(0, 6) == "OUTPUT"){ -- ++servOutputs; -- createEmptyStatsIfNeeded(myStream); -- streamStats[myStream].outputs++; -- streamStats[myStream].currOuts++; -- sessionType = SESS_OUTPUT; -- }else{ -- ++servViewers; -- createEmptyStatsIfNeeded(myStream); -- streamStats[myStream].viewers++; -- streamStats[myStream].currViews++; -- sessionType = SESS_VIEWER; -- } -+ if (sessionType == SESS_UNSET){ -+ if (curConnector.size() >= 5 && curConnector.substr(0, 5) == "INPUT"){ -+ ++servInputs; -+ createEmptyStatsIfNeeded(streamName); -+ streamStats[streamName].inputs++; -+ streamStats[streamName].currIns++; -+ sessionType = SESS_INPUT; -+ }else if (curConnector.size() >= 6 && curConnector.substr(0, 6) == "OUTPUT"){ -+ ++servOutputs; -+ createEmptyStatsIfNeeded(streamName); -+ streamStats[streamName].outputs++; -+ streamStats[streamName].currOuts++; -+ sessionType = SESS_OUTPUT; -+ }else{ -+ ++servViewers; -+ createEmptyStatsIfNeeded(streamName); -+ streamStats[streamName].viewers++; -+ streamStats[streamName].currViews++; -+ sessionType = SESS_VIEWER; - } -- // If previous < COUNTABLE_BYTES, we haven't counted any data so far. -- // We need to count all the data in that case, otherwise we only count the difference. -- if (noBWCount != 2){ //only count connections that are countable -- if (prevUp + prevDown < COUNTABLE_BYTES){ -- if (!myStream.size() || myStream[0] == 0){ -- if (streamStats.count(myStream)){streamStats.erase(myStream);} -- }else{ -- createEmptyStatsIfNeeded(myStream); -- streamStats[myStream].upBytes += currUp; -- streamStats[myStream].downBytes += currDown; -- streamStats[myStream].packSent += currPktSent; -- streamStats[myStream].packLoss += currPktLost; -- streamStats[myStream].packRetrans += currPktRetrans; -- if (sessionType == SESS_VIEWER){streamStats[myStream].viewSeconds += lastSec - firstSec;} -- } -- }else{ -- if (!myStream.size() || myStream[0] == 0){ -- if (streamStats.count(myStream)){streamStats.erase(myStream);} -- }else{ -- createEmptyStatsIfNeeded(myStream); -- streamStats[myStream].upBytes += currUp - prevUp; -- streamStats[myStream].downBytes += currDown - prevDown; -- streamStats[myStream].packSent += currPktSent - prevPktSent; -- streamStats[myStream].packLoss += currPktLost - prevPktLost; -- streamStats[myStream].packRetrans += currPktRetrans - prevPktRetrans; -- if (sessionType == SESS_VIEWER){streamStats[myStream].viewSeconds += secIncr;} -- } -- } -+ } -+ // Only count connections that are countable -+ if (noBWCount != 2){ -+ if (!streamName.size() || streamName[0] == 0){ -+ if (streamStats.count(streamName)){streamStats.erase(streamName);} -+ }else{ -+ createEmptyStatsIfNeeded(streamName); -+ streamStats[streamName].upBytes += currUp - prevUp; -+ streamStats[streamName].downBytes += currDown - prevDown; -+ streamStats[streamName].packSent += currPktSent - prevPktSent; -+ streamStats[streamName].packLoss += currPktLost - prevPktLost; -+ streamStats[streamName].packRetrans += currPktRetrans - prevPktRetrans; -+ if (sessionType == SESS_VIEWER){streamStats[streamName].viewSeconds += secIncr;} - } - } - } -@@ -759,53 +608,19 @@ Controller::sessType Controller::statSession::getSessType(){ - return sessionType; - } - --/// Archives connection log entries older than the given cutOff point. --void Controller::statSession::wipeOld(uint64_t cutOff){ -- if (firstSec > cutOff){return;} -- firstSec = 0xFFFFFFFFFFFFFFFFull; -- if (oldConns.size()){ -- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ -- while (it->log.size() && it->log.begin()->first < cutOff){ -- if (it->log.size() == 1){ -- wipedDown += it->log.begin()->second.down; -- wipedUp += it->log.begin()->second.up; -- wipedPktCount += it->log.begin()->second.pktCount; -- wipedPktLost += it->log.begin()->second.pktLost; -- wipedPktRetransmit += it->log.begin()->second.pktRetransmit; -- } -- it->log.erase(it->log.begin()); -- } -- if (it->log.size()){ -- if (firstSec > it->log.begin()->first){firstSec = it->log.begin()->first;} -- } -- } -- while (oldConns.size() && !oldConns.begin()->log.size()){oldConns.pop_front();} -- } -- if (curConns.size()){ -- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ -- while (it->second.log.size() > 1 && it->second.log.begin()->first < cutOff){ -- it->second.log.erase(it->second.log.begin()); -- } -- if (it->second.log.size()){ -- if (firstSec > it->second.log.begin()->first){firstSec = it->second.log.begin()->first;} -- } -- } -- } --} -- --void Controller::statSession::dropSession(const Controller::sessIndex &index){ -- if (!tracked || curConns.size()){return;} -+Controller::statSession::~statSession(){ -+ if (!tracked){return;} - switch (sessionType){ -- case SESS_INPUT: -- if (streamStats.count(index.streamName) && streamStats[index.streamName].currIns){streamStats[index.streamName].currIns--;} -- break; -- case SESS_OUTPUT: -- if (streamStats.count(index.streamName) && streamStats[index.streamName].currOuts){streamStats[index.streamName].currOuts--;} -- break; -- case SESS_VIEWER: -- if (streamStats.count(index.streamName) && streamStats[index.streamName].currViews){streamStats[index.streamName].currViews--;} -- break; -- default: break; -+ case SESS_INPUT: -+ if (streamStats.count(streamName) && streamStats[streamName].currIns){streamStats[streamName].currIns--;} -+ break; -+ case SESS_OUTPUT: -+ if (streamStats.count(streamName) && streamStats[streamName].currOuts){streamStats[streamName].currOuts--;} -+ break; -+ case SESS_VIEWER: -+ if (streamStats.count(streamName) && streamStats[streamName].currViews){streamStats[streamName].currViews--;} -+ break; -+ default: break; - } - uint64_t duration = lastSec - firstActive; - if (duration < 1){duration = 1;} -@@ -815,13 +630,13 @@ void Controller::statSession::dropSession(const Controller::sessIndex &index){ - tagStream << "[" << *it << "]"; - } - } -- Controller::logAccess(index.ID, index.streamName, index.connector, index.host, duration, getUp(), -+ Controller::logAccess(sessId, streamName, curConnector, host, duration, getUp(), - getDown(), tagStream.str()); - if (Controller::accesslog.size()){ - if (Controller::accesslog == "LOG"){ - std::stringstream accessStr; -- accessStr << "Session <" << index.ID << "> " << index.streamName << " (" << index.connector -- << ") from " << index.host << " ended after " << duration << "s, avg " -+ accessStr << "Session <" << sessId << "> " << streamName << " (" << curConnector -+ << ") from " << host << " ended after " << duration << "s, avg " - << getUp() / duration / 1024 << "KB/s up " << getDown() / duration / 1024 << "KB/s down."; - if (tags.size()){accessStr << " Tags: " << tagStream.str();} - Controller::Log("ACCS", accessStr.str()); -@@ -845,8 +660,8 @@ void Controller::statSession::dropSession(const Controller::sessIndex &index){ - time(&rawtime); - timeinfo = localtime_r(&rawtime, &tmptime); - strftime(buffer, 100, "%F %H:%M:%S", timeinfo); -- accLogFile << buffer << ", " << index.ID << ", " << index.streamName << ", " -- << index.connector << ", " << index.host << ", " << duration << ", " -+ accLogFile << buffer << ", " << sessId << ", " << streamName << ", " -+ << curConnector << ", " << host << ", " << duration << ", " - << getUp() / duration / 1024 << ", " << getDown() / duration / 1024 << ", "; - if (tags.size()){accLogFile << tagStream.str();} - accLogFile << std::endl; -@@ -857,77 +672,21 @@ void Controller::statSession::dropSession(const Controller::sessIndex &index){ - firstActive = 0; - firstSec = 0xFFFFFFFFFFFFFFFFull; - lastSec = 0; -- wipedUp = 0; -- wipedDown = 0; -- wipedPktCount = 0; -- wipedPktLost = 0; -- wipedPktRetransmit = 0; -- oldConns.clear(); - sessionType = SESS_UNSET; - } - --/// Archives the given connection. --void Controller::statSession::finish(uint64_t index){ -- oldConns.push_back(curConns[index]); -- curConns.erase(index); --} -- - /// Constructs an empty session - Controller::statSession::statSession(){ - firstActive = 0; - tracked = false; - firstSec = 0xFFFFFFFFFFFFFFFFull; - lastSec = 0; -- sync = 1; -- wipedUp = 0; -- wipedDown = 0; -- wipedPktCount = 0; -- wipedPktLost = 0; -- wipedPktRetransmit = 0; - sessionType = SESS_UNSET; - noBWCount = 0; --} -- --/// Moves the given connection to the given session --void Controller::statSession::switchOverTo(statSession &newSess, uint64_t index){ -- // add to the given session first -- newSess.curConns[index] = curConns[index]; -- // if this connection has data, update firstSec/lastSec if needed -- if (curConns[index].log.size()){ -- if (newSess.firstSec > curConns[index].log.begin()->first){ -- newSess.firstSec = curConns[index].log.begin()->first; -- } -- if (newSess.lastSec < curConns[index].log.rbegin()->first){ -- newSess.lastSec = curConns[index].log.rbegin()->first; -- } -- } -- // remove from current session -- curConns.erase(index); -- // if there was any data, recalculate this session's firstSec and lastSec. -- if (newSess.curConns[index].log.size()){ -- firstSec = 0xFFFFFFFFFFFFFFFFull; -- lastSec = 0; -- if (oldConns.size()){ -- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ -- if (it->log.size()){ -- if (firstSec > it->log.begin()->first){firstSec = it->log.begin()->first;} -- if (lastSec < it->log.rbegin()->first){lastSec = it->log.rbegin()->first;} -- } -- } -- } -- if (curConns.size()){ -- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ -- if (it->second.log.size()){ -- if (firstSec > it->second.log.begin()->first){ -- firstSec = it->second.log.begin()->first; -- } -- if (lastSec < it->second.log.rbegin()->first){ -- lastSec = it->second.log.rbegin()->first; -- } -- } -- } -- } -- } -+ streamName = ""; -+ host = ""; -+ curConnector = ""; -+ sessId = ""; - } - - /// Returns the first measured timestamp in this session. -@@ -944,39 +703,34 @@ uint64_t Controller::statSession::getEnd(){ - bool Controller::statSession::hasDataFor(uint64_t t){ - if (lastSec < t){return false;} - if (firstSec > t){return false;} -- if (oldConns.size()){ -- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ -- if (it->hasDataFor(t)){return true;} -- } -- } -- if (curConns.size()){ -- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ -- if (it->second.hasDataFor(t)){return true;} -- } -- } -- return false; --} -- --/// Returns true if there is any data for this session. --bool Controller::statSession::hasData(){ -- if (!firstSec && !lastSec){return false;} -- if (curConns.size()){return true;} -- if (oldConns.size()){ -- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ -- if (it->log.size()){return true;} -- } -- } -+ if (curData.hasDataFor(t)){return true;} - return false; - } - - /// Returns true if this session should count as a viewer on the given timestamp. - bool Controller::statSession::isViewerOn(uint64_t t){ -- return getUp(t) + getDown(t) > COUNTABLE_BYTES; -+ return getUp(t) + getDown(t); -+} -+ -+std::string Controller::statSession::getStreamName(){ -+ return streamName; -+} -+ -+std::string Controller::statSession::getHost(){ -+ return host; -+} -+ -+std::string Controller::statSession::getSessId(){ -+ return sessId; -+} -+ -+std::string Controller::statSession::getCurrentProtocols(){ -+ return curConnector; - } - - /// Returns true if this session should be considered connected --bool Controller::statSession::isConnected(){ -- return curConns.size(); -+uint64_t Controller::statSession::newestDataPoint(){ -+ return lastSec; - } - - /// Returns true if this session has started (tracked == true) but not yet ended (log entry written) -@@ -986,188 +740,103 @@ bool Controller::statSession::isTracked(){ - - /// Returns the cumulative connected time for this session at timestamp t. - uint64_t Controller::statSession::getConnTime(uint64_t t){ -- uint64_t retVal = 0; -- if (oldConns.size()){ -- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ -- if (it->hasDataFor(t)){retVal += it->getDataFor(t).time;} -- } -+ if (curData.hasDataFor(t)){ -+ return curData.getDataFor(t).time; - } -- if (curConns.size()){ -- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ -- if (it->second.hasDataFor(t)){retVal += it->second.getDataFor(t).time;} -- } -+ return 0; -+} -+ -+/// Returns the cumulative connected time for this session. -+uint64_t Controller::statSession::getConnTime(){ -+ if (curData.log.size()){ -+ return curData.log.rbegin()->second.time; - } -- return retVal; -+ return 0; - } - - /// Returns the last requested media timestamp for this session at timestamp t. - uint64_t Controller::statSession::getLastSecond(uint64_t t){ -- if (curConns.size()){ -- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ -- if (it->second.hasDataFor(t)){return it->second.getDataFor(t).lastSecond;} -- } -- } -- if (oldConns.size()){ -- for (std::deque::reverse_iterator it = oldConns.rbegin(); it != oldConns.rend(); ++it){ -- if (it->hasDataFor(t)){return it->getDataFor(t).lastSecond;} -- } -+ if (curData.hasDataFor(t)){ -+ return curData.getDataFor(t).lastSecond; - } - return 0; - } - - /// Returns the cumulative downloaded bytes for this session at timestamp t. - uint64_t Controller::statSession::getDown(uint64_t t){ -- uint64_t retVal = wipedDown; -- if (oldConns.size()){ -- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ -- if (it->hasDataFor(t)){retVal += it->getDataFor(t).down;} -- } -+ if (curData.hasDataFor(t)){ -+ return curData.getDataFor(t).down; - } -- if (curConns.size()){ -- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ -- if (it->second.hasDataFor(t)){retVal += it->second.getDataFor(t).down;} -- } -- } -- return retVal; -+ return 0; - } - - /// Returns the cumulative uploaded bytes for this session at timestamp t. - uint64_t Controller::statSession::getUp(uint64_t t){ -- uint64_t retVal = wipedUp; -- if (oldConns.size()){ -- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ -- if (it->hasDataFor(t)){retVal += it->getDataFor(t).up;} -- } -- } -- if (curConns.size()){ -- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ -- if (it->second.hasDataFor(t)){retVal += it->second.getDataFor(t).up;} -- } -+ if (curData.hasDataFor(t)){ -+ return curData.getDataFor(t).up; - } -- return retVal; -+ return 0; - } - - /// Returns the cumulative downloaded bytes for this session at timestamp t. - uint64_t Controller::statSession::getDown(){ -- uint64_t retVal = wipedDown; -- if (oldConns.size()){ -- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ -- if (it->log.size()){retVal += it->log.rbegin()->second.down;} -- } -+ if (curData.log.size()){ -+ return curData.log.rbegin()->second.down; - } -- if (curConns.size()){ -- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ -- if (it->second.log.size()){retVal += it->second.log.rbegin()->second.down;} -- } -- } -- return retVal; -+ return 0; - } - - /// Returns the cumulative uploaded bytes for this session at timestamp t. - uint64_t Controller::statSession::getUp(){ -- uint64_t retVal = wipedUp; -- if (oldConns.size()){ -- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ -- if (it->log.size()){retVal += it->log.rbegin()->second.up;} -- } -- } -- if (curConns.size()){ -- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ -- if (it->second.log.size()){retVal += it->second.log.rbegin()->second.up;} -- } -+ if (curData.log.size()){ -+ return curData.log.rbegin()->second.up; - } -- return retVal; -+ return 0; - } - - uint64_t Controller::statSession::getPktCount(uint64_t t){ -- uint64_t retVal = wipedPktCount; -- if (oldConns.size()){ -- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ -- if (it->hasDataFor(t)){retVal += it->getDataFor(t).pktCount;} -- } -+ if (curData.hasDataFor(t)){ -+ return curData.getDataFor(t).pktCount; - } -- if (curConns.size()){ -- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ -- if (it->second.hasDataFor(t)){retVal += it->second.getDataFor(t).pktCount;} -- } -- } -- return retVal; -+ return 0; - } - - /// Returns the cumulative uploaded bytes for this session at timestamp t. - uint64_t Controller::statSession::getPktCount(){ -- uint64_t retVal = wipedPktCount; -- if (oldConns.size()){ -- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ -- if (it->log.size()){retVal += it->log.rbegin()->second.pktCount;} -- } -- } -- if (curConns.size()){ -- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ -- if (it->second.log.size()){retVal += it->second.log.rbegin()->second.pktCount;} -- } -+ if (curData.log.size()){ -+ return curData.log.rbegin()->second.pktCount; - } -- return retVal; -+ return 0; - } -+ - uint64_t Controller::statSession::getPktLost(uint64_t t){ -- uint64_t retVal = wipedPktLost; -- if (oldConns.size()){ -- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ -- if (it->hasDataFor(t)){retVal += it->getDataFor(t).pktLost;} -- } -+ if (curData.hasDataFor(t)){ -+ return curData.getDataFor(t).pktLost; - } -- if (curConns.size()){ -- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ -- if (it->second.hasDataFor(t)){retVal += it->second.getDataFor(t).pktLost;} -- } -- } -- return retVal; -+ return 0; - } - - /// Returns the cumulative uploaded bytes for this session at timestamp t. - uint64_t Controller::statSession::getPktLost(){ -- uint64_t retVal = wipedPktLost; -- if (oldConns.size()){ -- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ -- if (it->log.size()){retVal += it->log.rbegin()->second.pktLost;} -- } -- } -- if (curConns.size()){ -- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ -- if (it->second.log.size()){retVal += it->second.log.rbegin()->second.pktLost;} -- } -+ if (curData.log.size()){ -+ return curData.log.rbegin()->second.pktLost; - } -- return retVal; -+ return 0; - } -+ - uint64_t Controller::statSession::getPktRetransmit(uint64_t t){ -- uint64_t retVal = wipedPktRetransmit; -- if (oldConns.size()){ -- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ -- if (it->hasDataFor(t)){retVal += it->getDataFor(t).pktRetransmit;} -- } -+ if (curData.hasDataFor(t)){ -+ return curData.getDataFor(t).pktRetransmit; - } -- if (curConns.size()){ -- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ -- if (it->second.hasDataFor(t)){retVal += it->second.getDataFor(t).pktRetransmit;} -- } -- } -- return retVal; -+ return 0; - } - - /// Returns the cumulative uploaded bytes for this session at timestamp t. - uint64_t Controller::statSession::getPktRetransmit(){ -- uint64_t retVal = wipedPktRetransmit; -- if (oldConns.size()){ -- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ -- if (it->log.size()){retVal += it->log.rbegin()->second.pktRetransmit;} -- } -- } -- if (curConns.size()){ -- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ -- if (it->second.log.size()){retVal += it->second.log.rbegin()->second.pktRetransmit;} -- } -+ if (curData.log.size()){ -+ return curData.log.rbegin()->second.pktRetransmit; - } -- return retVal; -+ return 0; - } - - /// Returns the cumulative downloaded bytes per second for this session at timestamp t. -@@ -1207,6 +876,7 @@ Controller::statLog &Controller::statStorage::getDataFor(unsigned long long t){ - empty.pktCount = 0; - empty.pktLost = 0; - empty.pktRetransmit = 0; -+ empty.connectors = ""; - return empty; - } - std::map::iterator it = log.upper_bound(t); -@@ -1216,7 +886,7 @@ Controller::statLog &Controller::statStorage::getDataFor(unsigned long long t){ - - /// This function is called by parseStatistics. - /// It updates the internally saved statistics data. --void Controller::statStorage::update(Comms::Statistics &statComm, size_t index){ -+void Controller::statStorage::update(Comms::Sessions &statComm, size_t index){ - statLog tmp; - tmp.time = statComm.getTime(index); - tmp.lastSecond = statComm.getLastSecond(index); -@@ -1225,56 +895,58 @@ void Controller::statStorage::update(Comms::Statistics &statComm, size_t index){ - tmp.pktCount = statComm.getPacketCount(index); - tmp.pktLost = statComm.getPacketLostCount(index); - tmp.pktRetransmit = statComm.getPacketRetransmitCount(index); -+ tmp.connectors = statComm.getConnector(index); - log[statComm.getNow(index)] = tmp; -- // wipe data older than approx. STAT_CUTOFF seconds -- /// \todo Remove least interesting data first. -- if (log.size() > STAT_CUTOFF){log.erase(log.begin());} -+ // wipe data older than STAT_CUTOFF seconds -+ while (log.size() && log.begin()->first < Util::bootSecs() - STAT_CUTOFF){log.erase(log.begin());} - } - - void Controller::statLeadIn(){ - statDropoff = Util::bootSecs() - 3; - } --void Controller::statOnActive(size_t id){ -- // calculate the current session index, store as idx. -- sessIndex idx(statComm, id); - -+void Controller::statOnActive(size_t id){ - if (statComm.getNow(id) >= statDropoff){ -- // if the connection was already indexed and it has changed, move it -- if (connToSession.count(id) && connToSession[id] != idx){ -- if (sessions[connToSession[id]].getSessType() != SESS_UNSET){ -- INFO_MSG("Switching connection %zu from active session %s over to %s", id, -- connToSession[id].toStr().c_str(), idx.toStr().c_str()); -- }else{ -- INFO_MSG("Switching connection %zu from inactive session %s over to %s", id, -- connToSession[id].toStr().c_str(), idx.toStr().c_str()); -- } -- sessions[connToSession[id]].switchOverTo(sessions[idx], id); -- // Destroy this session without calling dropSession, because it was merged into another. What session? We never made it. Stop asking hard questions. Go, shoo. *sprays water* -- if (!sessions[connToSession[id]].hasData()){sessions.erase(connToSession[id]);} -- } -- if (!connToSession.count(id)){ -- INSANE_MSG("New connection: %zu as %s", id, idx.toStr().c_str()); -- } -- // store the index for later comparison -- connToSession[id] = idx; - // update the session with the latest data -- sessions[idx].update(id, statComm); -+ sessions[statComm.getSessId(id)].update(id, statComm); - } - } -+ - void Controller::statOnDisconnect(size_t id){ -- sessIndex idx(statComm, id); -- INSANE_MSG("Ended connection: %zu as %s", id, idx.toStr().c_str()); -- sessions[idx].finish(id); -- connToSession.erase(id); -+ // Check to see if cleanup is required (when a Session binary fails) -+ const std::string thisSessionId = statComm.getSessId(id); -+ // Try to lock to see if the session crashed during boot -+ IPC::semaphore sessionLock; -+ char semName[NAME_BUFFER_SIZE]; -+ snprintf(semName, NAME_BUFFER_SIZE, SEM_SESSION, thisSessionId.c_str()); -+ sessionLock.open(semName, O_CREAT | O_RDWR, ACCESSPERMS, 1); -+ if (!sessionLock.tryWaitOneSecond()){ -+ // Session likely crashed during boot. Remove the session lock which was created on bootup of the session -+ sessionLock.unlink(); -+ }else if (!statComm.sessIdExists(thisSessionId)){ -+ // There is no running process managing this session, so check if the data page still exists -+ IPC::sharedPage dataPage; -+ char userPageName[NAME_BUFFER_SIZE]; -+ snprintf(userPageName, NAME_BUFFER_SIZE, COMMS_SESSIONS, thisSessionId.c_str()); -+ dataPage.init(userPageName, 1, false, false); -+ if(dataPage){ -+ // Session likely crashed while it was running -+ dataPage.init(userPageName, 1, true); -+ FAIL_MSG("Session '%s' got canceled unexpectedly. Hoovering up the left overs...", thisSessionId.c_str()); -+ } -+ // Finally remove the session lock which was created on bootup of the session -+ sessionLock.unlink(); -+ } - } -+ - void Controller::statLeadOut(){} - - /// Returns true if this stream has at least one connected client. - bool Controller::hasViewers(std::string streamName){ - if (sessions.size()){ - long long currTime = Util::bootSecs(); -- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ -- if (it->first.streamName == streamName && -+ for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ -+ if (it->second.getStreamName() == streamName && - (it->second.hasDataFor(currTime) || it->second.hasDataFor(currTime - 1))){ - return true; - } -@@ -1377,7 +1049,6 @@ void Controller::fillClients(JSON::Value &req, JSON::Value &rep){ - if (fields & STAT_CLI_UP){rep["fields"].append("up");} - if (fields & STAT_CLI_BPS_DOWN){rep["fields"].append("downbps");} - if (fields & STAT_CLI_BPS_UP){rep["fields"].append("upbps");} -- if (fields & STAT_CLI_CRC){rep["fields"].append("crc");} - if (fields & STAT_CLI_SESSID){rep["fields"].append("sessid");} - if (fields & STAT_CLI_PKTCOUNT){rep["fields"].append("pktcount");} - if (fields & STAT_CLI_PKTLOST){rep["fields"].append("pktlost");} -@@ -1386,26 +1057,25 @@ void Controller::fillClients(JSON::Value &req, JSON::Value &rep){ - rep["data"].null(); - // loop over all sessions - if (sessions.size()){ -- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ -+ for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ - unsigned long long time = reqTime; - if (now && reqTime - it->second.getEnd() < 5){time = it->second.getEnd();} - // data present and wanted? insert it! - if ((it->second.getEnd() >= time && it->second.getStart() <= time) && -- (!streams.size() || streams.count(it->first.streamName)) && -- (!protos.size() || protos.count(it->first.connector))){ -+ (!streams.size() || streams.count(it->second.getStreamName())) && -+ (!protos.size() || protos.count(it->second.getCurrentProtocols()))){ - if (it->second.hasDataFor(time)){ - JSON::Value d; -- if (fields & STAT_CLI_HOST){d.append(it->first.host);} -- if (fields & STAT_CLI_STREAM){d.append(it->first.streamName);} -- if (fields & STAT_CLI_PROTO){d.append(it->first.connector);} -+ if (fields & STAT_CLI_HOST){d.append(it->second.getHost());} -+ if (fields & STAT_CLI_STREAM){d.append(it->second.getStreamName());} -+ if (fields & STAT_CLI_PROTO){d.append(it->second.getCurrentProtocols());} - if (fields & STAT_CLI_CONNTIME){d.append(it->second.getConnTime(time));} - if (fields & STAT_CLI_POSITION){d.append(it->second.getLastSecond(time));} - if (fields & STAT_CLI_DOWN){d.append(it->second.getDown(time));} - if (fields & STAT_CLI_UP){d.append(it->second.getUp(time));} - if (fields & STAT_CLI_BPS_DOWN){d.append(it->second.getBpsDown(time));} - if (fields & STAT_CLI_BPS_UP){d.append(it->second.getBpsUp(time));} -- if (fields & STAT_CLI_CRC){d.append(it->first.crc);} -- if (fields & STAT_CLI_SESSID){d.append(it->first.ID);} -+ if (fields & STAT_CLI_SESSID){d.append(it->second.getSessId());} - if (fields & STAT_CLI_PKTCOUNT){d.append(it->second.getPktCount(time));} - if (fields & STAT_CLI_PKTLOST){d.append(it->second.getPktLost(time));} - if (fields & STAT_CLI_PKTRETRANSMIT){d.append(it->second.getPktRetransmit(time));} -@@ -1463,12 +1133,12 @@ void Controller::fillHasStats(JSON::Value &req, JSON::Value &rep){ - { - tthread::lock_guard guard(statsMutex); - if (sessions.size()){ -- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ -+ for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ - if (it->second.getSessType() == SESS_INPUT){ -- streams.insert(it->first.streamName); -+ streams.insert(it->second.getStreamName()); - }else{ -- streams.insert(it->first.streamName); -- if (it->second.getSessType() == SESS_VIEWER){clients[it->first.streamName]++;} -+ streams.insert(it->second.getStreamName()); -+ if (it->second.getSessType() == SESS_VIEWER){clients[it->second.getStreamName()]++;} - } - } - } -@@ -1742,12 +1412,12 @@ void Controller::fillTotals(JSON::Value &req, JSON::Value &rep){ - // loop over all sessions - /// \todo Make the interval configurable instead of 1 second - if (sessions.size()){ -- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ -+ for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ - // data present and wanted? insert it! - if ((it->second.getEnd() >= (unsigned long long)reqStart || - it->second.getStart() <= (unsigned long long)reqEnd) && -- (!streams.size() || streams.count(it->first.streamName)) && -- (!protos.size() || protos.count(it->first.connector))){ -+ (!streams.size() || streams.count(it->second.getStreamName())) && -+ (!protos.size() || protos.count(it->second.getCurrentProtocols()))){ - for (unsigned long long i = reqStart; i <= reqEnd; ++i){ - if (it->second.hasDataFor(i)){ - totalsCount[i].add(it->second.getBpsDown(i), it->second.getBpsUp(i), it->second.getSessType(), it->second.getPktCount(), it->second.getPktLost(), it->second.getPktRetransmit()); -@@ -1831,6 +1501,25 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int - H.SetHeader("Server", APPIDENT); - H.StartResponse("200", "OK", H, conn, true); - -+ // Counters of current active viewers, inputs and outputs of the Session stats cache -+ std::map outputs; -+ uint32_t totViewers = 0; -+ uint32_t totInputs = 0; -+ uint32_t totOutputs = 0; -+ for (uint64_t idx = 0; idx < statComm.recordCount(); idx++){ -+ if (statComm.getStatus(idx) == COMM_STATUS_INVALID || statComm.getStatus(idx) & COMM_STATUS_DISCONNECT){continue;} -+ const std::string thisSessId = statComm.getSessId(idx); -+ // Count active viewers, inputs, outputs and protocols -+ if (thisSessId[0] == 'I'){ -+ totInputs++; -+ }else if (thisSessId[0] == 'O'){ -+ totOutputs++; -+ outputs[statComm.getConnector(idx)]++; -+ }else{ -+ totViewers++; -+ } -+ } -+ - // Collect core server stats - uint64_t mem_total = 0, mem_free = 0, mem_bufcache = 0; - uint64_t bw_up_total = 0, bw_down_total = 0; -@@ -1904,109 +1593,69 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int - response << "# TYPE mist_shm_used gauge\n"; - response << "mist_shm_used " << (shm_total - shm_free) << "\n\n"; - -- if (Controller::triggerStats.size()){ -- response << "# HELP mist_trigger_count Total executions for the given trigger\n"; -- response << "# HELP mist_trigger_time Total execution time in millis for the given trigger\n"; -- response << "# HELP mist_trigger_fails Total failed executions for the given trigger\n"; -- for (std::map::iterator it = Controller::triggerStats.begin(); -- it != Controller::triggerStats.end(); it++){ -- response << "mist_trigger_count{trigger=\"" << it->first << "\"}" << it->second.totalCount << "\n"; -- response << "mist_trigger_time{trigger=\"" << it->first << "\"}" << it->second.ms << "\n"; -- response << "mist_trigger_fails{trigger=\"" << it->first << "\"}" << it->second.failCount << "\n"; -+ response << "# HELP mist_viewseconds_total Number of seconds any media was received by a viewer.\n"; -+ response << "# TYPE mist_viewseconds_total counter\n"; -+ response << "mist_viewseconds_total " << servSeconds + viewSecondsTotal << "\n"; -+ -+ response << "\n# HELP mist_sessions_count Counts of unique sessions by type since server " -+ "start.\n"; -+ response << "# TYPE mist_sessions_count counter\n"; -+ response << "mist_sessions_count{sessType=\"viewers\"}" << servViewers << "\n"; -+ response << "mist_sessions_count{sessType=\"incoming\"}" << servInputs << "\n"; -+ response << "mist_sessions_count{sessType=\"outgoing\"}" << servOutputs << "\n\n"; -+ -+ response << "# HELP mist_bw_total Count of bytes handled since server start, by direction.\n"; -+ response << "# TYPE mist_bw_total counter\n"; -+ response << "stat_bw_total{direction=\"up\"}" << bw_up_total << "\n"; -+ response << "stat_bw_total{direction=\"down\"}" << bw_down_total << "\n\n"; -+ response << "mist_bw_total{direction=\"up\"}" << servUpBytes << "\n"; -+ response << "mist_bw_total{direction=\"down\"}" << servDownBytes << "\n\n"; -+ response << "mist_bw_other{direction=\"up\"}" << servUpOtherBytes << "\n"; -+ response << "mist_bw_other{direction=\"down\"}" << servDownOtherBytes << "\n\n"; -+ response << "mist_bw_limit " << bwLimit << "\n\n"; -+ -+ response << "# HELP mist_packets_total Total number of packets sent/received/lost over lossy protocols, server-wide.\n"; -+ response << "# TYPE mist_packets_total counter\n"; -+ response << "mist_packets_total{pkttype=\"sent\"}" << servPackSent << "\n"; -+ response << "mist_packets_total{pkttype=\"lost\"}" << servPackLoss << "\n"; -+ response << "mist_packets_total{pkttype=\"retrans\"}" << servPackRetrans << "\n"; -+ -+ if (outputs.size()){ -+ response << "# HELP mist_outputs Number of viewers active right now, server-wide, by output type.\n"; -+ response << "# TYPE mist_outputs gauge\n"; -+ for (std::map::iterator it = outputs.begin(); it != outputs.end(); ++it){ -+ response << "mist_outputs{output=\"" << it->first << "\"}" << it->second << "\n"; - } - response << "\n"; - } - - {// Scope for shortest possible blocking of statsMutex - tthread::lock_guard guard(statsMutex); -- // collect the data first -- std::map outputs; -- unsigned long totViewers = 0, totInputs = 0, totOutputs = 0; -- unsigned int tOut = Util::bootSecs() - STATS_DELAY; -- unsigned int tIn = Util::bootSecs() - STATS_INPUT_DELAY; -- // check all sessions -- if (sessions.size()){ -- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ -- switch (it->second.getSessType()){ -- case SESS_UNSET: break; -- case SESS_VIEWER: -- if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){ -- outputs[it->first.connector]++; -- totViewers++; -- } -- break; -- case SESS_INPUT: -- if (it->second.hasDataFor(tIn) && it->second.isViewerOn(tIn)){totInputs++;} -- break; -- case SESS_OUTPUT: -- if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){totOutputs++;} -- break; -- } -- } -- } - -- response << "# HELP mist_sessions_total Number of sessions active right now, server-wide, by " -- "type.\n"; -+ response << "# HELP mist_sessions_total Number of sessions active right now, server-wide, by type.\n"; - response << "# TYPE mist_sessions_total gauge\n"; - response << "mist_sessions_total{sessType=\"viewers\"}" << totViewers << "\n"; - response << "mist_sessions_total{sessType=\"incoming\"}" << totInputs << "\n"; - response << "mist_sessions_total{sessType=\"outgoing\"}" << totOutputs << "\n"; -- response << "mist_sessions_total{sessType=\"cached\"}" << sessions.size() << "\n\n"; -- -- response << "# HELP mist_viewseconds_total Number of seconds any media was received by a viewer.\n"; -- response << "# TYPE mist_viewseconds_total counter\n"; -- response << "mist_viewseconds_total " << servSeconds << "\n"; -+ response << "mist_sessions_total{sessType=\"cached\"}" << sessions.size() << "\n"; - -- response << "# HELP mist_outputs Number of viewers active right now, server-wide, by output type.\n"; -- response << "# TYPE mist_outputs gauge\n"; -- for (std::map::iterator it = outputs.begin(); it != outputs.end(); ++it){ -- response << "mist_outputs{output=\"" << it->first << "\"}" << it->second << "\n"; -- } -- response << "\n"; -- -- response << "# HELP mist_sessions_count Counts of unique sessions by type since server " -- "start.\n"; -- response << "# TYPE mist_sessions_count counter\n"; -- response << "mist_sessions_count{sessType=\"viewers\"}" << servViewers << "\n"; -- response << "mist_sessions_count{sessType=\"incoming\"}" << servInputs << "\n"; -- response << "mist_sessions_count{sessType=\"outgoing\"}" << servOutputs << "\n\n"; -- -- response << "# HELP mist_bw_total Count of bytes handled since server start, by direction.\n"; -- response << "# TYPE mist_bw_total counter\n"; -- response << "stat_bw_total{direction=\"up\"}" << bw_up_total << "\n"; -- response << "stat_bw_total{direction=\"down\"}" << bw_down_total << "\n\n"; -- response << "mist_bw_total{direction=\"up\"}" << servUpBytes << "\n"; -- response << "mist_bw_total{direction=\"down\"}" << servDownBytes << "\n\n"; -- response << "mist_bw_other{direction=\"up\"}" << servUpOtherBytes << "\n"; -- response << "mist_bw_other{direction=\"down\"}" << servDownOtherBytes << "\n\n"; -- response << "mist_bw_limit " << bwLimit << "\n\n"; -- -- response << "# HELP mist_packets_total Total number of packets sent/received/lost over lossy protocols, server-wide.\n"; -- response << "# TYPE mist_packets_total counter\n"; -- response << "mist_packets_total{pkttype=\"sent\"}" << servPackSent << "\n"; -- response << "mist_packets_total{pkttype=\"lost\"}" << servPackLoss << "\n"; -- response << "mist_packets_total{pkttype=\"retrans\"}" << servPackRetrans << "\n"; -- -- response << "\n# HELP mist_viewers Number of sessions by type and stream active right now.\n"; -- response << "# TYPE mist_viewers gauge\n"; -- response << "# HELP mist_viewcount Count of unique viewer sessions since stream start, per " -+ response << "\n# HELP mist_viewcount Count of unique viewer sessions since stream start, per " - "stream.\n"; - response << "# TYPE mist_viewcount counter\n"; -- response << "# HELP mist_bw Count of bytes handled since stream start, by direction.\n"; -- response << "# TYPE mist_bw counter\n"; - response << "# HELP mist_viewseconds Number of seconds any media was received by a viewer.\n"; - response << "# TYPE mist_viewseconds counter\n"; -+ response << "# HELP mist_bw Count of bytes handled since stream start, by direction.\n"; -+ response << "# TYPE mist_bw counter\n"; - response << "# HELP mist_packets Total number of packets sent/received/lost over lossy protocols.\n"; - response << "# TYPE mist_packets counter\n"; -- response << "mist_viewseconds_total " << servSeconds << "\n"; - for (std::map::iterator it = streamStats.begin(); -- it != streamStats.end(); ++it){ -+ it != streamStats.end(); ++it){ - response << "mist_sessions{stream=\"" << it->first << "\",sessType=\"viewers\"}" -- << it->second.currViews << "\n"; -+ << it->second.currViews << "\n"; - response << "mist_sessions{stream=\"" << it->first << "\",sessType=\"incoming\"}" -- << it->second.currIns << "\n"; -+ << it->second.currIns << "\n"; - response << "mist_sessions{stream=\"" << it->first << "\",sessType=\"outgoing\"}" -- << it->second.currOuts << "\n"; -+ << it->second.currOuts << "\n"; - response << "mist_viewcount{stream=\"" << it->first << "\"}" << it->second.viewers << "\n"; - response << "mist_viewseconds{stream=\"" << it->first << "\"} " << it->second.viewSeconds << "\n"; - response << "mist_bw{stream=\"" << it->first << "\",direction=\"up\"}" << it->second.upBytes << "\n"; -@@ -2015,6 +1664,19 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int - response << "mist_packets{stream=\"" << it->first << "\",pkttype=\"lost\"}" << it->second.packLoss << "\n"; - response << "mist_packets{stream=\"" << it->first << "\",pkttype=\"retrans\"}" << it->second.packRetrans << "\n"; - } -+ -+ if (Controller::triggerStats.size()){ -+ response << "\n# HELP mist_trigger_count Total executions for the given trigger\n"; -+ response << "# HELP mist_trigger_time Total execution time in millis for the given trigger\n"; -+ response << "# HELP mist_trigger_fails Total failed executions for the given trigger\n"; -+ for (std::map::iterator it = Controller::triggerStats.begin(); -+ it != Controller::triggerStats.end(); it++){ -+ response << "mist_trigger_count{trigger=\"" << it->first << "\"}" << it->second.totalCount << "\n"; -+ response << "mist_trigger_time{trigger=\"" << it->first << "\"}" << it->second.ms << "\n"; -+ response << "mist_trigger_fails{trigger=\"" << it->first << "\"}" << it->second.failCount << "\n"; -+ } -+ response << "\n"; -+ } - } - H.Chunkify(response.str(), conn); - } -@@ -2026,58 +1688,33 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int - resp["shm_total"] = shm_total; - resp["shm_used"] = (shm_total - shm_free); - resp["logs"] = Controller::logCounter; -- if (Controller::triggerStats.size()){ -- for (std::map::iterator it = Controller::triggerStats.begin(); -- it != Controller::triggerStats.end(); it++){ -- JSON::Value &tVal = resp["triggers"][it->first]; -- tVal["count"] = it->second.totalCount; -- tVal["ms"] = it->second.ms; -- tVal["fails"] = it->second.failCount; -- } -- } -+ resp["curr"].append(totViewers); -+ resp["curr"].append(totInputs); -+ resp["curr"].append(totOutputs); -+ resp["tot"].append(servViewers); -+ resp["tot"].append(servInputs); -+ resp["tot"].append(servOutputs); -+ resp["st"].append(bw_up_total); -+ resp["st"].append(bw_down_total); -+ resp["bw"].append(servUpBytes); -+ resp["bw"].append(servDownBytes); -+ resp["pkts"].append(servPackSent); -+ resp["pkts"].append(servPackLoss); -+ resp["pkts"].append(servPackRetrans); -+ resp["bwlimit"] = bwLimit; - {// Scope for shortest possible blocking of statsMutex - tthread::lock_guard guard(statsMutex); -- // collect the data first -- std::map outputs; -- uint64_t totViewers = 0, totInputs = 0, totOutputs = 0; -- uint64_t tOut = Util::bootSecs() - STATS_DELAY; -- uint64_t tIn = Util::bootSecs() - STATS_INPUT_DELAY; -- // check all sessions -- if (sessions.size()){ -- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ -- switch (it->second.getSessType()){ -- case SESS_UNSET: break; -- case SESS_VIEWER: -- if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){ -- outputs[it->first.connector]++; -- totViewers++; -- } -- break; -- case SESS_INPUT: -- if (it->second.hasDataFor(tIn) && it->second.isViewerOn(tIn)){totInputs++;} -- break; -- case SESS_OUTPUT: -- if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){totOutputs++;} -- break; -- } -+ resp["curr"].append((uint64_t)sessions.size()); -+ -+ if (Controller::triggerStats.size()){ -+ for (std::map::iterator it = Controller::triggerStats.begin(); -+ it != Controller::triggerStats.end(); it++){ -+ JSON::Value &tVal = resp["triggers"][it->first]; -+ tVal["count"] = it->second.totalCount; -+ tVal["ms"] = it->second.ms; -+ tVal["fails"] = it->second.failCount; - } - } -- -- resp["curr"].append(totViewers); -- resp["curr"].append(totInputs); -- resp["curr"].append(totOutputs); -- resp["curr"].append((uint64_t)sessions.size()); -- resp["tot"].append(servViewers); -- resp["tot"].append(servInputs); -- resp["tot"].append(servOutputs); -- resp["st"].append(bw_up_total); -- resp["st"].append(bw_down_total); -- resp["bw"].append(servUpBytes); -- resp["bw"].append(servDownBytes); -- resp["pkts"].append(servPackSent); -- resp["pkts"].append(servPackLoss); -- resp["pkts"].append(servPackRetrans); -- resp["bwlimit"] = bwLimit; - if (Storage["config"].isMember("location") && Storage["config"]["location"].isMember("lat") && Storage["config"]["location"].isMember("lon")){ - resp["loc"]["lat"] = Storage["config"]["location"]["lat"].asDouble(); - resp["loc"]["lon"] = Storage["config"]["location"]["lon"].asDouble(); -diff --git a/src/controller/controller_statistics.h b/src/controller/controller_statistics.h -index 1cc4a82d..f798f811 100644 ---- a/src/controller/controller_statistics.h -+++ b/src/controller/controller_statistics.h -@@ -18,8 +18,7 @@ - namespace Controller{ - - extern bool killOnExit; -- extern unsigned int maxConnsPerIP; -- -+ - /// This function is ran whenever a stream becomes active. - void streamStarted(std::string stream); - /// This function is ran whenever a stream becomes inactive. -@@ -35,34 +34,14 @@ namespace Controller{ - uint64_t pktCount; - uint64_t pktLost; - uint64_t pktRetransmit; -+ std::string connectors; - }; - - enum sessType{SESS_UNSET = 0, SESS_INPUT, SESS_OUTPUT, SESS_VIEWER}; - -- /// This is a comparison and storage class that keeps sessions apart from each other. -- /// Whenever two of these objects are not equal, it will create a new session. -- class sessIndex{ -- public: -- sessIndex(); -- sessIndex(const Comms::Statistics &statComm, size_t id); -- std::string ID; -- std::string host; -- unsigned int crc; -- std::string streamName; -- std::string connector; -- -- bool operator==(const sessIndex &o) const; -- bool operator!=(const sessIndex &o) const; -- bool operator>(const sessIndex &o) const; -- bool operator<=(const sessIndex &o) const; -- bool operator<(const sessIndex &o) const; -- bool operator>=(const sessIndex &o) const; -- std::string toStr(); -- }; -- - class statStorage{ - public: -- void update(Comms::Statistics &statComm, size_t index); -+ void update(Comms::Sessions &statComm, size_t index); - bool hasDataFor(unsigned long long); - statLog &getDataFor(unsigned long long); - std::map log; -@@ -75,36 +54,33 @@ namespace Controller{ - uint64_t firstActive; - uint64_t firstSec; - uint64_t lastSec; -- uint64_t wipedUp; -- uint64_t wipedDown; -- uint64_t wipedPktCount; -- uint64_t wipedPktLost; -- uint64_t wipedPktRetransmit; -- std::deque oldConns; - sessType sessionType; - bool tracked; - uint8_t noBWCount; ///< Set to 2 when not to count for external bandwidth -+ std::string streamName; -+ std::string host; -+ std::string curConnector; -+ std::string sessId; -+ - public: - statSession(); -- uint32_t invalidate(); -- uint32_t kill(); -- char sync; -- std::map curConns; -+ ~statSession(); -+ statStorage curData; - std::set tags; - sessType getSessType(); -- void wipeOld(uint64_t); -- void finish(uint64_t index); -- void switchOverTo(statSession &newSess, uint64_t index); -- void update(uint64_t index, Comms::Statistics &data); -- void dropSession(const sessIndex &index); -+ void update(uint64_t index, Comms::Sessions &data); - uint64_t getStart(); - uint64_t getEnd(); - bool isViewerOn(uint64_t time); -- bool isConnected(); - bool isTracked(); - bool hasDataFor(uint64_t time); -- bool hasData(); -+ std::string getStreamName(); -+ std::string getHost(); -+ std::string getSessId(); -+ std::string getCurrentProtocols(); -+ uint64_t newestDataPoint(); - uint64_t getConnTime(uint64_t time); -+ uint64_t getConnTime(); - uint64_t getLastSecond(uint64_t time); - uint64_t getDown(uint64_t time); - uint64_t getUp(); -@@ -122,8 +98,6 @@ namespace Controller{ - uint64_t getBpsUp(uint64_t start, uint64_t end); - }; - -- extern std::map sessions; -- extern std::map connToSession; - extern tthread::mutex statsMutex; - extern uint64_t statDropoff; - -@@ -155,6 +129,7 @@ namespace Controller{ - void sessions_shutdown(const std::string &streamname, const std::string &protocol = ""); - bool hasViewers(std::string streamName); - void writeSessionCache(); /*LTS*/ -+ void killConnections(std::string sessId); - - #define PROMETHEUS_TEXT 0 - #define PROMETHEUS_JSON 1 -diff --git a/src/controller/controller_storage.cpp b/src/controller/controller_storage.cpp -index 58e7b521..bdd52893 100644 ---- a/src/controller/controller_storage.cpp -+++ b/src/controller/controller_storage.cpp -@@ -91,19 +91,6 @@ namespace Controller{ - rlxAccs->setString("tags", tags, newEndPos); - rlxAccs->setEndPos(newEndPos + 1); - } -- if (Triggers::shouldTrigger("USER_END", strm)){ -- std::stringstream plgen; -- plgen << sessId << "\n" -- << strm << "\n" -- << conn << "\n" -- << host << "\n" -- << duration << "\n" -- << up << "\n" -- << down << "\n" -- << tags; -- std::string payload = plgen.str(); -- Triggers::doTrigger("USER_END", payload, strm); -- } - } - - void normalizeTrustedProxies(JSON::Value &tp){ -@@ -450,7 +437,8 @@ namespace Controller{ - systemBoot = globAccX.getInt("systemBoot"); - } - if(!globAccX.getFieldAccX("defaultStream") -- || !globAccX.getFieldAccX("systemBoot")){ -+ || !globAccX.getFieldAccX("systemBoot") -+ || !globAccX.getFieldAccX("sessionMode")){ - globAccX.setReload(); - globCfg.master = true; - globCfg.close(); -@@ -461,12 +449,16 @@ namespace Controller{ - if (!globAccX.isReady()){ - globAccX.addField("defaultStream", RAX_128STRING); - globAccX.addField("systemBoot", RAX_64UINT); -+ globAccX.addField("sessionMode", RAX_64UINT); -+ if (!Storage["config"]["sessionMode"]){ -+ Storage["config"]["sessionMode"] = SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID; -+ } - globAccX.setRCount(1); - globAccX.setEndPos(1); - globAccX.setReady(); - } - globAccX.setString("defaultStream", Storage["config"]["defaultStream"].asStringRef()); -- globAccX.setInt("systemBoot", systemBoot); -+ globAccX.setInt("sessionMode", Storage["config"]["sessionMode"].asInt()); - globCfg.master = false; // leave the page after closing - } - } -diff --git a/src/input/input.cpp b/src/input/input.cpp -index a86fe3cd..0524f55b 100644 ---- a/src/input/input.cpp -+++ b/src/input/input.cpp -@@ -794,7 +794,7 @@ namespace Mist{ - void Input::streamMainLoop(){ - uint64_t statTimer = 0; - uint64_t startTime = Util::bootSecs(); -- Comms::Statistics statComm; -+ Comms::Connections statComm; - getNext(); - if (thisPacket && !userSelect.count(thisIdx)){ - userSelect[thisIdx].reload(streamName, thisIdx, COMM_STATUS_ACTIVE | COMM_STATUS_SOURCE | COMM_STATUS_DONOTTRACK); -@@ -820,7 +820,7 @@ namespace Mist{ - - if (Util::bootSecs() - statTimer > 1){ - // Connect to stats for INPUT detection -- if (!statComm){statComm.reload();} -+ if (!statComm){statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID);} - if (statComm){ - if (!statComm){ - config->is_active = false; -@@ -829,7 +829,6 @@ namespace Mist{ - } - uint64_t now = Util::bootSecs(); - statComm.setNow(now); -- statComm.setCRC(getpid()); - statComm.setStream(streamName); - statComm.setConnector("INPUT:" + capa["name"].asStringRef()); - statComm.setTime(now - startTime); -@@ -842,7 +841,7 @@ namespace Mist{ - } - } - -- void Input::connStats(Comms::Statistics &statComm){ -+ void Input::connStats(Comms::Connections &statComm){ - statComm.setUp(0); - statComm.setDown(streamByteCount()); - statComm.setHost(getConnectedBinHost()); -@@ -853,7 +852,7 @@ namespace Mist{ - uint64_t statTimer = 0; - uint64_t startTime = Util::bootSecs(); - size_t idx; -- Comms::Statistics statComm; -+ Comms::Connections statComm; - - - DTSC::Meta liveMeta(config->getString("streamname"), false); -@@ -985,7 +984,7 @@ namespace Mist{ - - if (Util::bootSecs() - statTimer > 1){ - // Connect to stats for INPUT detection -- if (!statComm){statComm.reload();} -+ if (!statComm){statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID);} - if (statComm){ - if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ - config->is_active = false; -@@ -994,7 +993,6 @@ namespace Mist{ - } - uint64_t now = Util::bootSecs(); - statComm.setNow(now); -- statComm.setCRC(getpid()); - statComm.setStream(streamName); - statComm.setConnector("INPUT:" + capa["name"].asStringRef()); - statComm.setTime(now - startTime); -diff --git a/src/input/input.h b/src/input/input.h -index 8d7e8891..ce7686c1 100644 ---- a/src/input/input.h -+++ b/src/input/input.h -@@ -69,7 +69,7 @@ namespace Mist{ - virtual void userOnActive(size_t id); - virtual void userOnDisconnect(size_t id); - virtual void userLeadOut(); -- virtual void connStats(Comms::Statistics & statComm); -+ virtual void connStats(Comms::Connections & statComm); - virtual void parseHeader(); - bool bufferFrame(size_t track, uint32_t keyNum); - -diff --git a/src/input/input_rtsp.cpp b/src/input/input_rtsp.cpp -index fef613bb..ebb812e1 100644 ---- a/src/input/input_rtsp.cpp -+++ b/src/input/input_rtsp.cpp -@@ -196,7 +196,7 @@ namespace Mist{ - } - - void InputRTSP::streamMainLoop(){ -- Comms::Statistics statComm; -+ Comms::Connections statComm; - uint64_t startTime = Util::epoch(); - uint64_t lastPing = Util::bootSecs(); - uint64_t lastSecs = 0; -@@ -210,7 +210,7 @@ namespace Mist{ - if (lastSecs != currSecs){ - lastSecs = currSecs; - // Connect to stats for INPUT detection -- statComm.reload(); -+ statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID); - if (statComm){ - if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ - config->is_active = false; -@@ -219,7 +219,6 @@ namespace Mist{ - } - uint64_t now = Util::bootSecs(); - statComm.setNow(now); -- statComm.setCRC(getpid()); - statComm.setStream(streamName); - statComm.setConnector("INPUT:" + capa["name"].asStringRef()); - statComm.setUp(tcpCon.dataUp()); -diff --git a/src/input/input_sdp.cpp b/src/input/input_sdp.cpp -index 62c02aef..3169a836 100644 ---- a/src/input/input_sdp.cpp -+++ b/src/input/input_sdp.cpp -@@ -193,7 +193,7 @@ namespace Mist{ - - // Updates stats and quits if parsePacket returns false - void InputSDP::streamMainLoop(){ -- Comms::Statistics statComm; -+ Comms::Connections statComm; - uint64_t startTime = Util::epoch(); - uint64_t lastSecs = 0; - // Get RTP packets from UDP socket and stop if this fails -@@ -202,7 +202,7 @@ namespace Mist{ - if (lastSecs != currSecs){ - lastSecs = currSecs; - // Connect to stats for INPUT detection -- statComm.reload(); -+ statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID); - if (statComm){ - if (statComm.getStatus() == COMM_STATUS_REQDISCONNECT){ - config->is_active = false; -@@ -211,7 +211,6 @@ namespace Mist{ - } - uint64_t now = Util::bootSecs(); - statComm.setNow(now); -- statComm.setCRC(getpid()); - statComm.setStream(streamName); - statComm.setConnector("INPUT:" + capa["name"].asStringRef()); - statComm.setDown(bytesRead); -diff --git a/src/input/input_ts.cpp b/src/input/input_ts.cpp -index cfe3f7cd..23311dc0 100644 ---- a/src/input/input_ts.cpp -+++ b/src/input/input_ts.cpp -@@ -527,7 +527,7 @@ namespace Mist{ - void inputTS::streamMainLoop(){ - meta.removeTrack(tmpIdx); - INFO_MSG("Removed temptrack %zu", tmpIdx); -- Comms::Statistics statComm; -+ Comms::Connections statComm; - uint64_t downCounter = 0; - uint64_t startTime = Util::bootSecs(); - uint64_t noDataSince = Util::bootSecs(); -@@ -621,7 +621,7 @@ namespace Mist{ - // Check for and spawn threads here. - if (Util::bootSecs() - threadCheckTimer > 1){ - // Connect to stats for INPUT detection -- statComm.reload(); -+ statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID); - if (statComm){ - if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ - config->is_active = false; -@@ -630,7 +630,6 @@ namespace Mist{ - } - uint64_t now = Util::bootSecs(); - statComm.setNow(now); -- statComm.setCRC(getpid()); - statComm.setStream(streamName); - statComm.setConnector("INPUT:" + capa["name"].asStringRef()); - statComm.setUp(0); -diff --git a/src/input/input_tssrt.cpp b/src/input/input_tssrt.cpp -index 4219ebbc..8fef6d7d 100644 ---- a/src/input/input_tssrt.cpp -+++ b/src/input/input_tssrt.cpp -@@ -282,7 +282,7 @@ namespace Mist{ - - void inputTSSRT::setSingular(bool newSingular){singularFlag = newSingular;} - -- void inputTSSRT::connStats(Comms::Statistics &statComm){ -+ void inputTSSRT::connStats(Comms::Connections &statComm){ - statComm.setUp(srtConn.dataUp()); - statComm.setDown(srtConn.dataDown()); - statComm.setHost(getConnectedBinHost()); -diff --git a/src/input/input_tssrt.h b/src/input/input_tssrt.h -index 4f337b48..143174cb 100644 ---- a/src/input/input_tssrt.h -+++ b/src/input/input_tssrt.h -@@ -40,7 +40,7 @@ namespace Mist{ - - Socket::SRTConnection srtConn; - bool singularFlag; -- virtual void connStats(Comms::Statistics &statComm); -+ virtual void connStats(Comms::Connections &statComm); - - Util::ResizeablePointer rawBuffer; - size_t rawIdx; -diff --git a/src/output/output.cpp b/src/output/output.cpp -index 1fa86eee..50d67549 100644 ---- a/src/output/output.cpp -+++ b/src/output/output.cpp -@@ -7,7 +7,7 @@ - #include - #include - --#include "output.h" -+#include "output.h" - #include - #include - #include -@@ -92,7 +92,7 @@ namespace Mist{ - firstTime = 0; - firstPacketTime = 0xFFFFFFFFFFFFFFFFull; - lastPacketTime = 0; -- crc = getpid(); -+ sid = ""; - parseData = false; - wantRequest = true; - sought = false; -@@ -100,7 +100,7 @@ namespace Mist{ - isBlocking = false; - needsLookAhead = 0; - extraKeepAway = 0; -- lastStats = 0; -+ lastStats = 0xFFFFFFFFFFFFFFFFull; - maxSkipAhead = 7500; - uaDelay = 10; - realTime = 1000; -@@ -111,6 +111,7 @@ namespace Mist{ - lastPushUpdate = 0; - previousFile = ""; - currentFile = ""; -+ sessionMode = 0xFFFFFFFFFFFFFFFFull; - - lastRecv = Util::bootSecs(); - if (myConn){ -@@ -211,95 +212,9 @@ namespace Mist{ - onFail("Not allowed to play (CONN_PLAY)"); - } - } -- doSync(true); - /*LTS-END*/ - } - -- /// If called with force set to true and a USER_NEW trigger enabled, forces a sync immediately. -- /// Otherwise, does nothing unless the sync byte is set to 2, in which case it forces a sync as well. -- /// May be called recursively because it calls stats() which calls this function. -- /// If this happens, the extra calls to the function return instantly. -- void Output::doSync(bool force){ -- if (!statComm){return;} -- if (recursingSync){return;} -- recursingSync = true; -- if (statComm.getSync() == 2 || force){ -- if (getStatsName() == capa["name"].asStringRef() && Triggers::shouldTrigger("USER_NEW", streamName)){ -- // sync byte 0 = no sync yet, wait for sync from controller... -- char initialSync = 0; -- // attempt to load sync status from session cache in shm -- { -- IPC::semaphore cacheLock(SEM_SESSCACHE, O_RDWR, ACCESSPERMS, 16); -- if (cacheLock){cacheLock.wait();} -- IPC::sharedPage shmSessions(SHM_SESSIONS, SHM_SESSIONS_SIZE, false, false); -- if (shmSessions.mapped){ -- char shmEmpty[SHM_SESSIONS_ITEM]; -- memset(shmEmpty, 0, SHM_SESSIONS_ITEM); -- std::string host; -- Socket::hostBytesToStr(statComm.getHost().data(), 16, host); -- uint32_t shmOffset = 0; -- const std::string &cName = capa["name"].asStringRef(); -- while (shmOffset + SHM_SESSIONS_ITEM < SHM_SESSIONS_SIZE){ -- // compare crc -- if (*(uint32_t*)(shmSessions.mapped + shmOffset) == crc){ -- // compare stream name -- if (strncmp(shmSessions.mapped + shmOffset + 4, streamName.c_str(), 100) == 0){ -- // compare connector -- if (strncmp(shmSessions.mapped + shmOffset + 104, cName.c_str(), 20) == 0){ -- // compare host -- if (strncmp(shmSessions.mapped + shmOffset + 124, host.c_str(), 40) == 0){ -- initialSync = shmSessions.mapped[shmOffset + 164]; -- HIGH_MSG("Instant-sync from session cache to %u", (unsigned int)initialSync); -- break; -- } -- } -- } -- } -- // stop if we reached the end -- if (memcmp(shmSessions.mapped + shmOffset, shmEmpty, SHM_SESSIONS_ITEM) == 0){ -- break; -- } -- shmOffset += SHM_SESSIONS_ITEM; -- } -- } -- if (cacheLock){cacheLock.post();} -- } -- unsigned int i = 0; -- statComm.setSync(initialSync); -- // wait max 10 seconds for sync -- while ((!statComm.getSync() || statComm.getSync() == 2) && i++ < 100){ -- Util::wait(100); -- stats(true); -- } -- HIGH_MSG("USER_NEW sync achieved: %u", statComm.getSync()); -- // 1 = check requested (connection is new) -- if (statComm.getSync() == 1){ -- std::string payload = streamName + "\n" + getConnectedHost() + "\n" + -- JSON::Value(crc).asString() + "\n" + capa["name"].asStringRef() + -- "\n" + reqUrl + "\n" + statComm.getSessId(); -- if (!Triggers::doTrigger("USER_NEW", payload, streamName)){ -- onFail("Not allowed to play (USER_NEW)"); -- statComm.setSync(100); // 100 = denied -- }else{ -- statComm.setSync(10); // 10 = accepted -- } -- } -- // 100 = denied -- if (statComm.getSync() == 100){onFail("Not allowed to play (USER_NEW cache)");} -- if (statComm.getSync() == 0){ -- onFail("Not allowed to play (USER_NEW init timeout)", true); -- } -- if (statComm.getSync() == 2){ -- onFail("Not allowed to play (USER_NEW re-init timeout)", true); -- } -- // anything else = accepted -- }else{ -- statComm.setSync(10); // auto-accept if no trigger -- } -- } -- recursingSync = false; -- } -- - std::string Output::getConnectedHost(){return myConn.getHost();} - - std::string Output::getConnectedBinHost(){ -@@ -433,10 +348,10 @@ namespace Mist{ - - //Connect to stats reporting, if not connected already - if (!statComm){ -- statComm.reload(); -+ statComm.reload(streamName, getConnectedHost(), sid, capa["name"].asStringRef(), reqUrl, sessionMode); - stats(true); - } -- -+ - //push inputs do not need to wait for stream to be ready for playback - if (isPushing()){return;} - -@@ -986,7 +901,7 @@ namespace Mist{ - INFO_MSG("Will split recording every %lld seconds", atoll(targetParams["split"].c_str())); - targetParams["nxt-split"] = JSON::Value((int64_t)(seekPos + endRec)).asString(); - } -- // Duration to record in seconds. Overrides recstop. -+ // Duration to record in seconds. Oversides recstop. - if (targetParams.count("duration")){ - long long endRec = atoll(targetParams["duration"].c_str()) * 1000; - targetParams["recstop"] = JSON::Value((int64_t)(seekPos + endRec)).asString(); -@@ -1301,6 +1216,7 @@ namespace Mist{ - /// request URL (if any) - /// ~~~~~~~~~~~~~~~ - int Output::run(){ -+ sessionMode = Util::getGlobalConfig("sessionMode").asInt(); - /*LTS-START*/ - // Connect to file target, if needed - if (isFileTarget()){ -@@ -1507,6 +1423,8 @@ namespace Mist{ - /*LTS-END*/ - - disconnect(); -+ stats(true); -+ userSelect.clear(); - myConn.close(); - return 0; - } -@@ -1822,7 +1740,7 @@ namespace Mist{ - // also cancel if it has been less than a second since the last update - // unless force is set to true - uint64_t now = Util::bootSecs(); -- if (now == lastStats && !force){return;} -+ if (now <= lastStats && !force){return;} - - if (isRecording()){ - if(lastPushUpdate == 0){ -@@ -1861,13 +1779,17 @@ namespace Mist{ - } - } - -- if (!statComm){statComm.reload();} -- if (!statComm){return;} -+ if (!statComm){statComm.reload(streamName, getConnectedHost(), sid, capa["name"].asStringRef(), reqUrl, sessionMode);} -+ if (!statComm){return;} -+ if (statComm.getExit()){ -+ onFail("Shutting down since this session is not allowed to view this stream"); -+ return; -+ } - - lastStats = now; - -- VERYHIGH_MSG("Writing stats: %s, %s, %u, %" PRIu64 ", %" PRIu64, getConnectedHost().c_str(), streamName.c_str(), -- crc & 0xFFFFFFFFu, myConn.dataUp(), myConn.dataDown()); -+ VERYHIGH_MSG("Writing stats: %s, %s, %s, %" PRIu64 ", %" PRIu64, getConnectedHost().c_str(), streamName.c_str(), -+ sid.c_str(), myConn.dataUp(), myConn.dataDown()); - /*LTS-START*/ - if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ - onFail("Shutting down on controller request"); -@@ -1875,9 +1797,6 @@ namespace Mist{ - } - /*LTS-END*/ - statComm.setNow(now); -- statComm.setHost(getConnectedBinHost()); -- statComm.setCRC(crc); -- statComm.setStream(streamName); - statComm.setConnector(getStatsName()); - connStats(now, statComm); - statComm.setLastSecond(thisPacket ? thisPacket.getTime() : 0); -@@ -1887,7 +1806,7 @@ namespace Mist{ - // Tag the session with the user agent - if (newUA && ((now - myConn.connTime()) >= uaDelay || !myConn) && UA.size()){ - std::string APIcall = -- "{\"tag_sessid\":{\"" + statComm.getSessId() + "\":" + JSON::string_escape("UA:" + UA) + "}}"; -+ "{\"tag_sessid\":{\"" + statComm.sessionId + "\":" + JSON::string_escape("UA:" + UA) + "}}"; - Socket::UDPConnection uSock; - uSock.SetDestination(UDP_API_HOST, UDP_API_PORT); - uSock.SendNow(APIcall); -@@ -1895,8 +1814,6 @@ namespace Mist{ - } - /*LTS-END*/ - -- doSync(); -- - if (isPushing()){ - for (std::map::iterator it = userSelect.begin(); it != userSelect.end(); it++){ - if (it->second.getStatus() & COMM_STATUS_REQDISCONNECT){ -@@ -1912,7 +1829,7 @@ namespace Mist{ - } - } - -- void Output::connStats(uint64_t now, Comms::Statistics &statComm){ -+ void Output::connStats(uint64_t now, Comms::Connections &statComm){ - statComm.setUp(myConn.dataUp()); - statComm.setDown(myConn.dataDown()); - statComm.setTime(now - myConn.connTime()); -diff --git a/src/output/output.h b/src/output/output.h -index 84d17482..173b3840 100644 ---- a/src/output/output.h -+++ b/src/output/output.h -@@ -86,7 +86,6 @@ namespace Mist{ - std::string hostLookup(std::string ip); - bool onList(std::string ip, std::string list); - std::string getCountry(std::string ip); -- void doSync(bool force = false); - /*LTS-END*/ - std::map currentPage; - void loadPageForKey(size_t trackId, size_t keyNum); -@@ -105,6 +104,7 @@ namespace Mist{ - bool firstData; - uint64_t lastPushUpdate; - bool newUA; -+ - protected: // these are to be messed with by child classes - virtual bool inlineRestartCapable() const{ - return false; -@@ -122,15 +122,16 @@ namespace Mist{ - virtual std::string getStatsName(); - virtual bool hasSessionIDs(){return false;} - -- virtual void connStats(uint64_t now, Comms::Statistics &statComm); -+ virtual void connStats(uint64_t now, Comms::Connections &statComm); - - std::set getSupportedTracks(const std::string &type = "") const; - - inline virtual bool keepGoing(){return config->is_active && myConn;} - -- Comms::Statistics statComm; -+ Comms::Connections statComm; - bool isBlocking; ///< If true, indicates that myConn is blocking. -- uint32_t crc; ///< Checksum, if any, for usage in the stats. -+ std::string sid; ///< Random identifier used to split connections into sessions -+ uint64_t sessionMode; - uint64_t nextKeyTime(); - - // stream delaying variables -diff --git a/src/output/output_cmaf.cpp b/src/output/output_cmaf.cpp -index 4582a8b6..c2fbc5bc 100644 ---- a/src/output/output_cmaf.cpp -+++ b/src/output/output_cmaf.cpp -@@ -101,7 +101,7 @@ namespace Mist{ - } - } - -- void OutCMAF::connStats(uint64_t now, Comms::Statistics &statComm){ -+ void OutCMAF::connStats(uint64_t now, Comms::Connections &statComm){ - // For non-push usage, call usual function. - if (!isRecording()){ - Output::connStats(now, statComm); -diff --git a/src/output/output_cmaf.h b/src/output/output_cmaf.h -index efc36511..390549cc 100644 ---- a/src/output/output_cmaf.h -+++ b/src/output/output_cmaf.h -@@ -39,7 +39,7 @@ namespace Mist{ - bool isReadyForPlay(); - - protected: -- virtual void connStats(uint64_t now, Comms::Statistics &statComm); -+ virtual void connStats(uint64_t now, Comms::Connections &statComm); - void onTrackEnd(size_t idx); - bool hasSessionIDs(){return !config->getBool("mergesessions");} - -@@ -72,6 +72,7 @@ namespace Mist{ - void startPushOut(); - void pushNext(); - -+ uint32_t crc; - HTTP::URL pushUrl; - std::map pushTracks; - void setupTrackObject(size_t idx); -diff --git a/src/output/output_http.cpp b/src/output/output_http.cpp -index 252f069a..f55b974d 100644 ---- a/src/output/output_http.cpp -+++ b/src/output/output_http.cpp -@@ -55,7 +55,6 @@ namespace Mist{ - } - - void HTTPOutput::onFail(const std::string &msg, bool critical){ -- INFO_MSG("Failing '%s': %s", H.url.c_str(), msg.c_str()); - if (!webSock && !isRecording() && !responded){ - H.Clean(); // make sure no parts of old requests are left in any buffers - H.SetHeader("Server", APPIDENT); -@@ -238,18 +237,6 @@ namespace Mist{ - } - /*LTS-END*/ - if (H.hasHeader("User-Agent")){UA = H.GetHeader("User-Agent");} -- if (hasSessionIDs()){ -- if (H.GetVar("sessId").size()){ -- std::string ua = H.GetVar("sessId"); -- crc = checksum::crc32(0, ua.data(), ua.size()); -- }else{ -- std::string ua = JSON::Value(getpid()).asString(); -- crc = checksum::crc32(0, ua.data(), ua.size()); -- } -- }else{ -- std::string mixed_ua = UA + H.GetHeader("X-Playback-Session-Id"); -- crc = checksum::crc32(0, mixed_ua.data(), mixed_ua.size()); -- } - - if (H.GetVar("audio") != ""){targetParams["audio"] = H.GetVar("audio");} - if (H.GetVar("video") != ""){targetParams["video"] = H.GetVar("video");} -@@ -281,6 +268,21 @@ namespace Mist{ - realTime = 0; - } - } -+ // Get session ID cookie or generate a random one if it wasn't set -+ if (!sid.size()){ -+ std::map storage; -+ const std::string koekjes = H.GetHeader("Cookie"); -+ HTTP::parseVars(koekjes, storage); -+ if (storage.count("sid")){ -+ // Get sid cookie, which is used to divide connections into sessions -+ sid = storage.at("sid"); -+ }else{ -+ // Else generate one -+ const std::string newSid = UA + JSON::Value(getpid()).asString(); -+ sid = JSON::Value(checksum::crc32(0, newSid.data(), newSid.size())).asString(); -+ H.SetHeader("sid", sid.c_str()); -+ } -+ } - // Handle upgrade to websocket if the output supports it - std::string upgradeHeader = H.GetHeader("Upgrade"); - Util::stringToLower(upgradeHeader); -diff --git a/src/output/output_tssrt.cpp b/src/output/output_tssrt.cpp -index db07dc91..dc04b247 100644 ---- a/src/output/output_tssrt.cpp -+++ b/src/output/output_tssrt.cpp -@@ -344,7 +344,7 @@ namespace Mist{ - } - } - -- void OutTSSRT::connStats(uint64_t now, Comms::Statistics &statComm){ -+ void OutTSSRT::connStats(uint64_t now, Comms::Connections &statComm){ - if (!srtConn){return;} - statComm.setUp(srtConn.dataUp()); - statComm.setDown(srtConn.dataDown()); -diff --git a/src/output/output_tssrt.h b/src/output/output_tssrt.h -index 1423af8d..71c9b72f 100644 ---- a/src/output/output_tssrt.h -+++ b/src/output/output_tssrt.h -@@ -15,7 +15,7 @@ namespace Mist{ - bool isReadyForPlay(){return true;} - virtual void requestHandler(); - protected: -- virtual void connStats(uint64_t now, Comms::Statistics &statComm); -+ virtual void connStats(uint64_t now, Comms::Connections &statComm); - virtual std::string getConnectedHost(){return srtConn.remotehost;} - virtual std::string getConnectedBinHost(){return srtConn.getBinHost();} - -diff --git a/src/output/output_webrtc.cpp b/src/output/output_webrtc.cpp -index b4289b64..058cd737 100644 ---- a/src/output/output_webrtc.cpp -+++ b/src/output/output_webrtc.cpp -@@ -1015,7 +1015,7 @@ namespace Mist{ - } - } - -- void OutWebRTC::connStats(uint64_t now, Comms::Statistics &statComm){ -+ void OutWebRTC::connStats(uint64_t now, Comms::Connections &statComm){ - statComm.setUp(myConn.dataUp()); - statComm.setDown(myConn.dataDown()); - statComm.setPacketCount(totalPkts); -diff --git a/src/output/output_webrtc.h b/src/output/output_webrtc.h -index 9c3db580..b2b528c5 100644 ---- a/src/output/output_webrtc.h -+++ b/src/output/output_webrtc.h -@@ -144,7 +144,7 @@ namespace Mist{ - void onDTSCConverterHasInitData(const size_t trackID, const std::string &initData); - void onRTPPacketizerHasRTPPacket(const char *data, size_t nbytes); - void onRTPPacketizerHasRTCPPacket(const char *data, uint32_t nbytes); -- virtual void connStats(uint64_t now, Comms::Statistics &statComm); -+ virtual void connStats(uint64_t now, Comms::Connections &statComm); - - private: - uint64_t lastRecv; -diff --git a/src/process/process_exec.cpp b/src/process/process_exec.cpp -index 7b775848..34ac53fd 100644 ---- a/src/process/process_exec.cpp -+++ b/src/process/process_exec.cpp -@@ -72,7 +72,7 @@ namespace Mist{ - } - bool needsLock(){return false;} - bool isSingular(){return false;} -- void connStats(Comms::Statistics &statComm){ -+ void connStats(Comms::Connections &statComm){ - for (std::map::iterator it = userSelect.begin(); it != userSelect.end(); it++){ - if (it->second){it->second.setStatus(COMM_STATUS_DONOTTRACK | it->second.getStatus());} - } -@@ -117,7 +117,7 @@ namespace Mist{ - realTime = 0; - OutEBML::sendHeader(); - }; -- void connStats(uint64_t now, Comms::Statistics &statComm){ -+ void connStats(uint64_t now, Comms::Connections &statComm){ - for (std::map::iterator it = userSelect.begin(); it != userSelect.end(); it++){ - if (it->second){it->second.setStatus(COMM_STATUS_DONOTTRACK | it->second.getStatus());} - } -diff --git a/src/session.cpp b/src/session.cpp -new file mode 100644 -index 00000000..3865e0ec ---- /dev/null -+++ b/src/session.cpp -@@ -0,0 +1,367 @@ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+// Stats of connections which have closed are added to these global counters -+uint64_t globalNow = 0; -+uint64_t globalTime = 0; -+uint64_t globalDown = 0; -+uint64_t globalUp = 0; -+uint64_t globalPktcount = 0; -+uint64_t globalPktloss = 0; -+uint64_t globalPktretrans = 0; -+// Counts the duration a connector has been active -+std::map connectorCount; -+std::map connectorLastActive; -+// Set to True when a session gets invalidated, so that we know to run a new USER_NEW trigger -+bool forceTrigger = false; -+void handleSignal(int signum){ -+ if (signum == SIGUSR1){ -+ forceTrigger = true; -+ } -+} -+ -+void userOnActive(uint64_t &connections){ -+ ++connections; -+} -+ -+std::string getEnvWithDefault(const std::string variableName, const std::string defaultValue){ -+ const char* value = getenv(variableName.c_str()); -+ if (value){ -+ unsetenv(variableName.c_str()); -+ return value; -+ }else{ -+ return defaultValue; -+ } -+} -+ -+/// \brief Adds stats of closed connections to global counters -+void userOnDisconnect(Comms::Connections & connections, size_t idx){ -+ std::string thisConnector = connections.getConnector(idx); -+ if (thisConnector != ""){ -+ connectorCount[thisConnector] += connections.getTime(idx); -+ } -+ globalTime += connections.getTime(idx); -+ globalDown += connections.getDown(idx); -+ globalUp += connections.getUp(idx); -+ globalPktcount += connections.getPacketCount(idx); -+ globalPktloss += connections.getPacketLostCount(idx); -+ globalPktretrans += connections.getPacketRetransmitCount(idx); -+} -+ -+int main(int argc, char **argv){ -+ Comms::Connections connections; -+ Comms::Sessions sessions; -+ uint64_t lastSeen = Util::bootSecs(); -+ uint64_t currentConnections = 0; -+ Util::redirectLogsIfNeeded(); -+ signal(SIGUSR1, handleSignal); -+ // Init config and parse arguments -+ Util::Config config = Util::Config("MistSession"); -+ JSON::Value option; -+ -+ option.null(); -+ option["arg_num"] = 1; -+ option["arg"] = "string"; -+ option["help"] = "Session identifier of the entire session"; -+ option["default"] = ""; -+ config.addOption("sessionid", option); -+ -+ option.null(); -+ option["long"] = "sessionmode"; -+ option["short"] = "m"; -+ option["arg"] = "integer"; -+ option["default"] = 0; -+ config.addOption("sessionmode", option); -+ -+ option.null(); -+ option["long"] = "streamname"; -+ option["short"] = "n"; -+ option["arg"] = "string"; -+ option["default"] = ""; -+ config.addOption("streamname", option); -+ -+ option.null(); -+ option["long"] = "ip"; -+ option["short"] = "i"; -+ option["arg"] = "string"; -+ option["default"] = ""; -+ config.addOption("ip", option); -+ -+ option.null(); -+ option["long"] = "sid"; -+ option["short"] = "s"; -+ option["arg"] = "string"; -+ option["default"] = ""; -+ config.addOption("sid", option); -+ -+ option.null(); -+ option["long"] = "protocol"; -+ option["short"] = "p"; -+ option["arg"] = "string"; -+ option["default"] = ""; -+ config.addOption("protocol", option); -+ -+ option.null(); -+ option["long"] = "requrl"; -+ option["short"] = "r"; -+ option["arg"] = "string"; -+ option["default"] = ""; -+ config.addOption("requrl", option); -+ -+ config.activate(); -+ if (!(config.parseArgs(argc, argv))){ -+ FAIL_MSG("Cannot start a new session due to invalid arguments"); -+ return 1; -+ } -+ -+ const uint64_t bootTime = Util::getMicros(); -+ // Get session ID, session mode and other variables used as payload for the USER_NEW and USER_END triggers -+ const std::string thisStreamName = config.getString("streamname"); -+ const std::string thisHost = config.getString("ip"); -+ const std::string thisSid = config.getString("sid"); -+ const std::string thisProtocol = config.getString("protocol"); -+ const std::string thisReqUrl = config.getString("requrl"); -+ const std::string thisSessionId = config.getString("sessionid"); -+ const uint64_t sessionMode = config.getInteger("sessionmode"); -+ -+ if (thisSessionId == "" || thisProtocol == "" || thisStreamName == ""){ -+ FAIL_MSG("Given the following incomplete arguments: SessionId: '%s', protocol: '%s', stream name: '%s'. Aborting opening a new session", -+ thisSessionId.c_str(), thisProtocol.c_str(), thisStreamName.c_str()); -+ return 1; -+ } -+ -+ MEDIUM_MSG("Starting a new session for sessionId '%s'", thisSessionId.c_str()); -+ if (sessionMode < 1 || sessionMode > 15) { -+ FAIL_MSG("Invalid session mode of value %lu. Should be larger than 0 and smaller than 16", sessionMode); -+ return 1; -+ } -+ -+ // Try to lock to ensure we are the only process initialising this session -+ IPC::semaphore sessionLock; -+ char semName[NAME_BUFFER_SIZE]; -+ snprintf(semName, NAME_BUFFER_SIZE, SEM_SESSION, thisSessionId.c_str()); -+ sessionLock.open(semName, O_CREAT | O_RDWR, ACCESSPERMS, 1); -+ // If the lock fails, the previous Session process must've failed in spectacular fashion -+ // It's the Controller's task to clean everything up. When the lock fails, this cleanup hasn't happened yet -+ if (!sessionLock.tryWaitOneSecond()){ -+ FAIL_MSG("Session '%s' already locked", thisSessionId.c_str()); -+ return 1; -+ } -+ -+ // Check if a page already exists for this session ID. If so, quit -+ { -+ IPC::sharedPage dataPage; -+ char userPageName[NAME_BUFFER_SIZE]; -+ snprintf(userPageName, NAME_BUFFER_SIZE, COMMS_SESSIONS, thisSessionId.c_str()); -+ dataPage.init(userPageName, 0, false, false); -+ if (dataPage){ -+ INFO_MSG("Session '%s' already has a running process", thisSessionId.c_str()); -+ sessionLock.post(); -+ return 0; -+ } -+ } -+ -+ // Claim a spot in shared memory for this session on the global statistics page -+ sessions.reload(); -+ if (!sessions){ -+ FAIL_MSG("Unable to register entry for session '%s' on the stats page", thisSessionId.c_str()); -+ sessionLock.post(); -+ return 1; -+ } -+ // Open the shared memory page containing statistics for each individual connection in this session -+ connections.reload(thisStreamName, thisHost, thisSid, thisProtocol, thisReqUrl, sessionMode, true, false); -+ // Initialise global session data -+ sessions.setHost(thisHost); -+ sessions.setSessId(thisSessionId); -+ sessions.setStream(thisStreamName); -+ sessionLock.post(); -+ -+ // Determine session type, since triggers only get run for viewer type sessions -+ uint64_t thisType = 0; -+ if (thisSessionId[0] == 'I'){ -+ INFO_MSG("Started new input session %s in %lu microseconds", thisSessionId.c_str(), Util::getMicros(bootTime)); -+ thisType = 1; -+ } -+ else if (thisSessionId[0] == 'O'){ -+ INFO_MSG("Started new output session %s in %lu microseconds", thisSessionId.c_str(), Util::getMicros(bootTime)); -+ thisType = 2; -+ } -+ else{ -+ INFO_MSG("Started new viewer session %s in %lu microseconds", thisSessionId.c_str(), Util::getMicros(bootTime)); -+ } -+ -+ // Do a USER_NEW trigger if it is defined for this stream -+ if (!thisType && Triggers::shouldTrigger("USER_NEW", thisStreamName)){ -+ std::string payload = thisStreamName + "\n" + thisHost + "\n" + -+ thisSid + "\n" + thisProtocol + -+ "\n" + thisReqUrl + "\n" + thisSessionId; -+ if (!Triggers::doTrigger("USER_NEW", payload, thisStreamName)){ -+ // Mark all connections of this session as finished, since this viewer is not allowed to view this stream -+ connections.setExit(); -+ connections.finishAll(); -+ } -+ } -+ -+ uint64_t lastSecond = 0; -+ uint64_t now = 0; -+ uint64_t time = 0; -+ uint64_t down = 0; -+ uint64_t up = 0; -+ uint64_t pktcount = 0; -+ uint64_t pktloss = 0; -+ uint64_t pktretrans = 0; -+ std::string connector = ""; -+ // Stay active until Mist exits or we no longer have an active connection -+ while (config.is_active && (currentConnections || Util::bootSecs() - lastSeen <= 10)){ -+ time = 0; -+ connector = ""; -+ down = 0; -+ up = 0; -+ pktcount = 0; -+ pktloss = 0; -+ pktretrans = 0; -+ currentConnections = 0; -+ -+ // Count active connections -+ COMM_LOOP(connections, userOnActive(currentConnections), userOnDisconnect(connections, id)); -+ // Loop through all connection entries to get a summary of statistics -+ for (uint64_t idx = 0; idx < connections.recordCount(); idx++){ -+ if (connections.getStatus(idx) == COMM_STATUS_INVALID || connections.getStatus(idx) & COMM_STATUS_DISCONNECT){continue;} -+ uint64_t thisLastSecond = connections.getLastSecond(idx); -+ std::string thisConnector = connections.getConnector(idx); -+ // Save info on the latest active connection separately -+ if (thisLastSecond > lastSecond){ -+ lastSecond = thisLastSecond; -+ now = connections.getNow(idx); -+ } -+ connectorLastActive[thisConnector] = thisLastSecond; -+ // Sum all other variables -+ time += connections.getTime(idx); -+ down += connections.getDown(idx); -+ up += connections.getUp(idx); -+ pktcount += connections.getPacketCount(idx); -+ pktloss += connections.getPacketLostCount(idx); -+ pktretrans += connections.getPacketRetransmitCount(idx); -+ } -+ -+ // Convert connector duration to string -+ std::stringstream connectorSummary; -+ bool addDelimiter = false; -+ connectorSummary << "{"; -+ for (std::map::iterator it = connectorLastActive.begin(); -+ it != connectorLastActive.end(); ++it){ -+ if (lastSecond - it->second < 10000){ -+ connectorSummary << (addDelimiter ? "," : "") << it->first; -+ addDelimiter = true; -+ } -+ } -+ connectorSummary << "}"; -+ -+ // Write summary to global statistics -+ sessions.setTime(time + globalTime); -+ sessions.setDown(down + globalDown); -+ sessions.setUp(up + globalUp); -+ sessions.setPacketCount(pktcount + globalPktcount); -+ sessions.setPacketLostCount(pktloss + globalPktloss); -+ sessions.setPacketRetransmitCount(pktretrans + globalPktretrans); -+ sessions.setLastSecond(lastSecond); -+ sessions.setConnector(connectorSummary.str()); -+ sessions.setNow(now); -+ -+ // Retrigger USER_NEW if a re-sync was requested -+ if (!thisType && forceTrigger){ -+ forceTrigger = false; -+ if (Triggers::shouldTrigger("USER_NEW", thisStreamName)){ -+ INFO_MSG("Triggering USER_NEW for stream %s", thisStreamName.c_str()); -+ std::string payload = thisStreamName + "\n" + thisHost + "\n" + -+ thisSid + "\n" + thisProtocol + -+ "\n" + thisReqUrl + "\n" + thisSessionId; -+ if (!Triggers::doTrigger("USER_NEW", payload, thisStreamName)){ -+ INFO_MSG("USER_NEW rejected stream %s", thisStreamName.c_str()); -+ connections.setExit(); -+ connections.finishAll(); -+ }else{ -+ INFO_MSG("USER_NEW accepted stream %s", thisStreamName.c_str()); -+ } -+ } -+ } -+ -+ // Invalidate connections if the session is marked as invalid -+ if(connections.getExit()){ -+ connections.finishAll(); -+ break; -+ } -+ // Remember latest activity so we know when this session ends -+ if (currentConnections){ -+ lastSeen = Util::bootSecs(); -+ } -+ Util::sleep(1000); -+ } -+ -+ // Trigger USER_END -+ if (!thisType && Triggers::shouldTrigger("USER_END", thisStreamName)){ -+ lastSecond = 0; -+ time = 0; -+ down = 0; -+ up = 0; -+ -+ // Get a final summary of this session -+ for (uint64_t idx = 0; idx < connections.recordCount(); idx++){ -+ if (connections.getStatus(idx) == COMM_STATUS_INVALID || connections.getStatus(idx) & COMM_STATUS_DISCONNECT){continue;} -+ uint64_t thisLastSecond = connections.getLastSecond(idx); -+ // Set last second to the latest entry -+ if (thisLastSecond > lastSecond){ -+ lastSecond = thisLastSecond; -+ } -+ // Count protocol durations across the entire session -+ std::string thisConnector = connections.getConnector(idx); -+ if (thisConnector != ""){ -+ connectorCount[thisConnector] += connections.getTime(idx); -+ } -+ // Sum all other variables -+ time += connections.getTime(idx); -+ down += connections.getDown(idx); -+ up += connections.getUp(idx); -+ } -+ -+ // Convert connector duration to string -+ std::stringstream connectorSummary; -+ bool addDelimiter = false; -+ connectorSummary << "{"; -+ for (std::map::iterator it = connectorCount.begin(); -+ it != connectorCount.end(); ++it){ -+ connectorSummary << (addDelimiter ? "," : "") << it->first << ":" << it->second; -+ addDelimiter = true; -+ } -+ connectorSummary << "}"; -+ -+ const uint64_t duration = lastSecond - (bootTime / 1000); -+ std::stringstream summary; -+ summary << thisSessionId << "\n" -+ << thisStreamName << "\n" -+ << connectorSummary.str() << "\n" -+ << thisHost << "\n" -+ << duration << "\n" -+ << up << "\n" -+ << down << "\n" -+ << sessions.getTags(); -+ Triggers::doTrigger("USER_END", summary.str(), thisStreamName); -+ } -+ -+ if (!thisType && connections.getExit()){ -+ WARN_MSG("Session %s has been invalidated since it is not allowed to view stream %s", thisSessionId.c_str(), thisStreamName.c_str()); -+ uint64_t sleepStart = Util::bootSecs(); -+ // Keep session invalidated for 10 minutes, or until the session stops -+ while (config.is_active && sleepStart - Util::bootSecs() < 600){ -+ Util::sleep(1000); -+ } -+ } -+ INFO_MSG("Shutting down session %s", thisSessionId.c_str()); -+ return 0; -+} --- -2.25.1 - - -From 8ac486b815cac711c434422a1b0955486c28bb68 Mon Sep 17 00:00:00 2001 -From: Marco van Dijk -Date: Wed, 16 Mar 2022 13:46:14 +0100 -Subject: [PATCH 27/38] Completed new sessions system - -Co-authored-by: Thulinma ---- - lib/comms.cpp | 206 ++++++++---- - lib/comms.h | 36 +- - lib/defines.h | 2 +- - lib/hls_support.cpp | 12 +- - lib/http_parser.cpp | 6 +- - lib/http_parser.h | 2 +- - lib/socket.cpp | 10 + - lib/socket.h | 1 + - lib/websocket.cpp | 22 +- - lib/websocket.h | 2 +- - src/controller/controller.cpp | 18 + - src/controller/controller_api.cpp | 11 +- - src/controller/controller_statistics.cpp | 375 +++++++++++++-------- - src/controller/controller_statistics.h | 31 +- - src/controller/controller_storage.cpp | 28 +- - src/controller/controller_storage.h | 1 + - src/input/input.cpp | 21 +- - src/input/input_rtsp.cpp | 3 +- - src/input/input_sdp.cpp | 3 +- - src/input/input_ts.cpp | 3 +- - src/input/input_tsrist.cpp | 2 +- - src/input/input_tsrist.h | 2 +- - src/output/output.cpp | 40 ++- - src/output/output.h | 3 +- - src/output/output_cmaf.cpp | 27 +- - src/output/output_hls.cpp | 28 +- - src/output/output_http.cpp | 51 ++- - src/output/output_http_internal.cpp | 179 +++++----- - src/output/output_http_internal.h | 8 +- - src/output/output_sdp.cpp | 12 + - src/output/output_sdp.h | 2 + - src/output/output_ts.cpp | 12 + - src/output/output_ts.h | 2 + - src/output/output_tsrist.cpp | 16 +- - src/output/output_tsrist.h | 4 +- - src/session.cpp | 410 +++++++++++++---------- - 36 files changed, 981 insertions(+), 610 deletions(-) - -diff --git a/lib/comms.cpp b/lib/comms.cpp -index 85f1e6d7..14faa980 100644 ---- a/lib/comms.cpp -+++ b/lib/comms.cpp -@@ -3,6 +3,7 @@ - #include "comms.h" - #include "defines.h" - #include "encode.h" -+#include "stream.h" - #include "procs.h" - #include "timing.h" - #include -@@ -10,6 +11,34 @@ - #include "config.h" - - namespace Comms{ -+ uint8_t sessionViewerMode = SESS_BUNDLE_DEFAULT_VIEWER; -+ uint8_t sessionInputMode = SESS_BUNDLE_DEFAULT_OTHER; -+ uint8_t sessionOutputMode = SESS_BUNDLE_DEFAULT_OTHER; -+ uint8_t sessionUnspecifiedMode = 0; -+ uint8_t sessionStreamInfoMode = SESS_DEFAULT_STREAM_INFO_MODE; -+ uint8_t tknMode = SESS_TKN_DEFAULT_MODE; -+ -+ /// \brief Refreshes the session configuration if the last update was more than 5 seconds ago -+ void sessionConfigCache(){ -+ static uint64_t lastUpdate = 0; -+ if (Util::bootSecs() > lastUpdate + 5){ -+ VERYHIGH_MSG("Updating session config"); -+ JSON::Value tmpVal = Util::getGlobalConfig("sessionViewerMode"); -+ if (!tmpVal.isNull()){ sessionViewerMode = tmpVal.asInt(); } -+ tmpVal = Util::getGlobalConfig("sessionInputMode"); -+ if (!tmpVal.isNull()){ sessionInputMode = tmpVal.asInt(); } -+ tmpVal = Util::getGlobalConfig("sessionOutputMode"); -+ if (!tmpVal.isNull()){ sessionOutputMode = tmpVal.asInt(); } -+ tmpVal = Util::getGlobalConfig("sessionUnspecifiedMode"); -+ if (!tmpVal.isNull()){ sessionUnspecifiedMode = tmpVal.asInt(); } -+ tmpVal = Util::getGlobalConfig("sessionStreamInfoMode"); -+ if (!tmpVal.isNull()){ sessionStreamInfoMode = tmpVal.asInt(); } -+ tmpVal = Util::getGlobalConfig("tknMode"); -+ if (!tmpVal.isNull()){ tknMode = tmpVal.asInt(); } -+ lastUpdate = Util::bootSecs(); -+ } -+ } -+ - Comms::Comms(){ - index = INVALID_RECORD_INDEX; - currentSize = 0; -@@ -17,7 +46,7 @@ namespace Comms{ - } - - Comms::~Comms(){ -- if (index != INVALID_RECORD_INDEX){ -+ if (index != INVALID_RECORD_INDEX && status){ - setStatus(COMM_STATUS_DISCONNECT | getStatus()); - } - if (master){ -@@ -123,6 +152,10 @@ namespace Comms{ - return; - } - dataAccX = Util::RelAccX(dataPage.mapped); -+ if (dataAccX.isExit()){ -+ dataPage.close(); -+ return; -+ } - fieldAccess(); - if (index == INVALID_RECORD_INDEX || reIssue){ - size_t reqCount = dataAccX.getRCount(); -@@ -170,19 +203,30 @@ namespace Comms{ - - void Sessions::addFields(){ - Connections::addFields(); -+ dataAccX.addField("tags", RAX_STRING, 512); - dataAccX.addField("sessid", RAX_STRING, 80); - } - - void Sessions::nullFields(){ - Connections::nullFields(); - setSessId(""); -+ setTags(""); - } - - void Sessions::fieldAccess(){ - Connections::fieldAccess(); -+ tags = dataAccX.getFieldAccX("tags"); - sessId = dataAccX.getFieldAccX("sessid"); - } - -+ std::string Sessions::getTags() const{return tags.string(index);} -+ std::string Sessions::getTags(size_t idx) const{return (master ? tags.string(idx) : 0);} -+ void Sessions::setTags(std::string _sid){tags.set(_sid, index);} -+ void Sessions::setTags(std::string _sid, size_t idx){ -+ if (!master){return;} -+ tags.set(_sid, idx); -+ } -+ - Users::Users() : Comms(){} - - Users::Users(const Users &rhs) : Comms(){ -@@ -251,31 +295,60 @@ namespace Comms{ - keyNum.set(_keyNum, idx); - } - -+ -+ -+ void Connections::reload(const std::string & sessId, bool _master, bool reIssue){ -+ // Open SEM_SESSION -+ if(!sem){ -+ char semName[NAME_BUFFER_SIZE]; -+ snprintf(semName, NAME_BUFFER_SIZE, SEM_SESSION, sessId.c_str()); -+ sem.open(semName, O_RDWR, ACCESSPERMS, 1); -+ if (!sem){return;} -+ } -+ char userPageName[NAME_BUFFER_SIZE]; -+ snprintf(userPageName, NAME_BUFFER_SIZE, COMMS_SESSIONS, sessId.c_str()); -+ Comms::reload(userPageName, COMMS_SESSIONS_INITSIZE, _master, reIssue); -+ } -+ - /// \brief Claims a spot on the connections page for the input/output which calls this function - /// Starts the MistSession binary for each session, which handles the statistics - /// and the USER_NEW and USER_END triggers - /// \param streamName: Name of the stream the input is providing or an output is making available to viewers - /// \param ip: IP address of the viewer which wants to access streamName. For inputs this value can be set to any value -- /// \param sid: Session ID given by the player or randomly generated -+ /// \param tkn: Session token given by the player or randomly generated - /// \param protocol: Protocol currently in use for this connection -- /// \param sessionMode: Determines how a viewer session is defined: -- // If set to 0, all connections with the same viewer IP and stream name are bundled. -- // If set to 1, all connections with the same viewer IP and player ID are bundled. -- // If set to 2, all connections with the same player ID and stream name are bundled. -- // If set to 3, all connections with the same viewer IP, player ID and stream name are bundled. - /// \param _master: If True, we are reading from this page. If False, we are writing (to our entry) on this page - /// \param reIssue: If True, claim a new entry on this page -- void Connections::reload(std::string streamName, std::string ip, std::string sid, std::string protocol, std::string reqUrl, uint64_t sessionMode, bool _master, bool reIssue){ -- if (sessionMode == 0xFFFFFFFFFFFFFFFFull){ -- FAIL_MSG("The session mode was not initialised properly. Assuming default behaviour of bundling by viewer IP, stream name and player id"); -- sessionMode = SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID; -- } -+ void Connections::reload(const std::string & streamName, const std::string & ip, const std::string & tkn, const std::string & protocol, const std::string & reqUrl, bool _master, bool reIssue){ -+ initialTkn = tkn; -+ uint8_t sessMode = sessionViewerMode; - // Generate a unique session ID for each viewer, input or output -- sessionId = generateSession(streamName, ip, sid, protocol, sessionMode); - if (protocol.size() >= 6 && protocol.substr(0, 6) == "INPUT:"){ -- sessionId = "I" + sessionId; -+ sessMode = sessionInputMode; -+ sessionId = "I" + generateSession(streamName, ip, tkn, protocol, sessMode); - }else if (protocol.size() >= 7 && protocol.substr(0, 7) == "OUTPUT:"){ -- sessionId = "O" + sessionId; -+ sessMode = sessionOutputMode; -+ sessionId = "O" + generateSession(streamName, ip, tkn, protocol, sessMode); -+ }else{ -+ // If the session only contains the HTTP connector, check sessionStreamInfoMode -+ if (protocol.size() == 4 && protocol == "HTTP"){ -+ if (sessionStreamInfoMode == SESS_HTTP_AS_VIEWER){ -+ sessionId = generateSession(streamName, ip, tkn, protocol, sessMode); -+ }else if (sessionStreamInfoMode == SESS_HTTP_AS_OUTPUT){ -+ sessMode = sessionOutputMode; -+ sessionId = "O" + generateSession(streamName, ip, tkn, protocol, sessMode); -+ }else if (sessionStreamInfoMode == SESS_HTTP_DISABLED){ -+ return; -+ }else if (sessionStreamInfoMode == SESS_HTTP_AS_UNSPECIFIED){ -+ // Set sessMode to include all variables when determining the session ID -+ sessMode = sessionUnspecifiedMode; -+ sessionId = "U" + generateSession(streamName, ip, tkn, protocol, sessMode); -+ }else{ -+ sessionId = generateSession(streamName, ip, tkn, protocol, sessMode); -+ } -+ }else{ -+ sessionId = generateSession(streamName, ip, tkn, protocol, sessMode); -+ } - } - char userPageName[NAME_BUFFER_SIZE]; - snprintf(userPageName, NAME_BUFFER_SIZE, COMMS_SESSIONS, sessionId.c_str()); -@@ -283,36 +356,59 @@ namespace Comms{ - if (!_master){ - dataPage.init(userPageName, 0, false, false); - if (!dataPage){ -+ std::string host; -+ Socket::hostBytesToStr(ip.data(), 16, host); - pid_t thisPid; - std::deque args; - args.push_back(Util::getMyPath() + "MistSession"); - args.push_back(sessionId); -- args.push_back("--sessionmode"); -- args.push_back(JSON::Value(sessionMode).asString()); -- args.push_back("--streamname"); -- args.push_back(streamName); -- args.push_back("--ip"); -- args.push_back(ip); -- args.push_back("--sid"); -- args.push_back(sid); -- args.push_back("--protocol"); -- args.push_back(protocol); -- args.push_back("--requrl"); -- args.push_back(reqUrl); -+ -+ // First bit defines whether to include stream name -+ if (sessMode & 0x08){ -+ args.push_back("--streamname"); -+ args.push_back(streamName); -+ }else{ -+ setenv("SESSION_STREAM", streamName.c_str(), 1); -+ } -+ // Second bit defines whether to include viewer ip -+ if (sessMode & 0x04){ -+ args.push_back("--ip"); -+ args.push_back(host); -+ }else{ -+ setenv("SESSION_IP", host.c_str(), 1); -+ } -+ // Third bit defines whether to include tkn -+ if (sessMode & 0x02){ -+ args.push_back("--tkn"); -+ args.push_back(tkn); -+ }else{ -+ setenv("SESSION_TKN", tkn.c_str(), 1); -+ } -+ // Fourth bit defines whether to include protocol -+ if (sessMode & 0x01){ -+ args.push_back("--protocol"); -+ args.push_back(protocol); -+ }else{ -+ setenv("SESSION_PROTOCOL", protocol.c_str(), 1); -+ } -+ setenv("SESSION_REQURL", reqUrl.c_str(), 1); - int err = fileno(stderr); - thisPid = Util::Procs::StartPiped(args, 0, 0, &err); - Util::Procs::forget(thisPid); -- HIGH_MSG("Spawned new session executeable (pid %u) for sessionId '%s', corresponding to host %s and stream %s", thisPid, sessionId.c_str(), ip.c_str(), streamName.c_str()); -+ unsetenv("SESSION_STREAM"); -+ unsetenv("SESSION_IP"); -+ unsetenv("SESSION_TKN"); -+ unsetenv("SESSION_PROTOCOL"); -+ unsetenv("SESSION_REQURL"); - } - } -- // Open SEM_SESSION -- if(!sem){ -- char semName[NAME_BUFFER_SIZE]; -- snprintf(semName, NAME_BUFFER_SIZE, SEM_SESSION, sessionId.c_str()); -- sem.open(semName, O_RDWR, ACCESSPERMS, 1); -+ reload(sessionId, _master, reIssue); -+ if (index != INVALID_RECORD_INDEX){ -+ setConnector(protocol); -+ setHost(ip); -+ setStream(streamName); -+ VERYHIGH_MSG("Reloading connection. Claimed record %lu", index); - } -- Comms::reload(userPageName, COMMS_SESSIONS_INITSIZE, _master, reIssue); -- VERYHIGH_MSG("Reloading connection. Claimed record %lu", index); - } - - /// \brief Marks the data page as closed, so that we longer write any new data to is -@@ -341,7 +437,6 @@ namespace Comms{ - dataAccX.addField("host", RAX_RAW, 16); - dataAccX.addField("stream", RAX_STRING, 100); - dataAccX.addField("connector", RAX_STRING, 20); -- dataAccX.addField("tags", RAX_STRING, 512); - dataAccX.addField("pktcount", RAX_64UINT); - dataAccX.addField("pktloss", RAX_64UINT); - dataAccX.addField("pktretrans", RAX_64UINT); -@@ -349,7 +444,6 @@ namespace Comms{ - - void Connections::nullFields(){ - Comms::nullFields(); -- setTags(""); - setConnector(""); - setStream(""); - setHost(""); -@@ -373,7 +467,6 @@ namespace Comms{ - host = dataAccX.getFieldAccX("host"); - stream = dataAccX.getFieldAccX("stream"); - connector = dataAccX.getFieldAccX("connector"); -- tags = dataAccX.getFieldAccX("tags"); - pktcount = dataAccX.getFieldAccX("pktcount"); - pktloss = dataAccX.getFieldAccX("pktloss"); - pktretrans = dataAccX.getFieldAccX("pktretrans"); -@@ -461,14 +554,6 @@ namespace Comms{ - return false; - } - -- std::string Connections::getTags() const{return tags.string(index);} -- std::string Connections::getTags(size_t idx) const{return (master ? tags.string(idx) : 0);} -- void Connections::setTags(std::string _sid){tags.set(_sid, index);} -- void Connections::setTags(std::string _sid, size_t idx){ -- if (!master){return;} -- tags.set(_sid, idx); -- } -- - uint64_t Connections::getPacketCount() const{return pktcount.uint(index);} - uint64_t Connections::getPacketCount(size_t idx) const{ - return (master ? pktcount.uint(idx) : 0); -@@ -501,31 +586,32 @@ namespace Comms{ - - /// \brief Generates a session ID which is unique per viewer - /// \return generated session ID as string -- std::string Connections::generateSession(std::string streamName, std::string ip, std::string sid, std::string connector, uint64_t sessionMode){ -+ std::string Connections::generateSession(const std::string & streamName, const std::string & ip, const std::string & tkn, const std::string & connector, uint64_t sessionMode){ - std::string concat; -+ std::string debugMsg = "Generating session id based on"; - // First bit defines whether to include stream name -- if (sessionMode > 7){ -+ if (sessionMode & 0x08){ - concat += streamName; -- sessionMode -= 8; -+ debugMsg += " stream name '" + streamName + "'"; - } - // Second bit defines whether to include viewer ip -- if (sessionMode > 3){ -+ if (sessionMode & 0x04){ - concat += ip; -- sessionMode -= 4; -+ std::string ipHex; -+ Socket::hostBytesToStr(ip.c_str(), ip.size(), ipHex); -+ debugMsg += " IP '" + ipHex + "'"; - } -- // Third bit defines whether to include player ip -- if (sessionMode > 1){ -- concat += sid; -- sessionMode -= 2; -+ // Third bit defines whether to include client-side session token -+ if (sessionMode & 0x02){ -+ concat += tkn; -+ debugMsg += " session token '" + tkn + "'"; - } - // Fourth bit defines whether to include protocol -- if (sessionMode == 1){ -+ if (sessionMode & 0x01){ - concat += connector; -- sessionMode = 0; -- } -- if (sessionMode > 0){ -- WARN_MSG("Could not resolve session mode of value %lu", sessionMode); -+ debugMsg += " protocol '" + connector + "'"; - } -+ VERYHIGH_MSG("%s", debugMsg.c_str()); - return Secure::sha256(concat.c_str(), concat.length()); - } - }// namespace Comms -diff --git a/lib/comms.h b/lib/comms.h -index 9a5c0ea9..ec36dcb0 100644 ---- a/lib/comms.h -+++ b/lib/comms.h -@@ -9,13 +9,21 @@ - #define COMM_STATUS_REQDISCONNECT 0x10 - #define COMM_STATUS_ACTIVE 0x1 - #define COMM_STATUS_INVALID 0x0 -+#define SESS_BUNDLE_DEFAULT_VIEWER 14 -+#define SESS_BUNDLE_DEFAULT_OTHER 15 -+#define SESS_DEFAULT_STREAM_INFO_MODE 1 -+#define SESS_HTTP_AS_VIEWER 1 -+#define SESS_HTTP_AS_OUTPUT 2 -+#define SESS_HTTP_DISABLED 3 -+#define SESS_HTTP_AS_UNSPECIFIED 4 -+#define SESS_TKN_DEFAULT_MODE 15 - - - #define COMM_LOOP(comm, onActive, onDisconnect) \ - {\ - for (size_t id = 0; id < comm.recordCount(); id++){\ - if (comm.getStatus(id) == COMM_STATUS_INVALID){continue;}\ -- if (!Util::Procs::isRunning(comm.getPid(id))){\ -+ if (!(comm.getStatus(id) & COMM_STATUS_DISCONNECT) && comm.getPid(id) && !Util::Procs::isRunning(comm.getPid(id))){\ - comm.setStatus(COMM_STATUS_DISCONNECT | comm.getStatus(id), id);\ - }\ - onActive;\ -@@ -27,6 +35,14 @@ - } - - namespace Comms{ -+ extern uint8_t sessionViewerMode; -+ extern uint8_t sessionInputMode; -+ extern uint8_t sessionOutputMode; -+ extern uint8_t sessionUnspecifiedMode; -+ extern uint8_t sessionStreamInfoMode; -+ extern uint8_t tknMode; -+ void sessionConfigCache(); -+ - class Comms{ - public: - Comms(); -@@ -66,11 +82,13 @@ namespace Comms{ - - class Connections : public Comms{ - public: -- void reload(std::string streamName, std::string ip, std::string sid, std::string protocol, std::string reqUrl, uint64_t sessionMode, bool _master = false, bool reIssue = false); -+ void reload(const std::string & streamName, const std::string & ip, const std::string & tkn, const std::string & protocol, const std::string & reqUrl, bool _master = false, bool reIssue = false); -+ void reload(const std::string & sessId, bool _master = false, bool reIssue = false); - void unload(); - operator bool() const{return dataPage.mapped && (master || index != INVALID_RECORD_INDEX);} -- std::string generateSession(std::string streamName, std::string ip, std::string sid, std::string connector, uint64_t sessionMode); -+ std::string generateSession(const std::string & streamName, const std::string & ip, const std::string & tkn, const std::string & connector, uint64_t sessionMode); - std::string sessionId; -+ std::string initialTkn; - - void setExit(); - bool getExit(); -@@ -79,6 +97,8 @@ namespace Comms{ - virtual void nullFields(); - virtual void fieldAccess(); - -+ const std::string & getTkn() const{return initialTkn;} -+ - uint64_t getNow() const; - uint64_t getNow(size_t idx) const; - void setNow(uint64_t _now); -@@ -120,11 +140,6 @@ namespace Comms{ - void setConnector(std::string _connector, size_t idx); - bool hasConnector(size_t idx, std::string protocol); - -- std::string getTags() const; -- std::string getTags(size_t idx) const; -- void setTags(std::string _sid); -- void setTags(std::string _sid, size_t idx); -- - uint64_t getPacketCount() const; - uint64_t getPacketCount(size_t idx) const; - void setPacketCount(uint64_t _count); -@@ -197,5 +212,10 @@ namespace Comms{ - virtual void addFields(); - virtual void nullFields(); - virtual void fieldAccess(); -+ -+ std::string getTags() const; -+ std::string getTags(size_t idx) const; -+ void setTags(std::string _sid); -+ void setTags(std::string _sid, size_t idx); - }; - }// namespace Comms -diff --git a/lib/defines.h b/lib/defines.h -index 203b75db..d08d1d04 100644 ---- a/lib/defines.h -+++ b/lib/defines.h -@@ -231,7 +231,7 @@ static inline void show_stackframe(){} - #define SEM_TRACKLIST "/MstTRKS%s" //%s stream name - #define SEM_SESSION "MstSess%s" - #define SEM_SESSCACHE "/MstSessCacheLock" --#define SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID 14 -+#define SESS_TIMEOUT 600 // Session timeout in seconds - #define SHM_CAPA "MstCapa" - #define SHM_PROTO "MstProt" - #define SHM_PROXY "MstProx" -diff --git a/lib/hls_support.cpp b/lib/hls_support.cpp -index c311a7ac..130e691d 100644 ---- a/lib/hls_support.cpp -+++ b/lib/hls_support.cpp -@@ -274,7 +274,7 @@ namespace HLS{ - if (trackData.mediaFormat == ".ts"){return;} - - result << "#EXT-X-MAP:URI=\"" << trackData.urlPrefix << "init" << trackData.mediaFormat; -- if (trackData.sessionId.size()){result << "?sessId=" << trackData.sessionId;} -+ if (trackData.sessionId.size()){result << "?tkn=" << trackData.sessionId;} - result << "\"\r\n"; - } - -@@ -327,7 +327,7 @@ namespace HLS{ - result << "?msn=" << fragData.currentFrag; - result << "&mTrack=" << trackData.timingTrackId; - result << "&dur=" << fragData.duration; -- if (trackData.sessionId.size()){result << "&sessId=" << trackData.sessionId;} -+ if (trackData.sessionId.size()){result << "&tkn=" << trackData.sessionId;} - result << "\r\n"; - } - -@@ -341,7 +341,7 @@ namespace HLS{ - result << "?msn=" << fragData.currentFrag; - result << "&mTrack=" << trackData.timingTrackId; - result << "&dur=" << duration; -- if (trackData.sessionId.size()){result << "&sessId=" << trackData.sessionId;} -+ if (trackData.sessionId.size()){result << "&tkn=" << trackData.sessionId;} - result << "\""; - - // NOTE: INDEPENDENT tags, specified ONLY for VIDEO tracks, indicate the first partial fragment -@@ -448,7 +448,7 @@ namespace HLS{ - result << "?msn=" << fragData.currentFrag - 1; - result << "&mTrack=" << trackData.timingTrackId; - result << "&dur=" << partDurationMaxMs; -- if (trackData.sessionId.size()){result << "&sessId=" << trackData.sessionId;} -+ if (trackData.sessionId.size()){result << "&tkn=" << trackData.sessionId;} - result << "\"\r\n"; - } - -@@ -509,7 +509,7 @@ namespace HLS{ - result << ",NAME=\"" << name << "\",URI=\"" << trackId << "/index.m3u8"; - result << "?mTrack=" << masterData.mainTrack; - result << "&iMsn=" << iFrag; -- if (masterData.hasSessId){result << "&sessId=" << masterData.sessId;} -+ if (masterData.sessId.size()){result << "&tkn=" << masterData.sessId;} - if (masterData.noLLHLS){result << "&llhls=0";} - result << "\"\r\n"; - } -@@ -529,7 +529,7 @@ namespace HLS{ - result << "/index.m3u8"; - result << "?mTrack=" << masterData.mainTrack; - result << "&iMsn=" << iFrag; -- if (masterData.hasSessId){result << "&sessId=" << masterData.sessId;} -+ if (masterData.sessId.size()){result << "&tkn=" << masterData.sessId;} - if (masterData.noLLHLS){result << "&llhls=0";} - result << "\r\n"; - } -diff --git a/lib/http_parser.cpp b/lib/http_parser.cpp -index c08e3d56..113fdb51 100644 ---- a/lib/http_parser.cpp -+++ b/lib/http_parser.cpp -@@ -742,13 +742,13 @@ bool HTTP::Parser::parse(std::string &HTTPbuffer, Util::DataCallback &cb){ - - /// HTTP variable parser to std::map structure. - /// Reads variables from data, decodes and stores them to storage. --void HTTP::parseVars(const std::string &data, std::map &storage){ -+void HTTP::parseVars(const std::string &data, std::map &storage, const std::string & separator){ - std::string varname; - std::string varval; - // position where a part starts (e.g. after &) - size_t pos = 0; - while (pos < data.length()){ -- size_t nextpos = data.find('&', pos); -+ size_t nextpos = data.find(separator, pos); - if (nextpos == std::string::npos){nextpos = data.length();} - size_t eq_pos = data.find('=', pos); - if (eq_pos < nextpos){ -@@ -769,7 +769,7 @@ void HTTP::parseVars(const std::string &data, std::map - break; - } - // erase & -- pos = nextpos + 1; -+ pos = nextpos + separator.size(); - } - } - -diff --git a/lib/http_parser.h b/lib/http_parser.h -index 9c843f20..a5528df3 100644 ---- a/lib/http_parser.h -+++ b/lib/http_parser.h -@@ -14,7 +14,7 @@ namespace HTTP{ - - /// HTTP variable parser to std::map structure. - /// Reads variables from data, decodes and stores them to storage. -- void parseVars(const std::string &data, std::map &storage); -+ void parseVars(const std::string &data, std::map &storage, const std::string & separator = "&"); - - /// Simple class for reading and writing HTTP 1.0 and 1.1. - class Parser : public Util::DataCallback{ -diff --git a/lib/socket.cpp b/lib/socket.cpp -index 6d63af00..47b2b6bd 100644 ---- a/lib/socket.cpp -+++ b/lib/socket.cpp -@@ -166,6 +166,8 @@ bool Socket::isBinAddress(const std::string &binAddr, std::string addr){ - /// Converts the given address with optional subnet to binary IPv6 form. - /// Returns 16 bytes of address, followed by 1 byte of subnet bits, zero or more times. - std::string Socket::getBinForms(std::string addr){ -+ // Check for empty address -+ if (!addr.size()){return std::string(17, (char)0);} - // Check if we need to do prefix matching - uint8_t prefixLen = 128; - if (addr.find('/') != std::string::npos){ -@@ -1796,6 +1798,14 @@ void Socket::UDPConnection::GetDestination(std::string &destIp, uint32_t &port){ - FAIL_MSG("Could not get destination for UDP socket"); - }// Socket::UDPConnection GetDestination - -+/// Gets the properties of the receiving end of this UDP socket. -+/// This will be the receiving end for all SendNow calls. -+std::string Socket::UDPConnection::getBinDestination(){ -+ std::string binList = getIPv6BinAddr(*(sockaddr_in6*)destAddr); -+ if (binList.size() < 16){ return std::string("\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000", 16); } -+ return binList.substr(0, 16); -+}// Socket::UDPConnection GetDestination -+ - /// Returns the port number of the receiving end of this socket. - /// Returns 0 on error. - uint32_t Socket::UDPConnection::getDestPort() const{ -diff --git a/lib/socket.h b/lib/socket.h -index a85fbf39..b369ac84 100644 ---- a/lib/socket.h -+++ b/lib/socket.h -@@ -215,6 +215,7 @@ namespace Socket{ - void setBlocking(bool blocking); - void SetDestination(std::string hostname, uint32_t port); - void GetDestination(std::string &hostname, uint32_t &port); -+ std::string getBinDestination(); - const void * getDestAddr(){return destAddr;} - size_t getDestAddrLen(){return destAddr_size;} - std::string getBoundAddress(); -diff --git a/lib/websocket.cpp b/lib/websocket.cpp -index 05703b55..c82b4cba 100644 ---- a/lib/websocket.cpp -+++ b/lib/websocket.cpp -@@ -73,31 +73,31 @@ namespace HTTP{ - } - - /// Takes an incoming HTTP::Parser request for a Websocket, and turns it into one. -- Websocket::Websocket(Socket::Connection &c, HTTP::Parser &h) : C(c){ -+ Websocket::Websocket(Socket::Connection &c, const HTTP::Parser &req, HTTP::Parser &resp) : C(c){ - frameType = 0; - maskOut = false; -- std::string connHeader = h.GetHeader("Connection"); -+ std::string connHeader = req.GetHeader("Connection"); - Util::stringToLower(connHeader); - if (connHeader.find("upgrade") == std::string::npos){ - FAIL_MSG("Could not negotiate websocket, connection header incorrect (%s).", connHeader.c_str()); - C.close(); - return; - } -- std::string upgradeHeader = h.GetHeader("Upgrade"); -+ std::string upgradeHeader = req.GetHeader("Upgrade"); - Util::stringToLower(upgradeHeader); - if (upgradeHeader != "websocket"){ - FAIL_MSG("Could not negotiate websocket, upgrade header incorrect (%s).", upgradeHeader.c_str()); - C.close(); - return; - } -- if (h.GetHeader("Sec-WebSocket-Version") != "13"){ -+ if (req.GetHeader("Sec-WebSocket-Version") != "13"){ - FAIL_MSG("Could not negotiate websocket, version incorrect (%s).", -- h.GetHeader("Sec-WebSocket-Version").c_str()); -+ req.GetHeader("Sec-WebSocket-Version").c_str()); - C.close(); - return; - } - #ifdef SSL -- std::string client_key = h.GetHeader("Sec-WebSocket-Key"); -+ std::string client_key = req.GetHeader("Sec-WebSocket-Key"); - if (!client_key.size()){ - FAIL_MSG("Could not negotiate websocket, missing key!"); - C.close(); -@@ -105,15 +105,13 @@ namespace HTTP{ - } - #endif - -- h.Clean(); -- h.setCORSHeaders(); -- h.SetHeader("Upgrade", "websocket"); -- h.SetHeader("Connection", "Upgrade"); -+ resp.SetHeader("Upgrade", "websocket"); -+ resp.SetHeader("Connection", "Upgrade"); - #ifdef SSL -- h.SetHeader("Sec-WebSocket-Accept", calculateKeyAccept(client_key)); -+ resp.SetHeader("Sec-WebSocket-Accept", calculateKeyAccept(client_key)); - #endif - // H.SetHeader("Sec-WebSocket-Protocol", "json"); -- h.SendResponse("101", "Websocket away!", C); -+ resp.SendResponse("101", "Websocket away!", C); - } - - /// Loops calling readFrame until the connection is closed, sleeping in between reads if needed. -diff --git a/lib/websocket.h b/lib/websocket.h -index 819f18c9..07861a19 100644 ---- a/lib/websocket.h -+++ b/lib/websocket.h -@@ -7,7 +7,7 @@ - namespace HTTP{ - class Websocket{ - public: -- Websocket(Socket::Connection &c, HTTP::Parser &h); -+ Websocket(Socket::Connection &c, const HTTP::Parser &req, HTTP::Parser &resp); - Websocket(Socket::Connection &c, const HTTP::URL & url, std::map * headers = 0); - Websocket(Socket::Connection &c, bool client); - operator bool() const; -diff --git a/src/controller/controller.cpp b/src/controller/controller.cpp -index 9e17ca6b..972c7ed7 100644 ---- a/src/controller/controller.cpp -+++ b/src/controller/controller.cpp -@@ -309,6 +309,24 @@ int main_loop(int argc, char **argv){ - Controller::Storage["config"]["prometheus"] = Controller::conf.getString("prometheus"); - Controller::Storage["config"]["accesslog"] = Controller::conf.getString("accesslog"); - Controller::normalizeTrustedProxies(Controller::Storage["config"]["trustedproxy"]); -+ if (!Controller::Storage["config"]["sessionViewerMode"]){ -+ Controller::Storage["config"]["sessionViewerMode"] = SESS_BUNDLE_DEFAULT_VIEWER; -+ } -+ if (!Controller::Storage["config"]["sessionInputMode"]){ -+ Controller::Storage["config"]["sessionInputMode"] = SESS_BUNDLE_DEFAULT_OTHER; -+ } -+ if (!Controller::Storage["config"]["sessionOutputMode"]){ -+ Controller::Storage["config"]["sessionOutputMode"] = SESS_BUNDLE_DEFAULT_OTHER; -+ } -+ if (!Controller::Storage["config"]["sessionUnspecifiedMode"]){ -+ Controller::Storage["config"]["sessionUnspecifiedMode"] = 0; -+ } -+ if (!Controller::Storage["config"]["sessionStreamInfoMode"]){ -+ Controller::Storage["config"]["sessionStreamInfoMode"] = SESS_DEFAULT_STREAM_INFO_MODE; -+ } -+ if (!Controller::Storage["config"].isMember("tknMode")){ -+ Controller::Storage["config"]["tknMode"] = SESS_TKN_DEFAULT_MODE; -+ } - Controller::prometheus = Controller::Storage["config"]["prometheus"].asStringRef(); - Controller::accesslog = Controller::Storage["config"]["accesslog"].asStringRef(); - Controller::writeConfig(); -diff --git a/src/controller/controller_api.cpp b/src/controller/controller_api.cpp -index 02491065..4e935aba 100644 ---- a/src/controller/controller_api.cpp -+++ b/src/controller/controller_api.cpp -@@ -188,7 +188,9 @@ void Controller::handleWebSocket(HTTP::Parser &H, Socket::Connection &C){ - std::string logs = H.GetVar("logs"); - std::string accs = H.GetVar("accs"); - bool doStreams = H.GetVar("streams").size(); -- HTTP::Websocket W(C, H); -+ HTTP::Parser req = H; -+ H.Clean(); -+ HTTP::Websocket W(C, req, H); - if (!W){return;} - - IPC::sharedPage shmLogs(SHM_STATE_LOGS, 1024 * 1024); -@@ -594,7 +596,12 @@ void Controller::handleAPICommands(JSON::Value &Request, JSON::Value &Response){ - out["prometheus"] = in["prometheus"]; - Controller::prometheus = out["prometheus"].asStringRef(); - } -- if (in.isMember("sessionMode")){out["sessionMode"] = in["sessionMode"];} -+ if (in.isMember("sessionViewerMode")){out["sessionViewerMode"] = in["sessionViewerMode"];} -+ if (in.isMember("sessionInputMode")){out["sessionInputMode"] = in["sessionInputMode"];} -+ if (in.isMember("sessionOutputMode")){out["sessionOutputMode"] = in["sessionOutputMode"];} -+ if (in.isMember("sessionUnspecifiedMode")){out["sessionUnspecifiedMode"] = in["sessionUnspecifiedMode"];} -+ if (in.isMember("sessionStreamInfoMode")){out["sessionStreamInfoMode"] = in["sessionStreamInfoMode"];} -+ if (in.isMember("tknMode")){out["tknMode"] = in["tknMode"];} - if (in.isMember("defaultStream")){out["defaultStream"] = in["defaultStream"];} - if (in.isMember("location") && in["location"].isObject()){ - out["location"]["lat"] = in["location"]["lat"].asDouble(); -diff --git a/src/controller/controller_statistics.cpp b/src/controller/controller_statistics.cpp -index 6e51f4ee..ffb89897 100644 ---- a/src/controller/controller_statistics.cpp -+++ b/src/controller/controller_statistics.cpp -@@ -58,6 +58,11 @@ static uint64_t cpu_use = 0; - char noBWCountMatches[1717]; - uint64_t bwLimit = 128 * 1024 * 1024; // gigabit default limit - -+const char nullAddress[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; -+static Controller::statLog emptyLogEntry = {0, 0, 0, 0, 0, 0 ,0 ,0, "", nullAddress, ""}; -+bool notEmpty(const Controller::statLog & dta){ -+ return dta.time || dta.firstActive || dta.lastSecond || dta.down || dta.up || dta.streamName.size() || dta.connectors.size(); -+} - - // For server-wide totals. Local to this file only. - struct streamTotals{ -@@ -66,9 +71,11 @@ struct streamTotals{ - uint64_t inputs; - uint64_t outputs; - uint64_t viewers; -+ uint64_t unspecified; - uint64_t currIns; - uint64_t currOuts; - uint64_t currViews; -+ uint64_t currUnspecified; - uint8_t status; - uint64_t viewSeconds; - uint64_t packSent; -@@ -84,6 +91,7 @@ static uint64_t servDownBytes = 0; - static uint64_t servUpOtherBytes = 0; - static uint64_t servDownOtherBytes = 0; - static uint64_t servInputs = 0; -+static uint64_t servUnspecified = 0; - static uint64_t servOutputs = 0; - static uint64_t servViewers = 0; - static uint64_t servSeconds = 0; -@@ -95,17 +103,19 @@ static uint64_t viewSecondsTotal = 0; - // Mapping of streamName -> summary of stream-wide statistics - static std::map streamStats; - --// If sessId does not exist yet in streamStats, create and init an entry for it --static void createEmptyStatsIfNeeded(const std::string & sessId){ -- if (streamStats.count(sessId)){return;} -- streamTotals & sT = streamStats[sessId]; -+// If streamName does not exist yet in streamStats, create and init an entry for it -+static void createEmptyStatsIfNeeded(const std::string & streamName){ -+ if (streamStats.count(streamName)){return;} -+ streamTotals & sT = streamStats[streamName]; - sT.upBytes = 0; - sT.downBytes = 0; - sT.inputs = 0; - sT.outputs = 0; - sT.viewers = 0; -+ sT.unspecified = 0; - sT.currIns = 0; - sT.currOuts = 0; -+ sT.currUnspecified = 0; - sT.currViews = 0; - sT.status = 0; - sT.viewSeconds = 0; -@@ -335,15 +345,23 @@ void Controller::SharedMemStats(void *config){ - it->second.currViews = 0; - it->second.currIns = 0; - it->second.currOuts = 0; -+ it->second.currUnspecified = 0; - } - } - // wipe old statistics and set session type counters - if (sessions.size()){ - std::list mustWipe; -- uint64_t cutOffPoint = Util::bootSecs() - STAT_CUTOFF; -+ // Ensure cutOffPoint is either time of boot or 10 minutes ago, whichever is closer. -+ // Prevents wrapping around to high values close to system boot time. -+ uint64_t cutOffPoint = Util::bootSecs(); -+ if (cutOffPoint > STAT_CUTOFF){ -+ cutOffPoint -= STAT_CUTOFF; -+ }else{ -+ cutOffPoint = 0; -+ } - for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ - // This part handles ending sessions, keeping them in cache for now -- if (it->second.getEnd() < cutOffPoint && it->second.newestDataPoint() < cutOffPoint){ -+ if (it->second.getEnd() < cutOffPoint){ - viewSecondsTotal += it->second.getConnTime(); - mustWipe.push_back(it->first); - // Don't count this session as a viewer -@@ -353,19 +371,24 @@ void Controller::SharedMemStats(void *config){ - switch (it->second.getSessType()){ - case SESS_UNSET: break; - case SESS_VIEWER: -- if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){ -- streamStats[it->first].currViews++; -+ if (it->second.hasDataFor(tOut)){ -+ streamStats[it->second.getStreamName()].currViews++; - } - servSeconds += it->second.getConnTime(); - break; - case SESS_INPUT: -- if (it->second.hasDataFor(tIn) && it->second.isViewerOn(tIn)){ -- streamStats[it->first].currIns++; -+ if (it->second.hasDataFor(tIn)){ -+ streamStats[it->second.getStreamName()].currIns++; - } - break; - case SESS_OUTPUT: -- if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){ -- streamStats[it->first].currOuts++; -+ if (it->second.hasDataFor(tOut)){ -+ streamStats[it->second.getStreamName()].currOuts++; -+ } -+ break; -+ case SESS_UNSPECIFIED: -+ if (it->second.hasDataFor(tOut)){ -+ streamStats[it->second.getStreamName()].currUnspecified++; - } - break; - } -@@ -406,6 +429,7 @@ void Controller::SharedMemStats(void *config){ - strmStats->setInt("viewers", it->second.currViews, strmPos); - strmStats->setInt("inputs", it->second.currIns, strmPos); - strmStats->setInt("outputs", it->second.currOuts, strmPos); -+ strmStats->setInt("unspecified", it->second.currUnspecified, strmPos); - ++strmPos; - } - } -@@ -489,47 +513,46 @@ void Controller::killConnections(std::string sessId){ - - /// Updates the given active connection with new stats data. - void Controller::statSession::update(uint64_t index, Comms::Sessions &statComm){ -- if (host == ""){ -- Socket::hostBytesToStr(statComm.getHost(index).data(), 16, host); -- } -- if (streamName == ""){ -- streamName = statComm.getStream(index); -- } -- if (curConnector == ""){ -- curConnector = statComm.getConnector(index); -- } - if (sessId == ""){ - sessId = statComm.getSessId(index); - } -- // Export tags to session -- if (tags.size()){ -- std::stringstream tagStream; -- for (std::set::iterator it = tags.begin(); it != tags.end(); ++it){ -- tagStream << "[" << *it << "]"; -+ -+ if (sessionType == SESS_UNSET){ -+ if (sessId[0] == 'I'){ -+ sessionType = SESS_INPUT; -+ }else if (sessId[0] == 'O'){ -+ sessionType = SESS_OUTPUT; -+ }else if (sessId[0] == 'U'){ -+ sessionType = SESS_UNSPECIFIED; -+ }else{ -+ sessionType = SESS_VIEWER; - } -- statComm.setTags(tagStream.str(), index); - } - -+ uint64_t prevNow = curData.log.size() ? curData.log.rbegin()->first : 0; -+ // only parse last received data, if newer -+ if (prevNow > statComm.getNow(index)){return;}; - long long prevDown = getDown(); - long long prevUp = getUp(); - uint64_t prevPktSent = getPktCount(); - uint64_t prevPktLost = getPktLost(); - uint64_t prevPktRetrans = getPktRetransmit(); -+ uint64_t prevFirstActive = getFirstActive(); -+ - curData.update(statComm, index); -- // store timestamp of first received data, if older -- if (firstSec > statComm.getNow(index)){firstSec = statComm.getNow(index);} -- uint64_t secIncr = 0; -- // store timestamp of last received data, if newer -- if (statComm.getNow(index) > lastSec){ -- lastSec = statComm.getNow(index); -- if (!tracked){ -- tracked = true; -- firstActive = firstSec; -- }else{ -- secIncr = (statComm.getNow(index) - lastSec); -+ const std::string& streamName = getStreamName(); -+ // Export tags to session -+ if (tags.size()){ -+ std::stringstream tagStream; -+ for (std::set::iterator it = tags.begin(); it != tags.end(); ++it){ -+ tagStream << "[" << *it << "]"; - } -- lastSec = statComm.getNow(index); -+ statComm.setTags(tagStream.str(), index); -+ } else { -+ statComm.setTags("", index); - } -+ -+ uint64_t secIncr = prevFirstActive ? (statComm.getNow(index) - prevNow) : 0; - long long currDown = getDown(); - long long currUp = getUp(); - uint64_t currPktSent = getPktCount(); -@@ -537,7 +560,7 @@ void Controller::statSession::update(uint64_t index, Comms::Sessions &statComm){ - uint64_t currPktRetrans = getPktRetransmit(); - if (currUp - prevUp < 0 || currDown - prevDown < 0){ - INFO_MSG("Negative data usage! %lldu/%lldd (u%lld->%lld) in %s over %s, #%" PRIu64, currUp - prevUp, -- currDown - prevDown, prevUp, currUp, streamName.c_str(), curConnector.c_str(), index); -+ currDown - prevDown, prevUp, currUp, streamName.c_str(), curData.log.rbegin()->second.connectors.c_str(), index); - }else{ - if (!noBWCount){ - size_t bwMatchOffset = 0; -@@ -567,40 +590,38 @@ void Controller::statSession::update(uint64_t index, Comms::Sessions &statComm){ - servPackRetrans += currPktRetrans - prevPktRetrans; - } - } -- if (sessionType == SESS_UNSET){ -- if (curConnector.size() >= 5 && curConnector.substr(0, 5) == "INPUT"){ -- ++servInputs; -- createEmptyStatsIfNeeded(streamName); -- streamStats[streamName].inputs++; -- streamStats[streamName].currIns++; -- sessionType = SESS_INPUT; -- }else if (curConnector.size() >= 6 && curConnector.substr(0, 6) == "OUTPUT"){ -- ++servOutputs; -- createEmptyStatsIfNeeded(streamName); -- streamStats[streamName].outputs++; -- streamStats[streamName].currOuts++; -- sessionType = SESS_OUTPUT; -- }else{ -- ++servViewers; -- createEmptyStatsIfNeeded(streamName); -- streamStats[streamName].viewers++; -- streamStats[streamName].currViews++; -- sessionType = SESS_VIEWER; -+ if (!prevFirstActive && streamName.size()){ -+ createEmptyStatsIfNeeded(streamName); -+ switch(sessionType){ -+ case SESS_INPUT: -+ ++servInputs; -+ streamStats[streamName].inputs++; -+ break; -+ case SESS_OUTPUT: -+ ++servOutputs; -+ streamStats[streamName].outputs++; -+ break; -+ case SESS_VIEWER: -+ ++servViewers; -+ streamStats[streamName].viewers++; -+ break; -+ case SESS_UNSPECIFIED: -+ ++servUnspecified; -+ streamStats[streamName].unspecified++; -+ break; -+ case SESS_UNSET: -+ break; - } - } - // Only count connections that are countable - if (noBWCount != 2){ -- if (!streamName.size() || streamName[0] == 0){ -- if (streamStats.count(streamName)){streamStats.erase(streamName);} -- }else{ -- createEmptyStatsIfNeeded(streamName); -- streamStats[streamName].upBytes += currUp - prevUp; -- streamStats[streamName].downBytes += currDown - prevDown; -- streamStats[streamName].packSent += currPktSent - prevPktSent; -- streamStats[streamName].packLoss += currPktLost - prevPktLost; -- streamStats[streamName].packRetrans += currPktRetrans - prevPktRetrans; -- if (sessionType == SESS_VIEWER){streamStats[streamName].viewSeconds += secIncr;} -- } -+ createEmptyStatsIfNeeded(streamName); -+ streamStats[streamName].upBytes += currUp - prevUp; -+ streamStats[streamName].downBytes += currDown - prevDown; -+ streamStats[streamName].packSent += currPktSent - prevPktSent; -+ streamStats[streamName].packLoss += currPktLost - prevPktLost; -+ streamStats[streamName].packRetrans += currPktRetrans - prevPktRetrans; -+ if (sessionType == SESS_VIEWER){streamStats[streamName].viewSeconds += secIncr;} - } - } - -@@ -608,21 +629,10 @@ Controller::sessType Controller::statSession::getSessType(){ - return sessionType; - } - --Controller::statSession::~statSession(){ -- if (!tracked){return;} -- switch (sessionType){ -- case SESS_INPUT: -- if (streamStats.count(streamName) && streamStats[streamName].currIns){streamStats[streamName].currIns--;} -- break; -- case SESS_OUTPUT: -- if (streamStats.count(streamName) && streamStats[streamName].currOuts){streamStats[streamName].currOuts--;} -- break; -- case SESS_VIEWER: -- if (streamStats.count(streamName) && streamStats[streamName].currViews){streamStats[streamName].currViews--;} -- break; -- default: break; -- } -- uint64_t duration = lastSec - firstActive; -+/// Ends the currently active session by inserting a null datapoint one second after the last datapoint -+void Controller::statSession::finish(){ -+ if (!getFirstActive()){return;} -+ uint64_t duration = getEnd() - getFirstActive(); - if (duration < 1){duration = 1;} - std::stringstream tagStream; - if (tags.size()){ -@@ -630,6 +640,9 @@ Controller::statSession::~statSession(){ - tagStream << "[" << *it << "]"; - } - } -+ const std::string& streamName = getStreamName(); -+ const std::string& curConnector = getConnectors(); -+ const std::string& host = getStrHost(); - Controller::logAccess(sessId, streamName, curConnector, host, duration, getUp(), - getDown(), tagStream.str()); - if (Controller::accesslog.size()){ -@@ -668,74 +681,99 @@ Controller::statSession::~statSession(){ - } - } - } -- tracked = false; -- firstActive = 0; -- firstSec = 0xFFFFFFFFFFFFFFFFull; -- lastSec = 0; -- sessionType = SESS_UNSET; -+ tags.clear(); -+ // Insert null datapoint -+ curData.log[curData.log.rbegin()->first + 1] = emptyLogEntry; - } - - /// Constructs an empty session - Controller::statSession::statSession(){ -- firstActive = 0; -- tracked = false; -- firstSec = 0xFFFFFFFFFFFFFFFFull; -- lastSec = 0; - sessionType = SESS_UNSET; - noBWCount = 0; -- streamName = ""; -- host = ""; -- curConnector = ""; - sessId = ""; - } - - /// Returns the first measured timestamp in this session. - uint64_t Controller::statSession::getStart(){ -- return firstSec; -+ if (!curData.log.size()){return 0;} -+ return curData.log.begin()->first; - } - - /// Returns the last measured timestamp in this session. - uint64_t Controller::statSession::getEnd(){ -- return lastSec; -+ if (!curData.log.size()){return 0;} -+ return curData.log.rbegin()->first; - } - - /// Returns true if there is data for this session at timestamp t. - bool Controller::statSession::hasDataFor(uint64_t t){ -- if (lastSec < t){return false;} -- if (firstSec > t){return false;} - if (curData.hasDataFor(t)){return true;} - return false; - } - --/// Returns true if this session should count as a viewer on the given timestamp. --bool Controller::statSession::isViewerOn(uint64_t t){ -- return getUp(t) + getDown(t); -+const std::string& Controller::statSession::getSessId(){ -+ return sessId; -+} -+ -+uint64_t Controller::statSession::getFirstActive(){ -+ if (curData.log.size()){ -+ return curData.log.rbegin()->second.firstActive; -+ } -+ return 0; -+} -+ -+const std::string& Controller::statSession::getStreamName(uint64_t t){ -+ if (curData.hasDataFor(t)){ -+ return curData.getDataFor(t).streamName; -+ } -+ return emptyLogEntry.streamName; - } - --std::string Controller::statSession::getStreamName(){ -- return streamName; -+const std::string& Controller::statSession::getStreamName(){ -+ if (curData.log.size()){ -+ return curData.log.rbegin()->second.streamName; -+ } -+ return emptyLogEntry.streamName; - } - --std::string Controller::statSession::getHost(){ -+std::string Controller::statSession::getStrHost(uint64_t t){ -+ std::string host; -+ Socket::hostBytesToStr(getHost(t).data(), 16, host); - return host; - } - --std::string Controller::statSession::getSessId(){ -- return sessId; -+std::string Controller::statSession::getStrHost(){ -+ std::string host; -+ Socket::hostBytesToStr(getHost().data(), 16, host); -+ return host; -+} -+ -+const std::string& Controller::statSession::getHost(uint64_t t){ -+ if (curData.hasDataFor(t)){ -+ return curData.getDataFor(t).host; -+ } -+ return emptyLogEntry.host; - } - --std::string Controller::statSession::getCurrentProtocols(){ -- return curConnector; -+const std::string& Controller::statSession::getHost(){ -+ if (curData.log.size()){ -+ return curData.log.rbegin()->second.host; -+ } -+ return emptyLogEntry.host; - } - --/// Returns true if this session should be considered connected --uint64_t Controller::statSession::newestDataPoint(){ -- return lastSec; -+const std::string& Controller::statSession::getConnectors(uint64_t t){ -+ if (curData.hasDataFor(t)){ -+ return curData.getDataFor(t).connectors; -+ } -+ return emptyLogEntry.connectors; - } - --/// Returns true if this session has started (tracked == true) but not yet ended (log entry written) --bool Controller::statSession::isTracked(){ -- return tracked; -+const std::string& Controller::statSession::getConnectors(){ -+ if (curData.log.size()){ -+ return curData.log.rbegin()->second.connectors; -+ } -+ return emptyLogEntry.connectors; - } - - /// Returns the cumulative connected time for this session at timestamp t. -@@ -842,7 +880,7 @@ uint64_t Controller::statSession::getPktRetransmit(){ - /// Returns the cumulative downloaded bytes per second for this session at timestamp t. - uint64_t Controller::statSession::getBpsDown(uint64_t t){ - uint64_t aTime = t - 5; -- if (aTime < firstSec){aTime = firstSec;} -+ if (aTime < curData.log.begin()->first){aTime = curData.log.begin()->first;} - if (t <= aTime){return 0;} - uint64_t valA = getDown(aTime); - uint64_t valB = getDown(t); -@@ -852,7 +890,7 @@ uint64_t Controller::statSession::getBpsDown(uint64_t t){ - /// Returns the cumulative uploaded bytes per second for this session at timestamp t. - uint64_t Controller::statSession::getBpsUp(uint64_t t){ - uint64_t aTime = t - 5; -- if (aTime < firstSec){aTime = firstSec;} -+ if (aTime < curData.log.begin()->first){aTime = curData.log.begin()->first;} - if (t <= aTime){return 0;} - uint64_t valA = getUp(aTime); - uint64_t valB = getUp(t); -@@ -867,17 +905,8 @@ bool Controller::statStorage::hasDataFor(unsigned long long t){ - - /// Returns a reference to the most current data available at timestamp t. - Controller::statLog &Controller::statStorage::getDataFor(unsigned long long t){ -- static statLog empty; - if (!log.size()){ -- empty.time = 0; -- empty.lastSecond = 0; -- empty.down = 0; -- empty.up = 0; -- empty.pktCount = 0; -- empty.pktLost = 0; -- empty.pktRetransmit = 0; -- empty.connectors = ""; -- return empty; -+ return emptyLogEntry; - } - std::map::iterator it = log.upper_bound(t); - if (it != log.begin()){it--;} -@@ -889,6 +918,11 @@ Controller::statLog &Controller::statStorage::getDataFor(unsigned long long t){ - void Controller::statStorage::update(Comms::Sessions &statComm, size_t index){ - statLog tmp; - tmp.time = statComm.getTime(index); -+ if (!log.size() || !log.rbegin()->second.firstActive){ -+ tmp.firstActive = statComm.getNow(index); -+ } else{ -+ tmp.firstActive = log.rbegin()->second.firstActive; -+ } - tmp.lastSecond = statComm.getLastSecond(index); - tmp.down = statComm.getDown(index); - tmp.up = statComm.getUp(index); -@@ -896,9 +930,19 @@ void Controller::statStorage::update(Comms::Sessions &statComm, size_t index){ - tmp.pktLost = statComm.getPacketLostCount(index); - tmp.pktRetransmit = statComm.getPacketRetransmitCount(index); - tmp.connectors = statComm.getConnector(index); -+ tmp.streamName = statComm.getStream(index); -+ tmp.host = statComm.getHost(index); - log[statComm.getNow(index)] = tmp; - // wipe data older than STAT_CUTOFF seconds -- while (log.size() && log.begin()->first < Util::bootSecs() - STAT_CUTOFF){log.erase(log.begin());} -+ // Ensure cutOffPoint is either time of boot or 10 minutes ago, whichever is closer. -+ // Prevents wrapping around to high values close to system boot time. -+ uint64_t cutOffPoint = Util::bootSecs(); -+ if (cutOffPoint > STAT_CUTOFF){ -+ cutOffPoint -= STAT_CUTOFF; -+ }else{ -+ cutOffPoint = 0; -+ } -+ while (log.size() && log.begin()->first < cutOffPoint){log.erase(log.begin());} - } - - void Controller::statLeadIn(){ -@@ -915,6 +959,7 @@ void Controller::statOnActive(size_t id){ - void Controller::statOnDisconnect(size_t id){ - // Check to see if cleanup is required (when a Session binary fails) - const std::string thisSessionId = statComm.getSessId(id); -+ sessions[thisSessionId].finish(); - // Try to lock to see if the session crashed during boot - IPC::semaphore sessionLock; - char semName[NAME_BUFFER_SIZE]; -@@ -932,7 +977,7 @@ void Controller::statOnDisconnect(size_t id){ - if(dataPage){ - // Session likely crashed while it was running - dataPage.init(userPageName, 1, true); -- FAIL_MSG("Session '%s' got canceled unexpectedly. Hoovering up the left overs...", thisSessionId.c_str()); -+ FAIL_MSG("Session '%s' got cancelled unexpectedly. Cleaning up the leftovers...", thisSessionId.c_str()); - } - // Finally remove the session lock which was created on bootup of the session - sessionLock.unlink(); -@@ -999,13 +1044,23 @@ void Controller::fillClients(JSON::Value &req, JSON::Value &rep){ - if (req.isMember("time")){reqTime = req["time"].asInt();} - // to make sure no nasty timing business takes place, we store the case "now" as a bool. - bool now = (reqTime == 0); -- //if greater than current bootsecs, assume unix time and subtract epoch from it -- if (reqTime > (int64_t)epoch - STAT_CUTOFF){reqTime -= (epoch-bSecs);} -+ //if in the last 600 seconds of unix time (or higher), assume unix time and subtract epoch from it -+ if (reqTime > (int64_t)epoch - STAT_CUTOFF){reqTime -= Controller::systemBoot/1000;} - // add the current time, if negative or zero. - if (reqTime < 0){reqTime += bSecs;} -- if (reqTime == 0){reqTime = bSecs - STAT_CUTOFF;} -+ if (reqTime == 0){ -+ // Ensure cutOffPoint is either time of boot or 10 minutes ago, whichever is closer. -+ // Prevents wrapping around to high values close to system boot time. -+ uint64_t cutOffPoint = bSecs; -+ if (cutOffPoint > STAT_CUTOFF){ -+ cutOffPoint -= STAT_CUTOFF; -+ }else{ -+ cutOffPoint = 0; -+ } -+ reqTime = cutOffPoint; -+ } - // at this point, we have the absolute timestamp in bootsecs. -- rep["time"] = reqTime + (epoch-bSecs); // fill the absolute timestamp -+ rep["time"] = reqTime + (Controller::systemBoot/1000); // fill the absolute timestamp - - unsigned int fields = 0; - // next, figure out the fields wanted -@@ -1062,13 +1117,14 @@ void Controller::fillClients(JSON::Value &req, JSON::Value &rep){ - if (now && reqTime - it->second.getEnd() < 5){time = it->second.getEnd();} - // data present and wanted? insert it! - if ((it->second.getEnd() >= time && it->second.getStart() <= time) && -- (!streams.size() || streams.count(it->second.getStreamName())) && -- (!protos.size() || protos.count(it->second.getCurrentProtocols()))){ -- if (it->second.hasDataFor(time)){ -+ (!streams.size() || streams.count(it->second.getStreamName(time))) && -+ (!protos.size() || protos.count(it->second.getConnectors(time)))){ -+ const statLog & dta = it->second.curData.getDataFor(time); -+ if (notEmpty(dta)){ - JSON::Value d; -- if (fields & STAT_CLI_HOST){d.append(it->second.getHost());} -- if (fields & STAT_CLI_STREAM){d.append(it->second.getStreamName());} -- if (fields & STAT_CLI_PROTO){d.append(it->second.getCurrentProtocols());} -+ if (fields & STAT_CLI_HOST){d.append(it->second.getStrHost(time));} -+ if (fields & STAT_CLI_STREAM){d.append(it->second.getStreamName(time));} -+ if (fields & STAT_CLI_PROTO){d.append(it->second.getConnectors(time));} - if (fields & STAT_CLI_CONNTIME){d.append(it->second.getConnTime(time));} - if (fields & STAT_CLI_POSITION){d.append(it->second.getLastSecond(time));} - if (fields & STAT_CLI_DOWN){d.append(it->second.getDown(time));} -@@ -1252,6 +1308,8 @@ void Controller::fillActive(JSON::Value &req, JSON::Value &rep){ - F = it->second.currIns; - }else if (j->asStringRef() == "outputs"){ - F = it->second.currOuts; -+ }else if (j->asStringRef() == "unspecified"){ -+ F = it->second.currUnspecified; - }else if (j->asStringRef() == "views"){ - F = it->second.viewers; - }else if (j->asStringRef() == "viewseconds"){ -@@ -1323,6 +1381,7 @@ public: - clients = 0; - inputs = 0; - outputs = 0; -+ unspecified = 0; - downbps = 0; - upbps = 0; - pktCount = 0; -@@ -1334,6 +1393,7 @@ public: - case Controller::SESS_VIEWER: clients++; break; - case Controller::SESS_INPUT: inputs++; break; - case Controller::SESS_OUTPUT: outputs++; break; -+ case Controller::SESS_UNSPECIFIED: unspecified++; break; - default: break; - } - downbps += down; -@@ -1345,6 +1405,7 @@ public: - uint64_t clients; - uint64_t inputs; - uint64_t outputs; -+ uint64_t unspecified; - uint64_t downbps; - uint64_t upbps; - uint64_t pktCount; -@@ -1363,11 +1424,21 @@ void Controller::fillTotals(JSON::Value &req, JSON::Value &rep){ - if (req.isMember("start")){reqStart = req["start"].asInt();} - if (req.isMember("end")){reqEnd = req["end"].asInt();} - //if the reqStart or reqEnd is greater than current bootsecs, assume unix time and subtract epoch from it -- if (reqStart > (int64_t)epoch - STAT_CUTOFF){reqStart -= (epoch-bSecs);} -- if (reqEnd > (int64_t)epoch - STAT_CUTOFF){reqEnd -= (epoch-bSecs);} -+ if (reqStart > (int64_t)epoch - STAT_CUTOFF){reqStart -= Controller::systemBoot/1000;} -+ if (reqEnd > (int64_t)epoch - STAT_CUTOFF){reqEnd -= Controller::systemBoot/1000;} - // add the current time, if negative or zero. - if (reqStart < 0){reqStart += bSecs;} -- if (reqStart == 0){reqStart = bSecs - STAT_CUTOFF;} -+ if (reqStart == 0){ -+ // Ensure cutOffPoint is either time of boot or 10 minutes ago, whichever is closer. -+ // Prevents wrapping around to high values close to system boot time. -+ uint64_t cutOffPoint = bSecs; -+ if (cutOffPoint > STAT_CUTOFF){ -+ cutOffPoint -= STAT_CUTOFF; -+ }else{ -+ cutOffPoint = 0; -+ } -+ reqStart = cutOffPoint; -+ } - if (reqEnd <= 0){reqEnd += bSecs;} - // at this point, reqStart and reqEnd are the absolute timestamp in bootsecs. - if (reqEnd < reqStart){reqEnd = reqStart;} -@@ -1417,7 +1488,7 @@ void Controller::fillTotals(JSON::Value &req, JSON::Value &rep){ - if ((it->second.getEnd() >= (unsigned long long)reqStart || - it->second.getStart() <= (unsigned long long)reqEnd) && - (!streams.size() || streams.count(it->second.getStreamName())) && -- (!protos.size() || protos.count(it->second.getCurrentProtocols()))){ -+ (!protos.size() || protos.count(it->second.getConnectors()))){ - for (unsigned long long i = reqStart; i <= reqEnd; ++i){ - if (it->second.hasDataFor(i)){ - totalsCount[i].add(it->second.getBpsDown(i), it->second.getBpsUp(i), it->second.getSessType(), it->second.getPktCount(), it->second.getPktLost(), it->second.getPktRetransmit()); -@@ -1436,8 +1507,8 @@ void Controller::fillTotals(JSON::Value &req, JSON::Value &rep){ - return; - } - // yay! We have data! -- rep["start"] = totalsCount.begin()->first + (epoch-bSecs); -- rep["end"] = totalsCount.rbegin()->first + (epoch-bSecs); -+ rep["start"] = totalsCount.begin()->first + (Controller::systemBoot/1000); -+ rep["end"] = totalsCount.rbegin()->first + (Controller::systemBoot/1000); - rep["data"].null(); - rep["interval"].null(); - uint64_t prevT = 0; -@@ -1506,12 +1577,15 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int - uint32_t totViewers = 0; - uint32_t totInputs = 0; - uint32_t totOutputs = 0; -+ uint32_t totUnspecified = 0; - for (uint64_t idx = 0; idx < statComm.recordCount(); idx++){ - if (statComm.getStatus(idx) == COMM_STATUS_INVALID || statComm.getStatus(idx) & COMM_STATUS_DISCONNECT){continue;} - const std::string thisSessId = statComm.getSessId(idx); - // Count active viewers, inputs, outputs and protocols - if (thisSessId[0] == 'I'){ - totInputs++; -+ }else if (thisSessId[0] == 'U'){ -+ totUnspecified++; - }else if (thisSessId[0] == 'O'){ - totOutputs++; - outputs[statComm.getConnector(idx)]++; -@@ -1602,6 +1676,7 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int - response << "# TYPE mist_sessions_count counter\n"; - response << "mist_sessions_count{sessType=\"viewers\"}" << servViewers << "\n"; - response << "mist_sessions_count{sessType=\"incoming\"}" << servInputs << "\n"; -+ response << "mist_sessions_count{sessType=\"unspecified\"}" << servUnspecified << "\n"; - response << "mist_sessions_count{sessType=\"outgoing\"}" << servOutputs << "\n\n"; - - response << "# HELP mist_bw_total Count of bytes handled since server start, by direction.\n"; -@@ -1637,6 +1712,7 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int - response << "mist_sessions_total{sessType=\"viewers\"}" << totViewers << "\n"; - response << "mist_sessions_total{sessType=\"incoming\"}" << totInputs << "\n"; - response << "mist_sessions_total{sessType=\"outgoing\"}" << totOutputs << "\n"; -+ response << "mist_sessions_total{sessType=\"unspecified\"}" << totUnspecified << "\n"; - response << "mist_sessions_total{sessType=\"cached\"}" << sessions.size() << "\n"; - - response << "\n# HELP mist_viewcount Count of unique viewer sessions since stream start, per " -@@ -1656,6 +1732,8 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int - << it->second.currIns << "\n"; - response << "mist_sessions{stream=\"" << it->first << "\",sessType=\"outgoing\"}" - << it->second.currOuts << "\n"; -+ response << "mist_sessions{stream=\"" << it->first << "\",sessType=\"unspecified\"}" -+ << it->second.currUnspecified << "\n"; - response << "mist_viewcount{stream=\"" << it->first << "\"}" << it->second.viewers << "\n"; - response << "mist_viewseconds{stream=\"" << it->first << "\"} " << it->second.viewSeconds << "\n"; - response << "mist_bw{stream=\"" << it->first << "\",direction=\"up\"}" << it->second.upBytes << "\n"; -@@ -1691,9 +1769,11 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int - resp["curr"].append(totViewers); - resp["curr"].append(totInputs); - resp["curr"].append(totOutputs); -+ resp["curr"].append(totUnspecified); - resp["tot"].append(servViewers); - resp["tot"].append(servInputs); - resp["tot"].append(servOutputs); -+ resp["tot"].append(servUnspecified); - resp["st"].append(bw_up_total); - resp["st"].append(bw_down_total); - resp["bw"].append(servUpBytes); -@@ -1735,6 +1815,7 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int - resp["streams"][it->first]["curr"].append(it->second.currViews); - resp["streams"][it->first]["curr"].append(it->second.currIns); - resp["streams"][it->first]["curr"].append(it->second.currOuts); -+ resp["streams"][it->first]["curr"].append(it->second.currUnspecified); - resp["streams"][it->first]["pkts"].append(it->second.packSent); - resp["streams"][it->first]["pkts"].append(it->second.packLoss); - resp["streams"][it->first]["pkts"].append(it->second.packRetrans); -diff --git a/src/controller/controller_statistics.h b/src/controller/controller_statistics.h -index f798f811..69605942 100644 ---- a/src/controller/controller_statistics.h -+++ b/src/controller/controller_statistics.h -@@ -28,16 +28,19 @@ namespace Controller{ - - struct statLog{ - uint64_t time; -+ uint64_t firstActive; - uint64_t lastSecond; - uint64_t down; - uint64_t up; - uint64_t pktCount; - uint64_t pktLost; - uint64_t pktRetransmit; -+ std::string streamName; -+ std::string host; - std::string connectors; - }; - -- enum sessType{SESS_UNSET = 0, SESS_INPUT, SESS_OUTPUT, SESS_VIEWER}; -+ enum sessType{SESS_UNSET = 0, SESS_INPUT, SESS_OUTPUT, SESS_VIEWER, SESS_UNSPECIFIED}; - - class statStorage{ - public: -@@ -51,34 +54,30 @@ namespace Controller{ - /// Allows for moving of connections to another session. - class statSession{ - private: -- uint64_t firstActive; -- uint64_t firstSec; -- uint64_t lastSec; - sessType sessionType; -- bool tracked; - uint8_t noBWCount; ///< Set to 2 when not to count for external bandwidth -- std::string streamName; -- std::string host; -- std::string curConnector; - std::string sessId; - - public: - statSession(); -- ~statSession(); -+ void finish(); - statStorage curData; - std::set tags; - sessType getSessType(); - void update(uint64_t index, Comms::Sessions &data); - uint64_t getStart(); - uint64_t getEnd(); -- bool isViewerOn(uint64_t time); -- bool isTracked(); - bool hasDataFor(uint64_t time); -- std::string getStreamName(); -- std::string getHost(); -- std::string getSessId(); -- std::string getCurrentProtocols(); -- uint64_t newestDataPoint(); -+ const std::string& getSessId(); -+ const std::string& getStreamName(uint64_t t); -+ const std::string& getStreamName(); -+ std::string getStrHost(uint64_t t); -+ std::string getStrHost(); -+ const std::string& getHost(uint64_t t); -+ const std::string& getHost(); -+ const std::string& getConnectors(uint64_t t); -+ const std::string& getConnectors(); -+ uint64_t getFirstActive(); - uint64_t getConnTime(uint64_t time); - uint64_t getConnTime(); - uint64_t getLastSecond(uint64_t time); -diff --git a/src/controller/controller_storage.cpp b/src/controller/controller_storage.cpp -index bdd52893..532ee4fe 100644 ---- a/src/controller/controller_storage.cpp -+++ b/src/controller/controller_storage.cpp -@@ -196,6 +196,7 @@ namespace Controller{ - rlxStrm->addField("viewers", RAX_64UINT); - rlxStrm->addField("inputs", RAX_64UINT); - rlxStrm->addField("outputs", RAX_64UINT); -+ rlxStrm->addField("unspecified", RAX_64UINT); - rlxStrm->setReady(); - } - rlxStrm->setRCount((1024 * 1024 - rlxStrm->getOffset()) / rlxStrm->getRSize()); -@@ -433,12 +434,17 @@ namespace Controller{ - - // if fields missing, recreate the page - if (globAccX.isReady()){ -- if(globAccX.getFieldAccX("systemBoot")){ -+ if(globAccX.getFieldAccX("systemBoot") && globAccX.getInt("systemBoot")){ - systemBoot = globAccX.getInt("systemBoot"); - } - if(!globAccX.getFieldAccX("defaultStream") - || !globAccX.getFieldAccX("systemBoot") -- || !globAccX.getFieldAccX("sessionMode")){ -+ || !globAccX.getFieldAccX("sessionViewerMode") -+ || !globAccX.getFieldAccX("sessionInputMode") -+ || !globAccX.getFieldAccX("sessionOutputMode") -+ || !globAccX.getFieldAccX("sessionUnspecifiedMode") -+ || !globAccX.getFieldAccX("sessionStreamInfoMode") -+ || !globAccX.getFieldAccX("tknMode")){ - globAccX.setReload(); - globCfg.master = true; - globCfg.close(); -@@ -449,16 +455,24 @@ namespace Controller{ - if (!globAccX.isReady()){ - globAccX.addField("defaultStream", RAX_128STRING); - globAccX.addField("systemBoot", RAX_64UINT); -- globAccX.addField("sessionMode", RAX_64UINT); -- if (!Storage["config"]["sessionMode"]){ -- Storage["config"]["sessionMode"] = SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID; -- } -+ globAccX.addField("sessionViewerMode", RAX_64UINT); -+ globAccX.addField("sessionInputMode", RAX_64UINT); -+ globAccX.addField("sessionOutputMode", RAX_64UINT); -+ globAccX.addField("sessionUnspecifiedMode", RAX_64UINT); -+ globAccX.addField("sessionStreamInfoMode", RAX_64UINT); -+ globAccX.addField("tknMode", RAX_64UINT); - globAccX.setRCount(1); - globAccX.setEndPos(1); - globAccX.setReady(); - } - globAccX.setString("defaultStream", Storage["config"]["defaultStream"].asStringRef()); -- globAccX.setInt("sessionMode", Storage["config"]["sessionMode"].asInt()); -+ globAccX.setInt("sessionViewerMode", Storage["config"]["sessionViewerMode"].asInt()); -+ globAccX.setInt("sessionInputMode", Storage["config"]["sessionInputMode"].asInt()); -+ globAccX.setInt("sessionOutputMode", Storage["config"]["sessionOutputMode"].asInt()); -+ globAccX.setInt("sessionUnspecifiedMode", Storage["config"]["sessionUnspecifiedMode"].asInt()); -+ globAccX.setInt("sessionStreamInfoMode", Storage["config"]["sessionStreamInfoMode"].asInt()); -+ globAccX.setInt("tknMode", Storage["config"]["tknMode"].asInt()); -+ globAccX.setInt("systemBoot", systemBoot); - globCfg.master = false; // leave the page after closing - } - } -diff --git a/src/controller/controller_storage.h b/src/controller/controller_storage.h -index 6339cfd0..974d989b 100644 ---- a/src/controller/controller_storage.h -+++ b/src/controller/controller_storage.h -@@ -16,6 +16,7 @@ namespace Controller{ - extern bool isTerminal; ///< True if connected to a terminal and not a log file. - extern bool isColorized; ///< True if we colorize the output - extern uint64_t logCounter; ///< Count of logged messages since boot -+ extern uint64_t systemBoot; ///< Unix time in milliseconds of system boot - - Util::RelAccX *logAccessor(); - Util::RelAccX *accesslogAccessor(); -diff --git a/src/input/input.cpp b/src/input/input.cpp -index 0524f55b..782dc4ef 100644 ---- a/src/input/input.cpp -+++ b/src/input/input.cpp -@@ -502,6 +502,7 @@ namespace Mist{ - } - - int Input::run(){ -+ Comms::sessionConfigCache(); - if (streamStatus){streamStatus.mapped[0] = STRMSTAT_BOOT;} - checkHeaderTimes(config->getString("input")); - if (needHeader()){ -@@ -623,6 +624,8 @@ namespace Mist{ - /// ~~~~~~~~~~~~~~~ - void Input::serve(){ - users.reload(streamName, true); -+ Comms::Connections statComm; -+ uint64_t startTime = Util::bootSecs(); - - if (!M){ - // Initialize meta page -@@ -636,6 +639,7 @@ namespace Mist{ - meta.setSource(config->getString("input")); - - bool internalOnly = (config->getString("input").find("INTERNAL_ONLY") != std::string::npos); -+ bool isBuffer = (capa["name"].asStringRef() == "Buffer"); - - /*LTS-START*/ - if (Triggers::shouldTrigger("STREAM_READY", config->getString("streamname"))){ -@@ -666,6 +670,18 @@ namespace Mist{ - }else{ - if (connectedUsers && M.getValidTracks().size()){activityCounter = Util::bootSecs();} - } -+ // Connect to stats for INPUT detection -+ if (!internalOnly && !isBuffer){ -+ if (!statComm){statComm.reload(streamName, getConnectedBinHost(), JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "");} -+ if (statComm){ -+ uint64_t now = Util::bootSecs(); -+ statComm.setNow(now); -+ statComm.setStream(streamName); -+ statComm.setTime(now - startTime); -+ statComm.setLastSecond(0); -+ connStats(statComm); -+ } -+ } - // if not shutting down, wait 1 second before looping - if (config->is_active){Util::wait(INPUT_USER_INTERVAL);} - } -@@ -820,7 +836,7 @@ namespace Mist{ - - if (Util::bootSecs() - statTimer > 1){ - // Connect to stats for INPUT detection -- if (!statComm){statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID);} -+ if (!statComm){statComm.reload(streamName, getConnectedBinHost(), JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "");} - if (statComm){ - if (!statComm){ - config->is_active = false; -@@ -830,7 +846,6 @@ namespace Mist{ - uint64_t now = Util::bootSecs(); - statComm.setNow(now); - statComm.setStream(streamName); -- statComm.setConnector("INPUT:" + capa["name"].asStringRef()); - statComm.setTime(now - startTime); - statComm.setLastSecond(0); - connStats(statComm); -@@ -984,7 +999,7 @@ namespace Mist{ - - if (Util::bootSecs() - statTimer > 1){ - // Connect to stats for INPUT detection -- if (!statComm){statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID);} -+ if (!statComm){statComm.reload(streamName, getConnectedBinHost(), JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "");} - if (statComm){ - if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ - config->is_active = false; -diff --git a/src/input/input_rtsp.cpp b/src/input/input_rtsp.cpp -index ebb812e1..25be6e23 100644 ---- a/src/input/input_rtsp.cpp -+++ b/src/input/input_rtsp.cpp -@@ -210,7 +210,7 @@ namespace Mist{ - if (lastSecs != currSecs){ - lastSecs = currSecs; - // Connect to stats for INPUT detection -- statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID); -+ statComm.reload(streamName, getConnectedBinHost(), JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), ""); - if (statComm){ - if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ - config->is_active = false; -@@ -225,7 +225,6 @@ namespace Mist{ - statComm.setDown(tcpCon.dataDown()); - statComm.setTime(now - startTime); - statComm.setLastSecond(0); -- statComm.setHost(getConnectedBinHost()); - } - } - } -diff --git a/src/input/input_sdp.cpp b/src/input/input_sdp.cpp -index 3169a836..0b8ddeb5 100644 ---- a/src/input/input_sdp.cpp -+++ b/src/input/input_sdp.cpp -@@ -202,7 +202,7 @@ namespace Mist{ - if (lastSecs != currSecs){ - lastSecs = currSecs; - // Connect to stats for INPUT detection -- statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID); -+ statComm.reload(streamName, getConnectedBinHost(), JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), ""); - if (statComm){ - if (statComm.getStatus() == COMM_STATUS_REQDISCONNECT){ - config->is_active = false; -@@ -217,7 +217,6 @@ namespace Mist{ - statComm.setUp(bytesUp); - statComm.setTime(now - startTime); - statComm.setLastSecond(0); -- statComm.setHost(getConnectedBinHost()); - } - } - // If the error flag is raised or we are lacking data, try to recover -diff --git a/src/input/input_ts.cpp b/src/input/input_ts.cpp -index 23311dc0..e405e3b6 100644 ---- a/src/input/input_ts.cpp -+++ b/src/input/input_ts.cpp -@@ -621,7 +621,7 @@ namespace Mist{ - // Check for and spawn threads here. - if (Util::bootSecs() - threadCheckTimer > 1){ - // Connect to stats for INPUT detection -- statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID); -+ statComm.reload(streamName, getConnectedBinHost(), JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), ""); - if (statComm){ - if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ - config->is_active = false; -@@ -636,7 +636,6 @@ namespace Mist{ - statComm.setDown(downCounter + tcpCon.dataDown()); - statComm.setTime(now - startTime); - statComm.setLastSecond(0); -- statComm.setHost(getConnectedBinHost()); - } - - std::set activeTracks = liveStream.getActiveTracks(); -diff --git a/src/input/input_tsrist.cpp b/src/input/input_tsrist.cpp -index 5b95aa9f..df658da1 100644 ---- a/src/input/input_tsrist.cpp -+++ b/src/input/input_tsrist.cpp -@@ -284,7 +284,7 @@ namespace Mist{ - } - - -- void inputTSRIST::connStats(Comms::Statistics &statComm){ -+ void inputTSRIST::connStats(Comms::Connections &statComm){ - statComm.setUp(0); - statComm.setDown(downBytes); - statComm.setHost(getConnectedBinHost()); -diff --git a/src/input/input_tsrist.h b/src/input/input_tsrist.h -index 731f9b04..a75f6601 100644 ---- a/src/input/input_tsrist.h -+++ b/src/input/input_tsrist.h -@@ -30,7 +30,7 @@ namespace Mist{ - int64_t timeStampOffset; - uint64_t lastTimeStamp; - -- virtual void connStats(Comms::Statistics &statComm); -+ virtual void connStats(Comms::Connections &statComm); - - struct rist_ctx *receiver_ctx; - -diff --git a/src/output/output.cpp b/src/output/output.cpp -index 50d67549..d525b8ea 100644 ---- a/src/output/output.cpp -+++ b/src/output/output.cpp -@@ -92,7 +92,7 @@ namespace Mist{ - firstTime = 0; - firstPacketTime = 0xFFFFFFFFFFFFFFFFull; - lastPacketTime = 0; -- sid = ""; -+ tkn = ""; - parseData = false; - wantRequest = true; - sought = false; -@@ -111,7 +111,6 @@ namespace Mist{ - lastPushUpdate = 0; - previousFile = ""; - currentFile = ""; -- sessionMode = 0xFFFFFFFFFFFFFFFFull; - - lastRecv = Util::bootSecs(); - if (myConn){ -@@ -230,7 +229,7 @@ namespace Mist{ - bool Output::isReadyForPlay(){ - // If a protocol does not support any codecs, we assume you know what you're doing - if (!capa.isMember("codecs")){return true;} -- if (!isInitialized){initialize();} -+ if (!isInitialized){return false;} - meta.reloadReplacedPagesIfNeeded(); - if (getSupportedTracks().size()){ - size_t minTracks = 2; -@@ -277,6 +276,7 @@ namespace Mist{ - /// Assumes streamName class member has been set already. - /// Will start input if not currently active, calls onFail() if this does not succeed. - void Output::reconnect(){ -+ Comms::sessionConfigCache(); - thisPacket.null(); - if (config->hasOption("noinput") && config->getBool("noinput")){ - Util::sanitizeName(streamName); -@@ -347,11 +347,10 @@ namespace Mist{ - isInitialized = true; - - //Connect to stats reporting, if not connected already -- if (!statComm){ -- statComm.reload(streamName, getConnectedHost(), sid, capa["name"].asStringRef(), reqUrl, sessionMode); -- stats(true); -- } -- -+ stats(true); -+ //Abort if the stats code shut us down just now -+ if (!isInitialized){return;} -+ - //push inputs do not need to wait for stream to be ready for playback - if (isPushing()){return;} - -@@ -1216,7 +1215,7 @@ namespace Mist{ - /// request URL (if any) - /// ~~~~~~~~~~~~~~~ - int Output::run(){ -- sessionMode = Util::getGlobalConfig("sessionMode").asInt(); -+ Comms::sessionConfigCache(); - /*LTS-START*/ - // Connect to file target, if needed - if (isFileTarget()){ -@@ -1257,6 +1256,7 @@ namespace Mist{ - /*LTS-END*/ - DONTEVEN_MSG("MistOut client handler started"); - while (keepGoing() && (wantRequest || parseData)){ -+ Comms::sessionConfigCache(); - if (wantRequest){requestHandler();} - if (parseData){ - if (!isInitialized){ -@@ -1779,27 +1779,35 @@ namespace Mist{ - } - } - -- if (!statComm){statComm.reload(streamName, getConnectedHost(), sid, capa["name"].asStringRef(), reqUrl, sessionMode);} -- if (!statComm){return;} -- if (statComm.getExit()){ -+ // Disable stats for HTTP internal output -+ if (Comms::sessionStreamInfoMode == SESS_HTTP_DISABLED && capa["name"].asStringRef() == "HTTP"){return;} -+ -+ // Set the token to the pid for outputs which do not generate it in the requestHandler -+ if (!tkn.size()){ tkn = JSON::Value(getpid()).asString(); } -+ -+ if (!statComm){ -+ statComm.reload(streamName, getConnectedBinHost(), tkn, getStatsName(), reqUrl); -+ } -+ if (!statComm || statComm.getExit()){ - onFail("Shutting down since this session is not allowed to view this stream"); -+ statComm.unload(); - return; -- } -+ } - - lastStats = now; - - VERYHIGH_MSG("Writing stats: %s, %s, %s, %" PRIu64 ", %" PRIu64, getConnectedHost().c_str(), streamName.c_str(), -- sid.c_str(), myConn.dataUp(), myConn.dataDown()); -+ tkn.c_str(), myConn.dataUp(), myConn.dataDown()); - /*LTS-START*/ - if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ - onFail("Shutting down on controller request"); -+ statComm.unload(); - return; - } - /*LTS-END*/ - statComm.setNow(now); -- statComm.setConnector(getStatsName()); - connStats(now, statComm); -- statComm.setLastSecond(thisPacket ? thisPacket.getTime() : 0); -+ statComm.setLastSecond(thisPacket ? thisPacket.getTime()/1000 : 0); - statComm.setPid(getpid()); - - /*LTS-START*/ -diff --git a/src/output/output.h b/src/output/output.h -index 173b3840..441c06ab 100644 ---- a/src/output/output.h -+++ b/src/output/output.h -@@ -130,8 +130,7 @@ namespace Mist{ - - Comms::Connections statComm; - bool isBlocking; ///< If true, indicates that myConn is blocking. -- std::string sid; ///< Random identifier used to split connections into sessions -- uint64_t sessionMode; -+ std::string tkn; ///< Random identifier used to split connections into sessions - uint64_t nextKeyTime(); - - // stream delaying variables -diff --git a/src/output/output_cmaf.cpp b/src/output/output_cmaf.cpp -index c2fbc5bc..eb7c87da 100644 ---- a/src/output/output_cmaf.cpp -+++ b/src/output/output_cmaf.cpp -@@ -222,24 +222,17 @@ namespace Mist{ - void OutCMAF::sendHlsMasterManifest(){ - selectDefaultTracks(); - -- std::string sessId = ""; -- if (hasSessionIDs()){ -- std::string ua = UA + JSON::Value(getpid()).asString(); -- crc = checksum::crc32(0, ua.data(), ua.size()); -- sessId = JSON::Value(crc).asString(); -- } -- - // check for forced "no low latency" parameter - bool noLLHLS = H.GetVar("llhls").size() ? H.GetVar("llhls") == "0" : false; - - // Populate the struct that will help generate the master playlist - const HLS::MasterData masterData ={ -- hasSessionIDs(), -+ false,//hasSessionIDs, unused - noLLHLS, - hlsMediaFormat == ".ts", - getMainSelectedTrack(), - H.GetHeader("User-Agent"), -- sessId, -+ (Comms::tknMode & 0x04)?tkn:"", - systemBoot, - bootMsOffset, - }; -@@ -261,11 +254,8 @@ namespace Mist{ - - // Chunkpath & Session ID logic - std::string urlPrefix = ""; -- std::string sessId = ""; - if (config->getString("chunkpath").size()){ - urlPrefix = HTTP::URL(config->getString("chunkpath")).link("./" + H.url).link("./").getUrl(); -- }else{ -- sessId = H.GetVar("sessId"); - } - - // check for forced "no low latency" parameter -@@ -279,7 +269,7 @@ namespace Mist{ - noLLHLS, - hlsMediaFormat, - M.getEncryption(requestTid), -- sessId, -+ (Comms::tknMode & 0x04)?tkn:"", - timingTid, - requestTid, - M.biggestFragment(timingTid) / 1000, -@@ -346,6 +336,16 @@ namespace Mist{ - std::string url = H.url.substr(H.url.find('/', 6) + 1); - HTTP::URL req(reqUrl); - -+ -+ if (tkn.size()){ -+ if (Comms::tknMode & 0x08){ -+ const std::string koekjes = H.GetHeader("Cookie"); -+ std::stringstream cookieHeader; -+ cookieHeader << "tkn=" << tkn << "; Max-Age=" << SESS_TIMEOUT; -+ H.SetHeader("Set-Cookie", cookieHeader.str()); -+ } -+ } -+ - // Send a dash manifest for any URL with .mpd in the path - if (req.getExt() == "mpd"){ - sendDashManifest(); -@@ -438,6 +438,7 @@ namespace Mist{ - H.SendResponse("400", "Bad Request: Could not parse the url", myConn); - return; - } -+ - std::string headerData = - CMAF::keyHeader(M, idx, startTime, targetTime, fragmentIndex, false, false); - -diff --git a/src/output/output_hls.cpp b/src/output/output_hls.cpp -index ad810aa7..112e7ee2 100644 ---- a/src/output/output_hls.cpp -+++ b/src/output/output_hls.cpp -@@ -11,7 +11,7 @@ const std::string hlsMediaFormat = ".ts"; - - namespace Mist{ - bool OutHLS::isReadyForPlay(){ -- if (!isInitialized){initialize();} -+ if (!isInitialized){return false;} - meta.reloadReplacedPagesIfNeeded(); - if (!M.getValidTracks().size()){return false;} - uint32_t mainTrack = M.mainTrack(); -@@ -110,25 +110,17 @@ namespace Mist{ - ///\return The master playlist file for (LL)HLS. - void OutHLS::sendHlsMasterManifest(){ - selectDefaultTracks(); -- -- std::string sessId = ""; -- if (hasSessionIDs()){ -- std::string ua = UA + JSON::Value(getpid()).asString(); -- crc = checksum::crc32(0, ua.data(), ua.size()); -- sessId = JSON::Value(crc).asString(); -- } -- - // check for forced "no low latency" parameter - bool noLLHLS = H.GetVar("llhls").size() ? H.GetVar("llhls") == "0" : false; - - // Populate the struct that will help generate the master playlist - const HLS::MasterData masterData ={ -- hasSessionIDs(), -+ false,//hasSessionIDs, unused - noLLHLS, - hlsMediaFormat == ".ts", - getMainSelectedTrack(), - H.GetHeader("User-Agent"), -- sessId, -+ (Comms::tknMode & 0x04)?tkn:"", - systemBoot, - bootMsOffset, - }; -@@ -150,11 +142,8 @@ namespace Mist{ - - // Chunkpath & Session ID logic - std::string urlPrefix = ""; -- std::string sessId = ""; - if (config->getString("chunkpath").size()){ - urlPrefix = HTTP::URL(config->getString("chunkpath")).link("./" + H.url).link("./").getUrl(); -- }else{ -- sessId = H.GetVar("sessId"); - } - - // check for forced "no low latency" parameter -@@ -168,7 +157,7 @@ namespace Mist{ - noLLHLS, - hlsMediaFormat, - M.getEncryption(requestTid), -- sessId, -+ (Comms::tknMode & 0x04)?tkn:"", - timingTid, - requestTid, - M.biggestFragment(timingTid) / 1000, -@@ -226,6 +215,15 @@ namespace Mist{ - bootMsOffset = 0; - if (M.getLive()){bootMsOffset = M.getBootMsOffset();} - -+ if (tkn.size()){ -+ if (Comms::tknMode & 0x08){ -+ const std::string koekjes = H.GetHeader("Cookie"); -+ std::stringstream cookieHeader; -+ cookieHeader << "tkn=" << tkn << "; Max-Age=" << SESS_TIMEOUT; -+ H.SetHeader("Set-Cookie", cookieHeader.str()); -+ } -+ } -+ - if (H.url == "/crossdomain.xml"){ - H.SetHeader("Content-Type", "text/xml"); - H.SetHeader("Server", APPIDENT); -diff --git a/src/output/output_http.cpp b/src/output/output_http.cpp -index f55b974d..ed8fa33b 100644 ---- a/src/output/output_http.cpp -+++ b/src/output/output_http.cpp -@@ -217,7 +217,9 @@ namespace Mist{ - myConn.close(); - return; - } -- if (handler != capa["name"].asStringRef() || H.GetVar("stream") != streamName){ -+ -+ //Check if we need to change binary and/or reconnect -+ if (handler != capa["name"].asStringRef() || H.GetVar("stream") != streamName || (statComm && (statComm.getHost() != getConnectedBinHost() || statComm.getTkn() != tkn))){ - MEDIUM_MSG("Switching from %s (%s) to %s (%s)", capa["name"].asStringRef().c_str(), - streamName.c_str(), handler.c_str(), H.GetVar("stream").c_str()); - streamName = H.GetVar("stream"); -@@ -268,21 +270,32 @@ namespace Mist{ - realTime = 0; - } - } -- // Get session ID cookie or generate a random one if it wasn't set -- if (!sid.size()){ -+ // Read the session token -+ if (Comms::tknMode & 0x01){ -+ // Get session token from the request url -+ if (H.GetVar("tkn") != ""){ -+ tkn = H.GetVar("tkn"); -+ } else if (H.GetVar("sid") != ""){ -+ tkn = H.GetVar("sid"); -+ } else if (H.GetVar("sessId") != ""){ -+ tkn = H.GetVar("sessId"); -+ } -+ } -+ if ((Comms::tknMode & 0x02) && !tkn.size()){ -+ // Get session token from the request cookie - std::map storage; - const std::string koekjes = H.GetHeader("Cookie"); -- HTTP::parseVars(koekjes, storage); -- if (storage.count("sid")){ -- // Get sid cookie, which is used to divide connections into sessions -- sid = storage.at("sid"); -- }else{ -- // Else generate one -- const std::string newSid = UA + JSON::Value(getpid()).asString(); -- sid = JSON::Value(checksum::crc32(0, newSid.data(), newSid.size())).asString(); -- H.SetHeader("sid", sid.c_str()); -+ HTTP::parseVars(koekjes, storage, "; "); -+ if (storage.count("tkn")){ -+ tkn = storage.at("tkn"); - } - } -+ // Generate a session token if it is being sent as a cookie or url parameter and we couldn't read one -+ if (!tkn.size() && Comms::tknMode > 3){ -+ const std::string newTkn = UA + JSON::Value(getpid()).asString(); -+ tkn = JSON::Value(checksum::crc32(0, newTkn.data(), newTkn.size())).asString(); -+ HIGH_MSG("Generated tkn '%s'", tkn.c_str()); -+ } - // Handle upgrade to websocket if the output supports it - std::string upgradeHeader = H.GetHeader("Upgrade"); - Util::stringToLower(upgradeHeader); -@@ -290,7 +303,9 @@ namespace Mist{ - INFO_MSG("Switching to Websocket mode"); - setBlocking(false); - preWebsocketConnect(); -- webSock = new HTTP::Websocket(myConn, H); -+ HTTP::Parser req = H; -+ H.Clean(); -+ webSock = new HTTP::Websocket(myConn, req, H); - if (!(*webSock)){ - delete webSock; - webSock = 0; -@@ -333,6 +348,14 @@ namespace Mist{ - void HTTPOutput::respondHTTP(const HTTP::Parser & req, bool headersOnly){ - //We generally want the CORS headers to be set for all responses - H.setCORSHeaders(); -+ H.SetHeader("Server", APPIDENT); -+ if (tkn.size()){ -+ if (Comms::tknMode & 0x08){ -+ std::stringstream cookieHeader; -+ cookieHeader << "tkn=" << tkn << "; Max-Age=" << SESS_TIMEOUT; -+ H.SetHeader("Set-Cookie", cookieHeader.str()); -+ } -+ } - //Set attachment header to force download, if applicable - if (req.GetVar("dl").size()){ - //If we want to download, and the string contains a dot, use as-is. -@@ -395,6 +418,8 @@ namespace Mist{ - ///\brief Handles requests by starting a corresponding output process. - ///\param connector The type of connector to be invoked. - void HTTPOutput::reConnector(std::string &connector){ -+ // Clear tkn in order to deal with reverse proxies -+ tkn = ""; - // taken from CheckProtocols (controller_connectors.cpp) - char *argarr[32]; - for (int i = 0; i < 32; i++){argarr[i] = 0;} -diff --git a/src/output/output_http_internal.cpp b/src/output/output_http_internal.cpp -index 97ca455f..be955196 100644 ---- a/src/output/output_http_internal.cpp -+++ b/src/output/output_http_internal.cpp -@@ -76,11 +76,11 @@ namespace Mist{ - std::string method = H.method; - // send logo icon - if (H.url.length() > 4 && H.url.substr(H.url.length() - 4, 4) == ".ico"){ -- sendIcon(); -+ sendIcon(false); - return; - } - if (H.url.length() > 6 && H.url.substr(H.url.length() - 5, 5) == ".html"){ -- HTMLResponse(); -+ HTMLResponse(H, false); - return; - } - if (H.url.size() >= 3 && H.url.substr(H.url.size() - 3) == ".js"){ -@@ -337,9 +337,9 @@ namespace Mist{ - } - } - -- void OutHTTP::HTMLResponse(){ -- std::string method = H.method; -- HTTP::URL fullURL(H.GetHeader("Host")); -+ void OutHTTP::HTMLResponse(const HTTP::Parser & req, bool headersOnly){ -+ HTTPOutput::respondHTTP(req, headersOnly); -+ HTTP::URL fullURL(req.GetHeader("Host")); - if (!fullURL.protocol.size()){fullURL.protocol = getProtocolForPort(fullURL.getPort());} - if (config->getString("pubaddr") != ""){ - HTTP::URL altURL(config->getString("pubaddr")); -@@ -349,24 +349,22 @@ namespace Mist{ - fullURL.path = altURL.path; - } - if (mistPath.size()){fullURL = mistPath;} -- std::string uAgent = H.GetHeader("User-Agent"); -+ std::string uAgent = req.GetHeader("User-Agent"); - - std::string forceType = ""; - if (H.GetVar("forcetype").size()){ -- forceType = ",forceType:\"" + H.GetVar("forcetype") + "\""; -+ forceType = ",forceType:\"" + req.GetVar("forcetype") + "\""; - } - - std::string devSkin = ""; -- if (H.GetVar("dev").size()){devSkin = ",skin:\"dev\"";} -- H.SetVar("stream", ""); -- H.SetVar("dev", ""); -+ if (req.GetVar("dev").size()){devSkin = ",skin:\"dev\"";} - devSkin += ",urlappend:\"" + H.allVars() + "\""; - H.SetVar("stream", streamName); - - std::string seekTo = ""; -- if (H.GetVar("t").size()){ -+ if (req.GetVar("t").size()){ - uint64_t autoSeekTime = 0; -- std::string sTime = H.GetVar("t"); -+ std::string sTime = req.GetVar("t"); - unsigned long long h = 0, m = 0, s = 0; - autoSeekTime = JSON::Value(sTime).asInt(); - if (sscanf(sTime.c_str(), "%llum%llus", &m, &s) == 2){autoSeekTime = m * 60 + s;} -@@ -385,13 +383,10 @@ namespace Mist{ - streamName + "\").addEventListener(\"initialized\",f);"; - } - } -- -- H.Clean(); -+ - H.SetHeader("Content-Type", "text/html"); - H.SetHeader("X-UA-Compatible", "IE=edge"); -- H.SetHeader("Server", APPIDENT); -- H.setCORSHeaders(); -- if (method == "OPTIONS" || method == "HEAD"){ -+ if (headersOnly){ - H.SendResponse("200", "OK", myConn); - responded = true; - H.Clean(); -@@ -427,6 +422,7 @@ namespace Mist{ - } - H.SendResponse("200", "OK", myConn); - responded = true; -+ H.Clean(); - } - - JSON::Value OutHTTP::getStatusJSON(std::string &reqHost, const std::string &useragent){ -@@ -634,23 +630,31 @@ namespace Mist{ - - // loop over the added sources, add them to json_resp["sources"] - for (std::set::iterator it = sources.begin(); it != sources.end(); it++){ -- if ((*it)["simul_tracks"].asInt() > 0){json_resp["source"].append(*it);} -+ if ((*it)["simul_tracks"].asInt() > 0){ -+ if (Comms::tknMode & 0x04){ -+ JSON::Value tmp; -+ tmp = (*it); -+ tmp["url"] = tmp["url"].asStringRef() + "?tkn=" + tkn; -+ tmp["relurl"] = tmp["relurl"].asStringRef() + "?tkn=" + tkn; -+ json_resp["source"].append(tmp); -+ }else{ -+ json_resp["source"].append(*it); -+ } -+ } - } - return json_resp; - } - -- void OutHTTP::onHTTP(){ -+ void OutHTTP::respondHTTP(const HTTP::Parser & req, bool headersOnly){ - origStreamName = streamName; -- std::string method = H.method; - -- if (H.GetHeader("X-Mst-Path").size()){mistPath = H.GetHeader("X-Mst-Path");} -+ if (req.GetHeader("X-Mst-Path").size()){mistPath = req.GetHeader("X-Mst-Path");} - - // Handle certbot validations -- if (H.url.substr(0, 28) == "/.well-known/acme-challenge/"){ -+ if (req.url.substr(0, 28) == "/.well-known/acme-challenge/"){ - std::string cbToken = H.url.substr(28); - jsonForEach(config->getOption("certbot", true), it){ - if (it->asStringRef().substr(0, cbToken.size() + 1) == cbToken + ":"){ -- H.Clean(); - H.SetHeader("Content-Type", "text/plain"); - H.SetHeader("Server", APPIDENT); - H.setCORSHeaders(); -@@ -661,9 +665,7 @@ namespace Mist{ - return; - } - } -- H.Clean(); - H.SetHeader("Content-Type", "text/plain"); -- H.SetHeader("Server", APPIDENT); - H.setCORSHeaders(); - H.SetBody("No matching validation found for token '" + cbToken + "'"); - H.SendResponse("404", "Not found", myConn); -@@ -672,12 +674,11 @@ namespace Mist{ - return; - } - -- if (H.url == "/crossdomain.xml"){ -- H.Clean(); -+ if (req.url == "/crossdomain.xml"){ - H.SetHeader("Content-Type", "text/xml"); - H.SetHeader("Server", APPIDENT); - H.setCORSHeaders(); -- if (method == "OPTIONS" || method == "HEAD"){ -+ if (headersOnly){ - H.SendResponse("200", "OK", myConn); - responded = true; - H.Clean(); -@@ -693,12 +694,11 @@ namespace Mist{ - return; - }// crossdomain.xml - -- if (H.url == "/clientaccesspolicy.xml"){ -- H.Clean(); -+ if (req.url == "/clientaccesspolicy.xml"){ - H.SetHeader("Content-Type", "text/xml"); - H.SetHeader("Server", APPIDENT); - H.setCORSHeaders(); -- if (method == "OPTIONS" || method == "HEAD"){ -+ if (headersOnly){ - H.SendResponse("200", "OK", myConn); - responded = true; - H.Clean(); -@@ -716,8 +716,7 @@ namespace Mist{ - return; - }// clientaccesspolicy.xml - -- if (H.url == "/flashplayer.swf"){ -- H.Clean(); -+ if (req.url == "/flashplayer.swf"){ - H.SetHeader("Content-Type", "application/x-shockwave-flash"); - H.SetHeader("Server", APPIDENT); - H.SetBody((const char *)FlashMediaPlayback_101_swf, FlashMediaPlayback_101_swf_len); -@@ -725,8 +724,7 @@ namespace Mist{ - responded = true; - return; - } -- if (H.url == "/oldflashplayer.swf"){ -- H.Clean(); -+ if (req.url == "/oldflashplayer.swf"){ - H.SetHeader("Content-Type", "application/x-shockwave-flash"); - H.SetHeader("Server", APPIDENT); - H.SetBody((const char *)FlashMediaPlayback_swf, FlashMediaPlayback_swf_len); -@@ -735,20 +733,21 @@ namespace Mist{ - return; - } - // send logo icon -- if (H.url.length() > 4 && H.url.substr(H.url.length() - 4, 4) == ".ico"){ -- sendIcon(); -+ if (req.url.length() > 4 && req.url.substr(req.url.length() - 4, 4) == ".ico"){ -+ sendIcon(headersOnly); - return; - } - - // send generic HTML page -- if (H.url.length() > 6 && H.url.substr(H.url.length() - 5, 5) == ".html"){ -- HTMLResponse(); -+ if (req.url.length() > 6 && req.url.substr(req.url.length() - 5, 5) == ".html"){ -+ HTMLResponse(req, headersOnly); - return; - } - - // send smil MBR index -- if (H.url.length() > 6 && H.url.substr(H.url.length() - 5, 5) == ".smil"){ -- std::string reqHost = HTTP::URL(H.GetHeader("Host")).host; -+ if (req.url.length() > 6 && req.url.substr(req.url.length() - 5, 5) == ".smil"){ -+ HTTPOutput::respondHTTP(req, headersOnly); -+ std::string reqHost = HTTP::URL(req.GetHeader("Host")).host; - std::string port, url_rel; - std::string trackSources; // this string contains all track sources for MBR smil - { -@@ -782,11 +781,8 @@ namespace Mist{ - } - } - -- H.Clean(); - H.SetHeader("Content-Type", "application/smil"); -- H.SetHeader("Server", APPIDENT); -- H.setCORSHeaders(); -- if (method == "OPTIONS" || method == "HEAD"){ -+ if (headersOnly){ - H.SendResponse("200", "OK", myConn); - responded = true; - H.Clean(); -@@ -800,24 +796,22 @@ namespace Mist{ - return; - } - -- if ((H.url.length() > 9 && H.url.substr(0, 6) == "/info_" && H.url.substr(H.url.length() - 3, 3) == ".js") || -- (H.url.length() > 9 && H.url.substr(0, 6) == "/json_" && H.url.substr(H.url.length() - 3, 3) == ".js")){ -- if (websocketHandler()){return;} -- std::string reqHost = HTTP::URL(H.GetHeader("Host")).host; -- std::string useragent = H.GetVar("ua"); -- if (!useragent.size()){useragent = H.GetHeader("User-Agent");} -+ if ((req.url.length() > 9 && req.url.substr(0, 6) == "/info_" && req.url.substr(req.url.length() - 3, 3) == ".js") || -+ (req.url.length() > 9 && req.url.substr(0, 6) == "/json_" && req.url.substr(req.url.length() - 3, 3) == ".js")){ -+ HTTPOutput::respondHTTP(req, headersOnly); -+ if (websocketHandler(req, headersOnly)){return;} -+ std::string reqHost = HTTP::URL(req.GetHeader("Host")).host; -+ std::string useragent = req.GetVar("ua"); -+ if (!useragent.size()){useragent = req.GetHeader("User-Agent");} - std::string response; -- std::string rURL = H.url; -- if (method != "OPTIONS" && method != "HEAD"){initialize();} -- H.Clean(); -- H.SetHeader("Server", APPIDENT); -- H.setCORSHeaders(); -+ std::string rURL = req.url; -+ if (headersOnly){initialize();} - if (rURL.substr(0, 6) != "/json_"){ - H.SetHeader("Content-Type", "application/javascript"); - }else{ - H.SetHeader("Content-Type", "application/json"); - } -- if (method == "OPTIONS" || method == "HEAD"){ -+ if (headersOnly){ - H.SendResponse("200", "OK", myConn); - responded = true; - H.Clean(); -@@ -837,9 +831,9 @@ namespace Mist{ - return; - }// embed code generator - -- if ((H.url == "/player.js") || ((H.url.substr(0, 7) == "/embed_") && (H.url.length() > 10) && -- (H.url.substr(H.url.length() - 3, 3) == ".js"))){ -- HTTP::URL fullURL(H.GetHeader("Host")); -+ if ((req.url == "/player.js") || ((req.url.substr(0, 7) == "/embed_") && (req.url.length() > 10) && -+ (req.url.substr(H.url.length() - 3, 3) == ".js"))){ -+ HTTP::URL fullURL(req.GetHeader("Host")); - if (!fullURL.protocol.size()){fullURL.protocol = getProtocolForPort(fullURL.getPort());} - if (config->getString("pubaddr") != ""){ - HTTP::URL altURL(config->getString("pubaddr")); -@@ -850,12 +844,17 @@ namespace Mist{ - } - if (mistPath.size()){fullURL = mistPath;} - std::string response; -- std::string rURL = H.url; -- H.Clean(); -+ std::string rURL = req.url; -+ -+ if ((rURL.substr(0, 7) == "/embed_") && (rURL.length() > 10) && -+ (rURL.substr(rURL.length() - 3, 3) == ".js")){ -+ HTTPOutput::respondHTTP(req, headersOnly); -+ } -+ - H.SetHeader("Server", APPIDENT); - H.setCORSHeaders(); - H.SetHeader("Content-Type", "application/javascript; charset=utf-8"); -- if (method == "OPTIONS" || method == "HEAD"){ -+ if (headersOnly){ - H.SendResponse("200", "OK", myConn); - responded = true; - H.Clean(); -@@ -933,14 +932,13 @@ namespace Mist{ - return; - } - -- if (H.url.substr(0, 7) == "/skins/"){ -+ if (req.url.substr(0, 7) == "/skins/"){ - std::string response; -- std::string url = H.url; -- H.Clean(); -+ std::string url = req.url; - H.SetHeader("Server", APPIDENT); - H.setCORSHeaders(); - H.SetHeader("Content-Type", "text/css"); -- if (method == "OPTIONS" || method == "HEAD"){ -+ if (headersOnly){ - H.SendResponse("200", "OK", myConn); - responded = true; - H.Clean(); -@@ -970,13 +968,12 @@ namespace Mist{ - H.Clean(); - return; - } -- if (H.url == "/videojs.js"){ -+ if (req.url == "/videojs.js"){ - std::string response; -- H.Clean(); - H.SetHeader("Server", APPIDENT); - H.setCORSHeaders(); - H.SetHeader("Content-Type", "application/javascript"); -- if (method == "OPTIONS" || method == "HEAD"){ -+ if (headersOnly){ - H.SendResponse("200", "OK", myConn); - responded = true; - H.Clean(); -@@ -992,13 +989,12 @@ namespace Mist{ - H.Clean(); - return; - } -- if (H.url == "/dashjs.js"){ -+ if (req.url == "/dashjs.js"){ - std::string response; -- H.Clean(); - H.SetHeader("Server", APPIDENT); - H.setCORSHeaders(); - H.SetHeader("Content-Type", "application/javascript"); -- if (method == "OPTIONS" || method == "HEAD"){ -+ if (headersOnly){ - H.SendResponse("200", "OK", myConn); - responded = true; - H.Clean(); -@@ -1016,13 +1012,12 @@ namespace Mist{ - H.Clean(); - return; - } -- if (H.url == "/webrtc.js"){ -+ if (req.url == "/webrtc.js"){ - std::string response; -- H.Clean(); - H.SetHeader("Server", APPIDENT); - H.setCORSHeaders(); - H.SetHeader("Content-Type", "application/javascript"); -- if (method == "OPTIONS" || method == "HEAD"){ -+ if (headersOnly){ - H.SendResponse("200", "OK", myConn); - responded = true; - H.Clean(); -@@ -1038,13 +1033,12 @@ namespace Mist{ - H.Clean(); - return; - } -- if (H.url == "/flv.js"){ -+ if (req.url == "/flv.js"){ - std::string response; -- H.Clean(); - H.SetHeader("Server", "MistServer/" PACKAGE_VERSION); - H.setCORSHeaders(); - H.SetHeader("Content-Type", "application/javascript"); -- if (method == "OPTIONS" || method == "HEAD"){ -+ if (headersOnly){ - H.SendResponse("200", "OK", myConn); - H.Clean(); - return; -@@ -1058,13 +1052,12 @@ namespace Mist{ - H.Clean(); - return; - } -- if (H.url == "/hlsjs.js"){ -+ if (req.url == "/hlsjs.js"){ - std::string response; -- H.Clean(); - H.SetHeader("Server", "MistServer/" PACKAGE_VERSION); - H.setCORSHeaders(); - H.SetHeader("Content-Type", "application/javascript"); -- if (method == "OPTIONS" || method == "HEAD"){ -+ if (headersOnly){ - H.SendResponse("200", "OK", myConn); - H.Clean(); - return; -@@ -1084,7 +1077,7 @@ namespace Mist{ - H.SetHeader("Server", "MistServer/" PACKAGE_VERSION); - H.setCORSHeaders(); - H.SetHeader("Content-Type", "application/javascript"); -- if (method == "OPTIONS" || method == "HEAD"){ -+ if (headersOnly){ - H.SendResponse("200", "OK", myConn); - H.Clean(); - return; -@@ -1100,15 +1093,13 @@ namespace Mist{ - } - } - -- void OutHTTP::sendIcon(){ -- std::string method = H.method; -- H.Clean(); -+ void OutHTTP::sendIcon(bool headersOnly){ - #include "../icon.h" - H.SetHeader("Content-Type", "image/x-icon"); - H.SetHeader("Server", APPIDENT); - H.SetHeader("Content-Length", icon_len); - H.setCORSHeaders(); -- if (method == "OPTIONS" || method == "HEAD"){ -+ if (headersOnly){ - H.SendResponse("200", "OK", myConn); - responded = true; - H.Clean(); -@@ -1120,16 +1111,16 @@ namespace Mist{ - H.Clean(); - } - -- bool OutHTTP::websocketHandler(){ -+ bool OutHTTP::websocketHandler(const HTTP::Parser & req, bool headersOnly){ - stayConnected = true; -- std::string reqHost = HTTP::URL(H.GetHeader("Host")).host; -- if (H.GetHeader("X-Mst-Path").size()){mistPath = H.GetHeader("X-Mst-Path");} -- std::string useragent = H.GetVar("ua"); -- if (!useragent.size()){useragent = H.GetHeader("User-Agent");} -- std::string upgradeHeader = H.GetHeader("Upgrade"); -+ std::string reqHost = HTTP::URL(req.GetHeader("Host")).host; -+ if (req.GetHeader("X-Mst-Path").size()){mistPath = req.GetHeader("X-Mst-Path");} -+ std::string useragent = req.GetVar("ua"); -+ if (!useragent.size()){useragent = req.GetHeader("User-Agent");} -+ std::string upgradeHeader = req.GetHeader("Upgrade"); - Util::stringToLower(upgradeHeader); - if (upgradeHeader != "websocket"){return false;} -- HTTP::Websocket ws(myConn, H); -+ HTTP::Websocket ws(myConn, req, H); - if (!ws){return false;} - setBlocking(false); - // start the stream, if needed -diff --git a/src/output/output_http_internal.h b/src/output/output_http_internal.h -index 774eb186..8e610145 100644 ---- a/src/output/output_http_internal.h -+++ b/src/output/output_http_internal.h -@@ -10,10 +10,10 @@ namespace Mist{ - virtual void onFail(const std::string &msg, bool critical = false); - /// preHTTP is disabled in the internal HTTP output, since most don't need the stream alive to work - virtual void preHTTP(){}; -- void HTMLResponse(); -- void onHTTP(); -- void sendIcon(); -- bool websocketHandler(); -+ void HTMLResponse(const HTTP::Parser & req, bool headersOnly); -+ void respondHTTP(const HTTP::Parser & req, bool headersOnly); -+ void sendIcon(bool headersOnly); -+ bool websocketHandler(const HTTP::Parser & req, bool headersOnly); - JSON::Value getStatusJSON(std::string &reqHost, const std::string &useragent = ""); - bool stayConnected; - virtual bool onFinish(){return stayConnected;} -diff --git a/src/output/output_sdp.cpp b/src/output/output_sdp.cpp -index 6baf9067..50d491e8 100644 ---- a/src/output/output_sdp.cpp -+++ b/src/output/output_sdp.cpp -@@ -141,6 +141,18 @@ namespace Mist{ - } - } - -+ std::string OutSDP::getConnectedHost(){ -+ if (!sdpState.tracks.size()) { return Output::getConnectedHost(); } -+ std::string hostname; -+ uint32_t port; -+ sdpState.tracks[0].data.GetDestination(hostname, port); -+ return hostname; -+ } -+ std::string OutSDP::getConnectedBinHost(){ -+ if (!sdpState.tracks.size()) { return Output::getConnectedBinHost(); } -+ return sdpState.tracks[0].data.getBinDestination(); -+ } -+ - void OutSDP::sendNext(){ - char *dataPointer = 0; - size_t dataLen = 0; -diff --git a/src/output/output_sdp.h b/src/output/output_sdp.h -index 2d4b3368..6c466753 100644 ---- a/src/output/output_sdp.h -+++ b/src/output/output_sdp.h -@@ -16,6 +16,8 @@ namespace Mist{ - void sendNext(); - void sendHeader(); - bool onFinish(); -+ std::string getConnectedHost(); -+ std::string getConnectedBinHost(); - - private: - void initTracks(uint32_t & port, std::string targetIP); -diff --git a/src/output/output_ts.cpp b/src/output/output_ts.cpp -index 561bf9aa..bf46663b 100644 ---- a/src/output/output_ts.cpp -+++ b/src/output/output_ts.cpp -@@ -239,6 +239,18 @@ namespace Mist{ - } - } - -+ std::string OutTS::getConnectedHost(){ -+ if (!pushOut) { return Output::getConnectedHost(); } -+ std::string hostname; -+ uint32_t port; -+ pushSock.GetDestination(hostname, port); -+ return hostname; -+ } -+ std::string OutTS::getConnectedBinHost(){ -+ if (!pushOut) { return Output::getConnectedBinHost(); } -+ return pushSock.getBinDestination(); -+ } -+ - bool OutTS::listenMode(){return !(config->getString("target").size());} - - void OutTS::onRequest(){ -diff --git a/src/output/output_ts.h b/src/output/output_ts.h -index 32aa6958..0c38cc70 100644 ---- a/src/output/output_ts.h -+++ b/src/output/output_ts.h -@@ -12,6 +12,8 @@ namespace Mist{ - virtual void initialSeek(); - bool isReadyForPlay(); - void onRequest(); -+ std::string getConnectedHost(); -+ std::string getConnectedBinHost(); - - private: - size_t udpSize; -diff --git a/src/output/output_tsrist.cpp b/src/output/output_tsrist.cpp -index c4e03f80..10b29673 100644 ---- a/src/output/output_tsrist.cpp -+++ b/src/output/output_tsrist.cpp -@@ -178,6 +178,20 @@ namespace Mist{ - } - - OutTSRIST::~OutTSRIST(){} -+ -+ std::string OutTSRIST::getConnectedHost(){ -+ if (!pushOut) { return Output::getConnectedHost(); } -+ return target.host; -+ } -+ std::string OutTSRIST::getConnectedBinHost(){ -+ if (!pushOut) { return Output::getConnectedBinHost(); } -+ std::string binHost = Socket::getBinForms(target.host); -+ if (binHost.size() > 16){ binHost = binHost.substr(0, 16); } -+ if (binHost.size() < 16){ -+ binHost = std::string("\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\001", 16); -+ } -+ return binHost; -+ } - - void OutTSRIST::init(Util::Config *cfg){ - Output::init(cfg); -@@ -319,7 +333,7 @@ namespace Mist{ - } - } - -- void OutTSRIST::connStats(uint64_t now, Comms::Statistics &statComm){ -+ void OutTSRIST::connStats(uint64_t now, Comms::Connections &statComm){ - if (!myConn){return;} - statComm.setUp(upBytes); - statComm.setDown(0); -diff --git a/src/output/output_tsrist.h b/src/output/output_tsrist.h -index c9a5db39..80ee2376 100644 ---- a/src/output/output_tsrist.h -+++ b/src/output/output_tsrist.h -@@ -16,9 +16,11 @@ namespace Mist{ - bool isReadyForPlay(){return true;} - virtual void requestHandler(); - static void listener(Util::Config &conf, int (*callback)(Socket::Connection &S)); -+ std::string getConnectedHost(); -+ std::string getConnectedBinHost(); - - protected: -- virtual void connStats(uint64_t now, Comms::Statistics &statComm); -+ virtual void connStats(uint64_t now, Comms::Connections &statComm); - //virtual std::string getConnectedHost(){ - // return srtConn.remotehost; - //} -diff --git a/src/session.cpp b/src/session.cpp -index 3865e0ec..d2331c53 100644 ---- a/src/session.cpp -+++ b/src/session.cpp -@@ -7,17 +7,31 @@ - #include - #include - #include --// Stats of connections which have closed are added to these global counters --uint64_t globalNow = 0; -+ -+// Global counters -+uint64_t now = Util::bootSecs(); -+uint64_t currentConnections = 0; -+uint64_t lastSecond = 0; - uint64_t globalTime = 0; - uint64_t globalDown = 0; - uint64_t globalUp = 0; - uint64_t globalPktcount = 0; - uint64_t globalPktloss = 0; - uint64_t globalPktretrans = 0; -+// Stores last values of each connection -+std::map connTime; -+std::map connDown; -+std::map connUp; -+std::map connPktcount; -+std::map connPktloss; -+std::map connPktretrans; - // Counts the duration a connector has been active - std::map connectorCount; - std::map connectorLastActive; -+std::map hostCount; -+std::map hostLastActive; -+std::map streamCount; -+std::map streamLastActive; - // Set to True when a session gets invalidated, so that we know to run a new USER_NEW trigger - bool forceTrigger = false; - void handleSignal(int signum){ -@@ -26,96 +40,141 @@ void handleSignal(int signum){ - } - } - --void userOnActive(uint64_t &connections){ -- ++connections; --} -+const char nullAddress[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - --std::string getEnvWithDefault(const std::string variableName, const std::string defaultValue){ -- const char* value = getenv(variableName.c_str()); -- if (value){ -- unsetenv(variableName.c_str()); -- return value; -- }else{ -- return defaultValue; -- } -+void userOnActive(Comms::Connections &connections, size_t idx){ -+ uint64_t lastUpdate = connections.getNow(idx); -+ if (lastUpdate < now - 10){return;} -+ ++currentConnections; -+ std::string thisConnector = connections.getConnector(idx); -+ std::string thisStreamName = connections.getStream(idx); -+ const std::string& thisHost = connections.getHost(idx); -+ -+ if (connections.getLastSecond(idx) > lastSecond){lastSecond = connections.getLastSecond(idx);} -+ // Save info on the latest active stream, protocol and host separately -+ if (thisConnector.size() && thisConnector != "HTTP"){ -+ connectorCount[thisConnector]++; -+ if (connectorLastActive[thisConnector] < lastUpdate){connectorLastActive[thisConnector] = lastUpdate;} -+ } -+ if (thisStreamName.size()){ -+ streamCount[thisStreamName]++; -+ if (streamLastActive[thisStreamName] < lastUpdate){streamLastActive[thisStreamName] = lastUpdate;} -+ } -+ if (memcmp(thisHost.data(), nullAddress, 16)){ -+ hostCount[thisHost]++; -+ if (!hostLastActive.count(thisHost) || hostLastActive[thisHost] < lastUpdate){hostLastActive[thisHost] = lastUpdate;} -+ } -+ // Sanity checks -+ if (connections.getDown(idx) < connDown[idx]){ -+ WARN_MSG("Connection downloaded bytes should be a counter, but has decreased in value"); -+ connDown[idx] = connections.getDown(idx); -+ } -+ if (connections.getUp(idx) < connUp[idx]){ -+ WARN_MSG("Connection uploaded bytes should be a counter, but has decreased in value"); -+ connUp[idx] = connections.getUp(idx); -+ } -+ if (connections.getPacketCount(idx) < connPktcount[idx]){ -+ WARN_MSG("Connection packet count should be a counter, but has decreased in value"); -+ connPktcount[idx] = connections.getPacketCount(idx); -+ } -+ if (connections.getPacketLostCount(idx) < connPktloss[idx]){ -+ WARN_MSG("Connection packet loss count should be a counter, but has decreased in value"); -+ connPktloss[idx] = connections.getPacketLostCount(idx); -+ } -+ if (connections.getPacketRetransmitCount(idx) < connPktretrans[idx]){ -+ WARN_MSG("Connection packets retransmitted should be a counter, but has decreased in value"); -+ connPktretrans[idx] = connections.getPacketRetransmitCount(idx); -+ } -+ // Add increase in stats to global stats -+ globalDown += connections.getDown(idx) - connDown[idx]; -+ globalUp += connections.getUp(idx) - connUp[idx]; -+ globalPktcount += connections.getPacketCount(idx) - connPktcount[idx]; -+ globalPktloss += connections.getPacketLostCount(idx) - connPktloss[idx]; -+ globalPktretrans += connections.getPacketRetransmitCount(idx) - connPktretrans[idx]; -+ // Set last values of this connection -+ connTime[idx]++; -+ connDown[idx] = connections.getDown(idx); -+ connUp[idx] = connections.getUp(idx); -+ connPktcount[idx] = connections.getPacketCount(idx); -+ connPktloss[idx] = connections.getPacketLostCount(idx); -+ connPktretrans[idx] = connections.getPacketRetransmitCount(idx); - } - --/// \brief Adds stats of closed connections to global counters -+/// \brief Remove mappings of inactive connections - void userOnDisconnect(Comms::Connections & connections, size_t idx){ -- std::string thisConnector = connections.getConnector(idx); -- if (thisConnector != ""){ -- connectorCount[thisConnector] += connections.getTime(idx); -- } -- globalTime += connections.getTime(idx); -- globalDown += connections.getDown(idx); -- globalUp += connections.getUp(idx); -- globalPktcount += connections.getPacketCount(idx); -- globalPktloss += connections.getPacketLostCount(idx); -- globalPktretrans += connections.getPacketRetransmitCount(idx); -+ connTime.erase(idx); -+ connDown.erase(idx); -+ connUp.erase(idx); -+ connPktcount.erase(idx); -+ connPktloss.erase(idx); -+ connPktretrans.erase(idx); - } - - int main(int argc, char **argv){ - Comms::Connections connections; - Comms::Sessions sessions; - uint64_t lastSeen = Util::bootSecs(); -- uint64_t currentConnections = 0; - Util::redirectLogsIfNeeded(); - signal(SIGUSR1, handleSignal); - // Init config and parse arguments - Util::Config config = Util::Config("MistSession"); - JSON::Value option; -+ char * tmpStr = 0; - - option.null(); - option["arg_num"] = 1; - option["arg"] = "string"; - option["help"] = "Session identifier of the entire session"; -- option["default"] = ""; - config.addOption("sessionid", option); - -- option.null(); -- option["long"] = "sessionmode"; -- option["short"] = "m"; -- option["arg"] = "integer"; -- option["default"] = 0; -- config.addOption("sessionmode", option); -- - option.null(); - option["long"] = "streamname"; -- option["short"] = "n"; -+ option["short"] = "s"; - option["arg"] = "string"; -- option["default"] = ""; -+ option["help"] = "Stream name initial value. May also be passed as SESSION_STREAM"; -+ tmpStr = getenv("SESSION_STREAM"); -+ option["default"] = tmpStr?tmpStr:""; - config.addOption("streamname", option); - - option.null(); - option["long"] = "ip"; - option["short"] = "i"; - option["arg"] = "string"; -- option["default"] = ""; -+ option["help"] = "IP address initial value. May also be passed as SESSION_IP"; -+ tmpStr = getenv("SESSION_IP"); -+ option["default"] = tmpStr?tmpStr:""; - config.addOption("ip", option); - - option.null(); -- option["long"] = "sid"; -- option["short"] = "s"; -+ option["long"] = "tkn"; -+ option["short"] = "t"; - option["arg"] = "string"; -- option["default"] = ""; -- config.addOption("sid", option); -+ option["help"] = "Client-side session ID initial value. May also be passed as SESSION_TKN"; -+ tmpStr = getenv("SESSION_TKN"); -+ option["default"] = tmpStr?tmpStr:""; -+ config.addOption("tkn", option); - - option.null(); - option["long"] = "protocol"; - option["short"] = "p"; - option["arg"] = "string"; -- option["default"] = ""; -+ option["help"] = "Protocol initial value. May also be passed as SESSION_PROTOCOL"; -+ tmpStr = getenv("SESSION_PROTOCOL"); -+ option["default"] = tmpStr?tmpStr:""; - config.addOption("protocol", option); - - option.null(); - option["long"] = "requrl"; - option["short"] = "r"; - option["arg"] = "string"; -- option["default"] = ""; -+ option["help"] = "Request URL initial value. May also be passed as SESSION_REQURL"; -+ tmpStr = getenv("SESSION_REQURL"); -+ option["default"] = tmpStr?tmpStr:""; - config.addOption("requrl", option); - - config.activate(); - if (!(config.parseArgs(argc, argv))){ -+ config.printHelp(std::cout); - FAIL_MSG("Cannot start a new session due to invalid arguments"); - return 1; - } -@@ -123,24 +182,17 @@ int main(int argc, char **argv){ - const uint64_t bootTime = Util::getMicros(); - // Get session ID, session mode and other variables used as payload for the USER_NEW and USER_END triggers - const std::string thisStreamName = config.getString("streamname"); -- const std::string thisHost = config.getString("ip"); -- const std::string thisSid = config.getString("sid"); -+ const std::string thisToken = config.getString("tkn"); - const std::string thisProtocol = config.getString("protocol"); - const std::string thisReqUrl = config.getString("requrl"); - const std::string thisSessionId = config.getString("sessionid"); -- const uint64_t sessionMode = config.getInteger("sessionmode"); -- -- if (thisSessionId == "" || thisProtocol == "" || thisStreamName == ""){ -- FAIL_MSG("Given the following incomplete arguments: SessionId: '%s', protocol: '%s', stream name: '%s'. Aborting opening a new session", -- thisSessionId.c_str(), thisProtocol.c_str(), thisStreamName.c_str()); -- return 1; -- } -+ std::string thisHost = Socket::getBinForms(config.getString("ip")); -+ if (thisHost.size() > 16){thisHost = thisHost.substr(0, 16);} - -- MEDIUM_MSG("Starting a new session for sessionId '%s'", thisSessionId.c_str()); -- if (sessionMode < 1 || sessionMode > 15) { -- FAIL_MSG("Invalid session mode of value %lu. Should be larger than 0 and smaller than 16", sessionMode); -- return 1; -- } -+ std::string ipHex; -+ Socket::hostBytesToStr(thisHost.c_str(), thisHost.size(), ipHex); -+ VERYHIGH_MSG("Starting a new session. Passed variables are stream name '%s', session token '%s', protocol '%s', requested URL '%s', IP '%s' and session id '%s'", -+ thisStreamName.c_str(), thisToken.c_str(), thisProtocol.c_str(), thisReqUrl.c_str(), ipHex.c_str(), thisSessionId.c_str()); - - // Try to lock to ensure we are the only process initialising this session - IPC::semaphore sessionLock; -@@ -174,194 +226,200 @@ int main(int argc, char **argv){ - sessionLock.post(); - return 1; - } -- // Open the shared memory page containing statistics for each individual connection in this session -- connections.reload(thisStreamName, thisHost, thisSid, thisProtocol, thisReqUrl, sessionMode, true, false); -+ - // Initialise global session data - sessions.setHost(thisHost); - sessions.setSessId(thisSessionId); - sessions.setStream(thisStreamName); -- sessionLock.post(); -+ if (thisProtocol.size() && thisProtocol != "HTTP"){connectorLastActive[thisProtocol] = now;} -+ if (thisStreamName.size()){streamLastActive[thisStreamName] = now;} -+ if (memcmp(thisHost.data(), nullAddress, 16)){hostLastActive[thisHost] = now;} -+ -+ // Open the shared memory page containing statistics for each individual connection in this session -+ connections.reload(thisSessionId, true); - - // Determine session type, since triggers only get run for viewer type sessions - uint64_t thisType = 0; - if (thisSessionId[0] == 'I'){ -- INFO_MSG("Started new input session %s in %lu microseconds", thisSessionId.c_str(), Util::getMicros(bootTime)); - thisType = 1; -- } -- else if (thisSessionId[0] == 'O'){ -- INFO_MSG("Started new output session %s in %lu microseconds", thisSessionId.c_str(), Util::getMicros(bootTime)); -+ } else if (thisSessionId[0] == 'O'){ - thisType = 2; -- } -- else{ -- INFO_MSG("Started new viewer session %s in %lu microseconds", thisSessionId.c_str(), Util::getMicros(bootTime)); -+ } else if (thisSessionId[0] == 'U'){ -+ thisType = 3; - } - - // Do a USER_NEW trigger if it is defined for this stream - if (!thisType && Triggers::shouldTrigger("USER_NEW", thisStreamName)){ -- std::string payload = thisStreamName + "\n" + thisHost + "\n" + -- thisSid + "\n" + thisProtocol + -+ std::string payload = thisStreamName + "\n" + config.getString("ip") + "\n" + -+ thisToken + "\n" + thisProtocol + - "\n" + thisReqUrl + "\n" + thisSessionId; - if (!Triggers::doTrigger("USER_NEW", payload, thisStreamName)){ - // Mark all connections of this session as finished, since this viewer is not allowed to view this stream -+ Util::logExitReason("Session rejected by USER_NEW"); - connections.setExit(); - connections.finishAll(); - } - } - -- uint64_t lastSecond = 0; -- uint64_t now = 0; -- uint64_t time = 0; -- uint64_t down = 0; -- uint64_t up = 0; -- uint64_t pktcount = 0; -- uint64_t pktloss = 0; -- uint64_t pktretrans = 0; -- std::string connector = ""; -+ //start allowing viewers -+ sessionLock.post(); -+ -+ INFO_MSG("Started new session %s in %.3f ms", thisSessionId.c_str(), (double)Util::getMicros(bootTime)/1000.0); -+ - // Stay active until Mist exits or we no longer have an active connection -- while (config.is_active && (currentConnections || Util::bootSecs() - lastSeen <= 10)){ -- time = 0; -- connector = ""; -- down = 0; -- up = 0; -- pktcount = 0; -- pktloss = 0; -- pktretrans = 0; -+ while (config.is_active && (currentConnections || now - lastSeen <= STATS_DELAY) && !connections.getExit()){ - currentConnections = 0; -+ lastSecond = 0; -+ now = Util::bootSecs(); - -- // Count active connections -- COMM_LOOP(connections, userOnActive(currentConnections), userOnDisconnect(connections, id)); - // Loop through all connection entries to get a summary of statistics -- for (uint64_t idx = 0; idx < connections.recordCount(); idx++){ -- if (connections.getStatus(idx) == COMM_STATUS_INVALID || connections.getStatus(idx) & COMM_STATUS_DISCONNECT){continue;} -- uint64_t thisLastSecond = connections.getLastSecond(idx); -- std::string thisConnector = connections.getConnector(idx); -- // Save info on the latest active connection separately -- if (thisLastSecond > lastSecond){ -- lastSecond = thisLastSecond; -- now = connections.getNow(idx); -- } -- connectorLastActive[thisConnector] = thisLastSecond; -- // Sum all other variables -- time += connections.getTime(idx); -- down += connections.getDown(idx); -- up += connections.getUp(idx); -- pktcount += connections.getPacketCount(idx); -- pktloss += connections.getPacketLostCount(idx); -- pktretrans += connections.getPacketRetransmitCount(idx); -+ COMM_LOOP(connections, userOnActive(connections, id), userOnDisconnect(connections, id)); -+ if (currentConnections){ -+ globalTime++; -+ lastSeen = now; - } - -- // Convert connector duration to string -- std::stringstream connectorSummary; -- bool addDelimiter = false; -- connectorSummary << "{"; -- for (std::map::iterator it = connectorLastActive.begin(); -- it != connectorLastActive.end(); ++it){ -- if (lastSecond - it->second < 10000){ -- connectorSummary << (addDelimiter ? "," : "") << it->first; -- addDelimiter = true; -- } -- } -- connectorSummary << "}"; -- -- // Write summary to global statistics -- sessions.setTime(time + globalTime); -- sessions.setDown(down + globalDown); -- sessions.setUp(up + globalUp); -- sessions.setPacketCount(pktcount + globalPktcount); -- sessions.setPacketLostCount(pktloss + globalPktloss); -- sessions.setPacketRetransmitCount(pktretrans + globalPktretrans); -+ -+ sessions.setTime(globalTime); -+ sessions.setDown(globalDown); -+ sessions.setUp(globalUp); -+ sessions.setPacketCount(globalPktcount); -+ sessions.setPacketLostCount(globalPktloss); -+ sessions.setPacketRetransmitCount(globalPktretrans); - sessions.setLastSecond(lastSecond); -- sessions.setConnector(connectorSummary.str()); - sessions.setNow(now); - -+ if (currentConnections){ -+ { -+ // Convert active protocols to string -+ std::stringstream connectorSummary; -+ for (std::map::iterator it = connectorLastActive.begin(); -+ it != connectorLastActive.end(); ++it){ -+ if (now - it->second < STATS_DELAY){ -+ connectorSummary << (connectorSummary.str().size() ? "," : "") << it->first; -+ } -+ } -+ sessions.setConnector(connectorSummary.str()); -+ } -+ -+ { -+ // Set active host to last active or 0 if there were various hosts active recently -+ std::string thisHost; -+ for (std::map::iterator it = hostLastActive.begin(); -+ it != hostLastActive.end(); ++it){ -+ if (now - it->second < STATS_DELAY){ -+ if (!thisHost.size()){ -+ thisHost = it->first; -+ }else if (thisHost != it->first){ -+ thisHost = nullAddress; -+ break; -+ } -+ } -+ } -+ if (!thisHost.size()){ -+ thisHost = nullAddress; -+ } -+ sessions.setHost(thisHost); -+ } -+ -+ { -+ // Set active stream name to last active or "" if there were multiple streams active recently -+ std::string thisStream = ""; -+ for (std::map::iterator it = streamLastActive.begin(); -+ it != streamLastActive.end(); ++it){ -+ if (now - it->second < STATS_DELAY){ -+ if (!thisStream.size()){ -+ thisStream = it->first; -+ }else if (thisStream != it->first){ -+ thisStream = ""; -+ break; -+ } -+ } -+ } -+ sessions.setStream(thisStream); -+ } -+ } -+ - // Retrigger USER_NEW if a re-sync was requested - if (!thisType && forceTrigger){ - forceTrigger = false; -+ std::string host; -+ Socket::hostBytesToStr(thisHost.data(), 16, host); - if (Triggers::shouldTrigger("USER_NEW", thisStreamName)){ - INFO_MSG("Triggering USER_NEW for stream %s", thisStreamName.c_str()); -- std::string payload = thisStreamName + "\n" + thisHost + "\n" + -- thisSid + "\n" + thisProtocol + -+ std::string payload = thisStreamName + "\n" + host + "\n" + -+ thisToken + "\n" + thisProtocol + - "\n" + thisReqUrl + "\n" + thisSessionId; - if (!Triggers::doTrigger("USER_NEW", payload, thisStreamName)){ - INFO_MSG("USER_NEW rejected stream %s", thisStreamName.c_str()); -+ Util::logExitReason("Session rejected by USER_NEW"); - connections.setExit(); - connections.finishAll(); -+ break; - }else{ - INFO_MSG("USER_NEW accepted stream %s", thisStreamName.c_str()); - } - } - } - -- // Invalidate connections if the session is marked as invalid -- if(connections.getExit()){ -- connections.finishAll(); -- break; -- } - // Remember latest activity so we know when this session ends - if (currentConnections){ -- lastSeen = Util::bootSecs(); - } -- Util::sleep(1000); -+ Util::wait(1000); -+ } -+ if (Util::bootSecs() - lastSeen > STATS_DELAY){ -+ Util::logExitReason("Session inactive for %d seconds", STATS_DELAY); - } - - // Trigger USER_END - if (!thisType && Triggers::shouldTrigger("USER_END", thisStreamName)){ -- lastSecond = 0; -- time = 0; -- down = 0; -- up = 0; -- -- // Get a final summary of this session -- for (uint64_t idx = 0; idx < connections.recordCount(); idx++){ -- if (connections.getStatus(idx) == COMM_STATUS_INVALID || connections.getStatus(idx) & COMM_STATUS_DISCONNECT){continue;} -- uint64_t thisLastSecond = connections.getLastSecond(idx); -- // Set last second to the latest entry -- if (thisLastSecond > lastSecond){ -- lastSecond = thisLastSecond; -- } -- // Count protocol durations across the entire session -- std::string thisConnector = connections.getConnector(idx); -- if (thisConnector != ""){ -- connectorCount[thisConnector] += connections.getTime(idx); -- } -- // Sum all other variables -- time += connections.getTime(idx); -- down += connections.getDown(idx); -- up += connections.getUp(idx); -- } - -- // Convert connector duration to string -+ // Convert connector, host and stream into lists and counts - std::stringstream connectorSummary; -- bool addDelimiter = false; -- connectorSummary << "{"; -- for (std::map::iterator it = connectorCount.begin(); -- it != connectorCount.end(); ++it){ -- connectorSummary << (addDelimiter ? "," : "") << it->first << ":" << it->second; -- addDelimiter = true; -+ std::stringstream connectorTimes; -+ for (std::map::iterator it = connectorCount.begin(); it != connectorCount.end(); ++it){ -+ connectorSummary << (connectorSummary.str().size() ? "," : "") << it->first; -+ connectorTimes << (connectorTimes.str().size() ? "," : "") << it->second; -+ } -+ std::stringstream hostSummary; -+ std::stringstream hostTimes; -+ for (std::map::iterator it = hostCount.begin(); it != hostCount.end(); ++it){ -+ std::string host; -+ Socket::hostBytesToStr(it->first.data(), 16, host); -+ hostSummary << (hostSummary.str().size() ? "," : "") << host; -+ hostTimes << (hostTimes.str().size() ? "," : "") << it->second; -+ } -+ std::stringstream streamSummary; -+ std::stringstream streamTimes; -+ for (std::map::iterator it = streamCount.begin(); it != streamCount.end(); ++it){ -+ streamSummary << (streamSummary.str().size() ? "," : "") << it->first; -+ streamTimes << (streamTimes.str().size() ? "," : "") << it->second; - } -- connectorSummary << "}"; - -- const uint64_t duration = lastSecond - (bootTime / 1000); - std::stringstream summary; -- summary << thisSessionId << "\n" -- << thisStreamName << "\n" -+ summary << thisToken << "\n" -+ << streamSummary.str() << "\n" - << connectorSummary.str() << "\n" -- << thisHost << "\n" -- << duration << "\n" -- << up << "\n" -- << down << "\n" -- << sessions.getTags(); -+ << hostSummary.str() << "\n" -+ << globalTime << "\n" -+ << globalUp << "\n" -+ << globalDown << "\n" -+ << sessions.getTags() << "\n" -+ << hostTimes.str() << "\n" -+ << connectorTimes.str() << "\n" -+ << streamTimes.str() << "\n" -+ << thisSessionId; - Triggers::doTrigger("USER_END", summary.str(), thisStreamName); - } - - if (!thisType && connections.getExit()){ -- WARN_MSG("Session %s has been invalidated since it is not allowed to view stream %s", thisSessionId.c_str(), thisStreamName.c_str()); - uint64_t sleepStart = Util::bootSecs(); - // Keep session invalidated for 10 minutes, or until the session stops -- while (config.is_active && sleepStart - Util::bootSecs() < 600){ -+ while (config.is_active && Util::bootSecs() - sleepStart < SESS_TIMEOUT){ - Util::sleep(1000); -+ if (forceTrigger){break;} - } - } -- INFO_MSG("Shutting down session %s", thisSessionId.c_str()); -+ INFO_MSG("Shutting down session %s: %s", thisSessionId.c_str(), Util::exitReason); - return 0; - } --- -2.25.1 - - -From 1a4a526a112d0bdeaa0f269a133fe55c39a0d129 Mon Sep 17 00:00:00 2001 -From: Cat -Date: Wed, 5 Oct 2022 03:10:04 +0200 -Subject: [PATCH 28/38] LSP: Moved some settings to new "General" tab, added - bitmask inputtype, removed LTSonly code - ---- - lsp/main.css | 30 +++ - lsp/minified.js | 396 ++++++++++++++++---------------- - lsp/mist.js | 582 +++++++++++++++++++++++++++++------------------- - 3 files changed, 590 insertions(+), 418 deletions(-) - -diff --git a/lsp/main.css b/lsp/main.css -index 6c72e6a7..9289d211 100644 ---- a/lsp/main.css -+++ b/lsp/main.css -@@ -279,6 +279,21 @@ main > button { - width: 1.5em; - flex-grow: 0; - } -+.input_container .field_container .field.bitmask { -+ display: flex; -+ flex-flow: row wrap; -+ justify-content: space-between; -+ padding: 0; -+ margin-bottom: 0.5em; -+} -+.input_container .field_container .field.bitmask > label { -+ display: flex; -+ flex-flow: row nowrap; -+ align-items: center; -+ margin-right: -0.25em; -+} -+.input_container .field_container .field.bitmask > label > input { margin: 0; } -+.input_container .field_container .field.bitmask > label > span { margin: 0 0.25em; } - textarea { - font-size: 0.8em; - } -@@ -349,6 +364,20 @@ select option[value=""] { - display: block; - z-index: 0; - } -+.input_container label.active { position: relative; } -+.input_container label.active:after { -+ content: ""; -+ display: block; -+ position: absolute; -+ z-index: -1; -+ top: -0.25em; -+ left: -0.25em; -+ width: 55.5em; -+ bottom: -0.25em; -+ background: linear-gradient(to right,#9ec2da,transparent 55em); -+ opacity: 0.3; -+ filter: blur(0.1em); -+} - .input_container label.active .ih_balloon, - .input_container > .help_container .ih_balloon { - display: block; -@@ -577,6 +606,7 @@ table.valigntop th { - } - .input_container .field_container .field.inputlist > * { - width: 100%; -+ box-sizing: border-box; - } - .input_container .field_container .field.checkcontainer { - font-size: 0.8em; -diff --git a/lsp/minified.js b/lsp/minified.js -index 52a6f6bc..9db52d63 100644 ---- a/lsp/minified.js -+++ b/lsp/minified.js -@@ -1,11 +1,11 @@ --var MD5=function(a){function b(a,b){var c,d,g,i,e;g=a&2147483648;i=b&2147483648;c=a&1073741824;d=b&1073741824;e=(a&1073741823)+(b&1073741823);return c&d?e^2147483648^g^i:c|d?e&1073741824?e^3221225472^g^i:e^1073741824^g^i:e^g^i}function c(a,c,d,g,i,e,f){a=b(a,b(b(c&d|~c&g,i),f));return b(a<>>32-e,c)}function d(a,c,d,g,i,e,f){a=b(a,b(b(c&g|d&~g,i),f));return b(a<>>32-e,c)}function e(a,c,d,g,e,i,f){a=b(a,b(b(c^d^g,e),f));return b(a<>>32-i,c)}function l(a,c,d,g,i,e,f){a=b(a,b(b(d^(c|~g), --i),f));return b(a<>>32-e,c)}function m(a){var b="",c="",d;for(d=0;3>=d;d++)c=a>>>8*d&255,c="0"+c.toString(16),b+=c.substr(c.length-2,2);return b}var f=[],t,o,k,w,h,g,i,j,f=a.replace(/\r\n/g,"\n"),a="";for(t=0;to?a+=String.fromCharCode(o):(127o?a+=String.fromCharCode(o>>6|192):(a+=String.fromCharCode(o>>12|224),a+=String.fromCharCode(o>>6&63|128)),a+=String.fromCharCode(o&63|128));f=a;a=f.length;t=a+8;o=16*((t-t%64)/64+1);k=Array(o-1);for(h=w=0;h>>29;f=k;h=1732584193;g=4023233417;i=2562383102;j=271733878;for(a=0;ac?1*d:a .menu"),main:$("main"),header:$("header"),connection:{status:$("#connection"),user_and_host:$("#user_and_host"),msg:$("#message")}};UI.buildMenu();UI.stored.getOpts();try{if("mistLogin"in sessionStorage){var a=JSON.parse(sessionStorage.mistLogin);mist.user.name=a.name;mist.user.password=a.password;mist.user.host=a.host}}catch(b){}location.hash&&(a=decodeURIComponent(location.hash).substring(1).split("@")[0].split("&"),mist.user.name=a[0],a[1]&&(mist.user.host= -+var MD5=function(a){function b(a,b){var c,d,g,f,e;g=a&2147483648;f=b&2147483648;c=a&1073741824;d=b&1073741824;e=(a&1073741823)+(b&1073741823);return c&d?e^2147483648^g^f:c|d?e&1073741824?e^3221225472^g^f:e^1073741824^g^f:e^g^f}function c(a,c,d,g,f,e,h){a=b(a,b(b(c&d|~c&g,f),h));return b(a<>>32-e,c)}function d(a,c,d,g,e,f,h){a=b(a,b(b(c&g|d&~g,e),h));return b(a<>>32-f,c)}function e(a,c,d,g,f,e,h){a=b(a,b(b(c^d^g,f),h));return b(a<>>32-e,c)}function l(a,c,d,g,e,f,h){a=b(a,b(b(d^(c|~g), -+e),h));return b(a<>>32-f,c)}function n(a){var b="",c="",d;for(d=0;3>=d;d++)c=a>>>8*d&255,c="0"+c.toString(16),b+=c.substr(c.length-2,2);return b}var h=[],t,m,k,w,i,g,f,j,h=a.replace(/\r\n/g,"\n"),a="";for(t=0;tm?a+=String.fromCharCode(m):(127m?a+=String.fromCharCode(m>>6|192):(a+=String.fromCharCode(m>>12|224),a+=String.fromCharCode(m>>6&63|128)),a+=String.fromCharCode(m&63|128));h=a;a=h.length;t=a+8;m=16*((t-t%64)/64+1);k=Array(m-1);for(i=w=0;i>>29;h=k;i=1732584193;g=4023233417;f=2562383102;j=271733878;for(a=0;ac?1*d:a .menu"),main:$("main"),header:$("header"),connection:{status:$("#connection"),user_and_host:$("#user_and_host"),msg:$("#message")}};UI.buildMenu();UI.stored.getOpts();try{if("mistLogin"in sessionStorage){var a=JSON.parse(sessionStorage.mistLogin);mist.user.name=a.name;mist.user.password=a.password;mist.user.host=a.host}}catch(b){}location.hash&&(a=decodeURIComponent(location.hash).substring(1).split("@")[0].split("&"),mist.user.name=a[0],a[1]&&(mist.user.host= - a[1]));mist.send(function(){$(window).trigger("hashchange")},{},{timeout:5,hide:!0});var c=0;$("body > div.filler").on("scroll",function(){var a=$(this).scrollLeft();a!=c&&UI.elements.header.css("margin-right",-1*a+"px");c=a})});$(window).on("hashchange",function(){var a=decodeURIComponent(location.hash).substring(1).split("@");a[1]||(a[1]="");a=a[1].split("&");""==a[0]&&(a[0]="Overview");UI.showTab(a[0],a[1])}); - var MistVideoObject={},otherhost={host:!1,https:!1},UI={debug:!1,elements:{},stored:{getOpts:function(){var a=localStorage.stored;a&&(a=JSON.parse(a));$.extend(!0,this.vars,a);return this.vars},saveOpt:function(a,b){this.vars[a]=b;localStorage.stored=JSON.stringify(this.vars);return this.vars},vars:{helpme:!0}},interval:{clear:function(){"undefined"!=typeof this.opts&&(clearInterval(this.opts.id),delete this.opts)},set:function(a,b){this.opts&&log("[interval]","Set called on interval, but an interval is already active."); - this.opts={delay:b,callback:a};this.opts.id=setInterval(a,b)}},returnTab:["Overview"],countrylist:{AF:"Afghanistan",AX:"Åland Islands",AL:"Albania",DZ:"Algeria",AS:"American Samoa",AD:"Andorra",AO:"Angola",AI:"Anguilla",AQ:"Antarctica",AG:"Antigua and Barbuda",AR:"Argentina",AM:"Armenia",AW:"Aruba",AU:"Australia",AT:"Austria",AZ:"Azerbaijan",BS:"Bahamas",BH:"Bahrain",BD:"Bangladesh",BB:"Barbados",BY:"Belarus",BE:"Belgium",BZ:"Belize",BJ:"Benin",BM:"Bermuda",BT:"Bhutan",BO:"Bolivia, Plurinational State of", -@@ -20,61 +20,61 @@ TV:"Tuvalu",UG:"Uganda",UA:"Ukraine",AE:"United Arab Emirates",GB:"United Kingdo - clearTimeout(this.hiding);delete this.hiding;var c=$(document).height()-$tooltip.outerHeight(),d=$(document).width()-$tooltip.outerWidth();$tooltip.css("left",Math.min(a.pageX+10,d-10));$tooltip.css("top",Math.min(a.pageY+25,c-10));$tooltip.show().addClass("show")},hide:function(){$tooltip=this.element;$tooltip.removeClass("show");this.hiding=setTimeout(function(){$tooltip.hide()},500)},element:$("
").attr("id","tooltip")},humanMime:function(a){var b=!1;switch(a){case "html5/application/vnd.apple.mpegurl":b= - "HLS (TS)";break;case "html5/application/vnd.apple.mpegurl;version=7":b="HLS (CMAF)";break;case "html5/video/webm":b="WebM";break;case "html5/video/mp4":b="MP4";break;case "dash/video/mp4":b="DASH";break;case "flash/11":b="HDS";break;case "flash/10":b="RTMP";break;case "flash/7":b="Progressive";break;case "html5/audio/mp3":b="MP3";break;case "html5/audio/wav":b="WAV";break;case "html5/video/mp2t":case "html5/video/mpeg":b="TS";break;case "html5/application/vnd.ms-sstr+xml":case "html5/application/vnd.ms-ss":b= - "Smooth Streaming";break;case "html5/text/vtt":b="VTT Subtitles";break;case "html5/text/plain":b="SRT Subtitles";break;case "html5/text/javascript":b="JSON Subtitles";break;case "rtsp":b="RTSP";break;case "webrtc":b="WebRTC"}return b},popup:{element:null,show:function(a){this.element=$("
").attr("id","popup").append($("