From e07fed402514b18f3399730744b24dfc82c5c155 Mon Sep 17 00:00:00 2001 From: Zach van Rijn Date: Tue, 11 Apr 2023 04:18:05 +0000 Subject: user/mistserver: bump { 2.18.1 --> 3.1 }. fixes #991. This is upstream mistserver plus a handful of patches to support mbedTLS 3.x. See #991 for more information. Grows a dependency on libsrtp-dev (new package). Concurrency is capped at 2 threads to avoid race conditions until someone figures out why the build scripts are racy. Tests are kneecapped until someone figures out how to fix. --- user/mistserver/APKBUILD | 34 +- user/mistserver/add-dtls-srtp-cmake-option.patch | 15 + user/mistserver/fix-cmake-test-format.patch | 37 + .../gizahnl-mbedtls-dev-from-origin-master.patch | 13180 +++++++++++++++++++ 4 files changed, 13258 insertions(+), 8 deletions(-) create mode 100644 user/mistserver/add-dtls-srtp-cmake-option.patch create mode 100644 user/mistserver/fix-cmake-test-format.patch create mode 100644 user/mistserver/gizahnl-mbedtls-dev-from-origin-master.patch (limited to 'user') diff --git a/user/mistserver/APKBUILD b/user/mistserver/APKBUILD index 91316629a..58dc7ebf3 100644 --- a/user/mistserver/APKBUILD +++ b/user/mistserver/APKBUILD @@ -1,22 +1,33 @@ # Contributor: Síle Ekaterin Liszka # Maintainer: Síle Ekaterin Liszka pkgname=mistserver -pkgver=2.18.1 +pkgver=3.1 pkgrel=0 pkgdesc="Multimedia streaming services" url="https://mistserver.org" arch="all" +options="" license="Zlib AND AGPL-3.0-only" depends="" install="$pkgname.pre-install" -makedepends="cmake libexecinfo-dev mbedtls-dev<3" +makedepends="cmake libexecinfo-dev mbedtls-dev libsrtp-dev" pkgusers="mistserver" pkggroups="mistserver" subpackages="$pkgname-dev $pkgname-openrc" +# ATTENTION MAINTAINERS: Generate a new aggregate patch from: +# (assuming rebased on upstream correctly) +# git clone https://github.com/gizahNL/mistserver.git +# cd mistserver +# git format-patch -M origin/master --stdout > gizahnl-mbedtls-dev-from-origin-master.patch source="mistserver-$pkgver.tar.gz::https://github.com/DDVTECH/mistserver/archive/refs/tags/$pkgver.tar.gz mistserver.confd mistserver.initd - link-execinfo.patch" + link-execinfo.patch + + add-dtls-srtp-cmake-option.patch + gizahnl-mbedtls-dev-from-origin-master.patch + fix-cmake-test-format.patch + " build() { if [ "$CBUILD" != "$CHOST" ]; then @@ -29,14 +40,18 @@ build() { -DCMAKE_BUILD_TYPE=RelWithDebugInfo \ -DCMAKE_CXX_FLAGS="$CXXFLAGS -fPIC" \ -DCMAKE_C_FLAGS="$CFLAGS -fPIC" \ + -DUSE_MBEDTLS_SSL_DTLS_SRTP=True \ ${CMAKE_CROSSOPTS} \ . - # FIXME: this usually succeeds on loop 3. should add a timeout/limit. - until make; do :; done + make -j2 # do not increase this (race conditions) } check() { - CTEST_OUTPUT_ON_FAILURE=TRUE ctest + # FIXME!!! + #CTEST_OUTPUT_ON_FAILURE=TRUE ctest + + # temporary sanity check + MistSession -v | grep Built } package() { @@ -48,7 +63,10 @@ package() { install -Dm644 "$srcdir"/mistserver.confd "$pkgdir"/etc/conf.d/mistserver } -sha512sums="671574687ecabf9b8134f2eef033b66fd13cc4463e2edcf34fe69ff9221b30ddb7ce6a6db3d82708c21fad0910bd0225c483060d36058e0ba778eadd5b8c6e82 mistserver-2.18.1.tar.gz +sha512sums="efcac86cf031c5cc13dd274a4d63292122f1ef3d46faea0457e075898cda01bdea29f011699b595e07c8ed984886a33da2a04395a67698d6b2b405325f1b9715 mistserver-3.1.tar.gz 7288adab6589f2facc1cb794057b1c5d9ec94e12e60d6afc8f6f25c54a8e908cc9841b83b5a6e608fa799fd6aa11767e92a963004182d45f7be9ccd3b65097e7 mistserver.confd e0c7df42f4d486983ece1ea50ab8f3006ebab5386881c14c4b2ff1246b6dd38ace935dc54f8f8a7687edb7ca5975b8c26abd6e99957b8c892862732263d49eb9 mistserver.initd -a27bac965078f7eafb339ae7be9e50519d5728ae4f5d725905d5eecbb3fdf048df3e150cfa881be4bab754ca674a11271343156d5d97758d2ca65bef5bff55a6 link-execinfo.patch" +a27bac965078f7eafb339ae7be9e50519d5728ae4f5d725905d5eecbb3fdf048df3e150cfa881be4bab754ca674a11271343156d5d97758d2ca65bef5bff55a6 link-execinfo.patch +f90737722ac4a2ecff64386a9287ce0ddd48e7b176239f3de26725cadace52667ab44febe536738d8e0dba1fee2047e8f65caa8a2f282c7c6e9dbcc4f8daa23a add-dtls-srtp-cmake-option.patch +48e835a09b8096f78e94f25429768debf65ab9f4b3152c45b69eb072ac5d3abc0036daae894e6b85c57ad7ae7993bf2940c730fbb02dcafecd2ed9716b86dfb9 gizahnl-mbedtls-dev-from-origin-master.patch +de8bc5279426c7fab58074a4e73001c590dd386c0d7cd1747e6fff74278aa901662e7720bf97ea645eb28fc1a6c3e4830afdcaf5e4a770c996c639936dd3e28b fix-cmake-test-format.patch" diff --git a/user/mistserver/add-dtls-srtp-cmake-option.patch b/user/mistserver/add-dtls-srtp-cmake-option.patch new file mode 100644 index 000000000..8704b5a08 --- /dev/null +++ b/user/mistserver/add-dtls-srtp-cmake-option.patch @@ -0,0 +1,15 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 6af16808..a12aa8cf 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -91,6 +91,10 @@ else() + message("SSL/TLS support is turned OFF") + endif() + ++if (USE_MBEDTLS_SSL_DTLS_SRTP) ++ add_definitions(-DHAVE_UPSTREAM_MBEDTLS_SRTP=1) ++endif() ++ + if (DEFINED DATASIZE ) + add_definitions(-DSHM_DATASIZE=${DATASIZE}) + endif() diff --git a/user/mistserver/fix-cmake-test-format.patch b/user/mistserver/fix-cmake-test-format.patch new file mode 100644 index 000000000..07161a2c5 --- /dev/null +++ b/user/mistserver/fix-cmake-test-format.patch @@ -0,0 +1,37 @@ +I don't know if this is correct. + +https://gitlab.kitware.com/cmake/cmake/-/issues/19109 + +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -931,24 +931,24 @@ + ######################################## + add_executable(urltest test/url.cpp ${BINARY_DIR}/mist/.headers) + target_link_libraries(urltest mist) +-add_test(URLTest COMMAND urltest) ++add_test(NAME URLTest COMMAND urltest) + add_executable(logtest test/log.cpp ${BINARY_DIR}/mist/.headers) + target_link_libraries(logtest mist) +-add_test(LOGTest COMMAND logtest) ++add_test(NAME LOGTest COMMAND logtest) + add_executable(downloadertest test/downloader.cpp ${BINARY_DIR}/mist/.headers) + target_link_libraries(downloadertest mist) +-add_test(DownloaderTest COMMAND downloadertest) ++add_test(NAME DownloaderTest COMMAND downloadertest) + add_executable(urireadertest test/urireader.cpp ${BINARY_DIR}/mist/.headers) + target_link_libraries(urireadertest mist) +-add_test(URIReaderTest COMMAND urireadertest) ++add_test(NAME URIReaderTest COMMAND urireadertest) + add_executable(jsontest test/json.cpp ${BINARY_DIR}/mist/.headers) + target_link_libraries(jsontest mist) +-add_test(JSONTest COMMAND jsontest) ++add_test(NAME JSONTest COMMAND jsontest) + add_executable(resolvetest test/resolve.cpp ${BINARY_DIR}/mist/.headers) + target_link_libraries(resolvetest mist) + add_executable(bitwritertest test/bitwriter.cpp ${BINARY_DIR}/mist/.headers) + target_link_libraries(bitwritertest mist) +-add_test(BitWriterTest COMMAND bitwritertest) ++add_test(NAME BitWriterTest COMMAND bitwritertest) + add_executable(streamstatustest test/status.cpp ${BINARY_DIR}/mist/.headers) + target_link_libraries(streamstatustest mist) + add_executable(websockettest test/websocket.cpp ${BINARY_DIR}/mist/.headers) diff --git a/user/mistserver/gizahnl-mbedtls-dev-from-origin-master.patch b/user/mistserver/gizahnl-mbedtls-dev-from-origin-master.patch new file mode 100644 index 000000000..29a6914e2 --- /dev/null +++ b/user/mistserver/gizahnl-mbedtls-dev-from-origin-master.patch @@ -0,0 +1,13180 @@ +From 0430a644dfcd0fcc0ea6494bdb8d431073b8361d Mon Sep 17 00:00:00 2001 +From: Matthew James +Date: Fri, 22 Jul 2022 00:44:28 +0800 +Subject: [PATCH 01/38] Enable Parameters on TSSRT listener + +--- + lib/socket_srt.cpp | 7 +++---- + lib/socket_srt.h | 2 +- + src/input/input_tssrt.cpp | 4 +++- + src/output/output_tssrt.cpp | 3 ++- + 4 files changed, 9 insertions(+), 7 deletions(-) + +diff --git a/lib/socket_srt.cpp b/lib/socket_srt.cpp +index a01117db..3e03bce4 100644 +--- a/lib/socket_srt.cpp ++++ b/lib/socket_srt.cpp +@@ -462,12 +462,11 @@ namespace Socket{ + + SRTServer::SRTServer(int fromSock){conn = SRTConnection(fromSock);} + +- SRTServer::SRTServer(int port, std::string hostname, bool nonblock, const std::string &_direction){ ++ SRTServer::SRTServer(int port, std::string hostname, std::map _params, bool nonblock, const std::string &_direction){ + // We always create a server as listening +- std::map listenMode; +- listenMode["mode"] = "listener"; ++ _params["mode"] = "listener"; + if (hostname == ""){hostname = "0.0.0.0";} +- conn.connect(hostname, port, _direction, listenMode); ++ conn.connect(hostname, port, _direction, _params); + conn.setBlocking(true); + if (!conn){ + ERROR_MSG("Unable to create socket"); +diff --git a/lib/socket_srt.h b/lib/socket_srt.h +index 5101d38d..07e947f8 100644 +--- a/lib/socket_srt.h ++++ b/lib/socket_srt.h +@@ -97,7 +97,7 @@ namespace Socket{ + public: + SRTServer(); + SRTServer(int existingSock); +- SRTServer(int port, std::string hostname, bool nonblock = false, const std::string &_direction = "input"); ++ SRTServer(int port, std::string hostname, std::map params, bool nonblock = false, const std::string &_direction = "input"); + SRTConnection accept(bool nonblock = false, const std::string &direction = "input"); + void setBlocking(bool blocking); + bool connected() const; +diff --git a/src/input/input_tssrt.cpp b/src/input/input_tssrt.cpp +index 4a7adcef..d12f9cba 100644 +--- a/src/input/input_tssrt.cpp ++++ b/src/input/input_tssrt.cpp +@@ -137,7 +137,9 @@ namespace Mist{ + HTTP::URL u(source); + INFO_MSG("Parsed url: %s", u.getUrl().c_str()); + if (Socket::interpretSRTMode(u) == "listener"){ +- sSock = Socket::SRTServer(u.getPort(), u.host, false); ++ std::map arguments; ++ HTTP::parseVars(u.args, arguments); ++ sSock = Socket::SRTServer(u.getPort(), u.host, arguments, false); + struct sigaction new_action; + struct sigaction cur_action; + new_action.sa_sigaction = signal_handler; +diff --git a/src/output/output_tssrt.cpp b/src/output/output_tssrt.cpp +index aa06189e..fb07b11e 100644 +--- a/src/output/output_tssrt.cpp ++++ b/src/output/output_tssrt.cpp +@@ -434,7 +434,8 @@ int main(int argc, char *argv[]){ + sigaction(SIGUSR1, &new_action, NULL); + } + if (conf.getInteger("port") && conf.getString("interface").size()){ +- server_socket = Socket::SRTServer(conf.getInteger("port"), conf.getString("interface"), false, "output"); ++ std::map arguments; ++ server_socket = Socket::SRTServer(conf.getInteger("port"), conf.getString("interface"), arguments, false, "output"); + } + if (!server_socket.connected()){ + DEVEL_MSG("Failure to open socket"); +-- +2.25.1 + + +From 508506c241b2980d1aed607cd269ea60e5265e41 Mon Sep 17 00:00:00 2001 +From: Thulinma +Date: Tue, 5 Jan 2021 15:14:04 +0100 +Subject: [PATCH 02/38] Fixes to UDP socket behaviour + +--- + lib/socket.cpp | 25 ++++++++++++++++++++----- + 1 file changed, 20 insertions(+), 5 deletions(-) + +diff --git a/lib/socket.cpp b/lib/socket.cpp +index 22f5e32b..865c3be7 100644 +--- a/lib/socket.cpp ++++ b/lib/socket.cpp +@@ -1713,15 +1713,19 @@ void Socket::UDPConnection::SetDestination(std::string destIp, uint32_t port){ + memset(&hints, 0, sizeof(struct addrinfo)); + hints.ai_family = family; + hints.ai_socktype = SOCK_DGRAM; +- hints.ai_flags = AI_ADDRCONFIG | AI_V4MAPPED; ++ hints.ai_flags = AI_ADDRCONFIG | AI_ALL; + hints.ai_protocol = IPPROTO_UDP; + hints.ai_canonname = NULL; + hints.ai_addr = NULL; + hints.ai_next = NULL; + int s = getaddrinfo(destIp.c_str(), ss.str().c_str(), &hints, &result); + if (s != 0){ +- FAIL_MSG("Could not connect UDP socket to %s:%i! Error: %s", destIp.c_str(), port, gai_strmagic(s)); +- return; ++ hints.ai_family = AF_UNSPEC; ++ s = getaddrinfo(destIp.c_str(), ss.str().c_str(), &hints, &result); ++ if (s != 0){ ++ FAIL_MSG("Could not connect UDP socket to %s:%i! Error: %s", destIp.c_str(), port, gai_strmagic(s)); ++ return; ++ } + } + + for (rp = result; rp != NULL; rp = rp->ai_next){ +@@ -1745,7 +1749,12 @@ void Socket::UDPConnection::SetDestination(std::string destIp, uint32_t port){ + bind(boundPort, boundAddr, boundMulti); + } + } +- HIGH_MSG("Set UDP destination: %s:%d (%s)", destIp.c_str(), port, addrFam(family)); ++ { ++ std::string trueDest; ++ uint32_t truePort; ++ GetDestination(trueDest, truePort); ++ HIGH_MSG("Set UDP destination: %s:%d => %s:%d (%s)", destIp.c_str(), port, trueDest.c_str(), truePort, addrFam(family)); ++ } + freeaddrinfo(result); + return; + //\todo Possibly detect and handle failure +@@ -1881,6 +1890,12 @@ uint16_t Socket::UDPConnection::bind(int port, std::string iface, const std::str + for (rp = addr_result; rp != NULL; rp = rp->ai_next){ + sock = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol); + if (sock == -1){continue;} ++ if (rp->ai_family == AF_INET6){ ++ const int optval = 0; ++ if (setsockopt(sock, SOL_SOCKET, IPV6_V6ONLY, &optval, sizeof(optval)) < 0){ ++ WARN_MSG("Could not set IPv6 UDP socket to be dual-stack! %s", strerror(errno)); ++ } ++ } + checkRecvBuf(); + char human_addr[INET6_ADDRSTRLEN]; + char human_port[16]; +@@ -1909,7 +1924,7 @@ uint16_t Socket::UDPConnection::bind(int port, std::string iface, const std::str + if (multicast){ + const int optval = 1; + if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval)) < 0){ +- WARN_MSG("Could not set multicast UDP socket re-use!"); ++ WARN_MSG("Could not set multicast UDP socket re-use! %s", strerror(errno)); + } + } + if (::bind(sock, rp->ai_addr, rp->ai_addrlen) == 0){ +-- +2.25.1 + + +From ed9910d587b5504d38e4f7ef2777f2ea7bd9b760 Mon Sep 17 00:00:00 2001 +From: Thulinma +Date: Wed, 3 Aug 2022 14:58:05 +0200 +Subject: [PATCH 03/38] Fix for list limit in HLS/CMAF outputs when using + non-live + +--- + src/output/output_cmaf.cpp | 2 +- + src/output/output_hls.cpp | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/output/output_cmaf.cpp b/src/output/output_cmaf.cpp +index 756fceb8..4582a8b6 100644 +--- a/src/output/output_cmaf.cpp ++++ b/src/output/output_cmaf.cpp +@@ -284,7 +284,7 @@ namespace Mist{ + requestTid, + M.biggestFragment(timingTid) / 1000, + atol(H.GetVar("iMsn").c_str()), +- config->getInteger("listlimit"), ++ M.getLive()? config->getInteger("listlimit") : 0, + urlPrefix, + systemBoot, + bootMsOffset, +diff --git a/src/output/output_hls.cpp b/src/output/output_hls.cpp +index 93410f54..19d083e6 100644 +--- a/src/output/output_hls.cpp ++++ b/src/output/output_hls.cpp +@@ -173,7 +173,7 @@ namespace Mist{ + requestTid, + M.biggestFragment(timingTid) / 1000, + atol(H.GetVar("iMsn").c_str()), +- config->getInteger("listlimit"), ++ M.getLive()? config->getInteger("listlimit") : 0, + urlPrefix, + systemBoot, + bootMsOffset, +-- +2.25.1 + + +From c37aac8898484903da8c2095d29125bf69888c0b Mon Sep 17 00:00:00 2001 +From: Thulinma +Date: Thu, 4 Aug 2022 09:14:51 +0200 +Subject: [PATCH 04/38] Remove "Rate=..." debug message from RTMP push output + +--- + src/output/output_rtmp.cpp | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/src/output/output_rtmp.cpp b/src/output/output_rtmp.cpp +index 69d5f91e..16a400d0 100644 +--- a/src/output/output_rtmp.cpp ++++ b/src/output/output_rtmp.cpp +@@ -618,7 +618,6 @@ namespace Mist{ + + if (type == "audio"){ + uint32_t rate = M.getRate(thisIdx); +- WARN_MSG("Rate=%i", rate); + rtmpheader[7] = 0x08; + if (codec == "AAC"){ + dataheader[0] += 0xA0; +-- +2.25.1 + + +From 4084768a3eb72026e901118e05fd17115dd785b3 Mon Sep 17 00:00:00 2001 +From: Thulinma +Date: Tue, 5 Jul 2022 15:21:04 +0200 +Subject: [PATCH 05/38] Improved incoming push accept timing + +--- + src/output/output.cpp | 7 ++++--- + 1 file changed, 4 insertions(+), 3 deletions(-) + +diff --git a/src/output/output.cpp b/src/output/output.cpp +index 5c785bb0..1fa86eee 100644 +--- a/src/output/output.cpp ++++ b/src/output/output.cpp +@@ -2050,7 +2050,7 @@ namespace Mist{ + INFO_MSG("Waiting for stream reset before attempting push input accept (%" PRIu64 " <= %" PRIu64 "+500)", twoTime, oneTime); + while (streamStatus != STRMSTAT_OFF && keepGoing()){ + userSelect.clear(); +- Util::wait(1000); ++ Util::wait(250); + streamStatus = Util::getStreamStatus(streamName); + } + reconnect(); +@@ -2059,14 +2059,15 @@ namespace Mist{ + while (((streamStatus != STRMSTAT_WAIT && streamStatus != STRMSTAT_READY) || !meta) && keepGoing()){ + INFO_MSG("Waiting for %s buffer to be ready... (%u)", streamName.c_str(), streamStatus); + disconnect(); +- Util::wait(1000); + streamStatus = Util::getStreamStatus(streamName); + if (streamStatus == STRMSTAT_OFF || streamStatus == STRMSTAT_WAIT || streamStatus == STRMSTAT_READY){ + INFO_MSG("Reconnecting to %s buffer... (%u)", streamName.c_str(), streamStatus); +- Util::wait(500); + reconnect(); + streamStatus = Util::getStreamStatus(streamName); + } ++ if (((streamStatus != STRMSTAT_WAIT && streamStatus != STRMSTAT_READY) || !meta) && keepGoing()){ ++ Util::wait(100); ++ } + } + if (streamStatus == STRMSTAT_READY || streamStatus == STRMSTAT_WAIT){reconnect();} + if (!meta){ +-- +2.25.1 + + +From 14bc94ece51affb7cb6d52a491b97921ed420587 Mon Sep 17 00:00:00 2001 +From: Thulinma +Date: Tue, 12 Jul 2022 15:47:29 +0200 +Subject: [PATCH 06/38] Prevent eternal sleep when waiting for RelAccX + structure to become ready + +--- + lib/util.cpp | 10 +++++++++- + 1 file changed, 9 insertions(+), 1 deletion(-) + +diff --git a/lib/util.cpp b/lib/util.cpp +index d522a151..8dd75727 100644 +--- a/lib/util.cpp ++++ b/lib/util.cpp +@@ -446,7 +446,15 @@ namespace Util{ + hdrOffset = (uint16_t*)(p+26); + hdrEndPos = (uint64_t*)(p+28); + if (waitReady){ +- while (!isReady()){Util::sleep(50);} ++ uint64_t maxWait = Util::bootMS() + 10000; ++ while (!isReady()){ ++ if (Util::bootMS() > maxWait){ ++ FAIL_MSG("Waiting for RelAccX structure to be ready timed out, aborting"); ++ p = 0; ++ return; ++ } ++ Util::sleep(50); ++ } + } + if (isReady()){ + uint16_t offset = RAXHDR_FIELDOFFSET; +-- +2.25.1 + + +From 5bdd4a416e964ae0cfcc56d99c062691d1b3931d Mon Sep 17 00:00:00 2001 +From: Thulinma +Date: Thu, 14 Jul 2022 10:45:53 +0200 +Subject: [PATCH 07/38] Fix load balancer CPU usage + +--- + src/input/input_balancer.cpp | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/src/input/input_balancer.cpp b/src/input/input_balancer.cpp +index 448e6eff..36db5709 100644 +--- a/src/input/input_balancer.cpp ++++ b/src/input/input_balancer.cpp +@@ -140,6 +140,9 @@ namespace Mist{ + startTime = 0; // note success + break; // break out of while loop + } ++ }else{ ++ //Prevent 100% CPU usage if the response is slow ++ Util::sleep(25); + } + } + if (startTime){ +-- +2.25.1 + + +From fffe98804cb9c126ec91b7a7b67f2c917561ce02 Mon Sep 17 00:00:00 2001 +From: Thulinma +Date: Thu, 30 Jun 2022 14:04:34 +0200 +Subject: [PATCH 08/38] Fixed TS SRT input not closing the connection when + stopping for internal reasons rather than external reasons + +--- + src/input/input_tssrt.cpp | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/src/input/input_tssrt.cpp b/src/input/input_tssrt.cpp +index d12f9cba..6cd614e7 100644 +--- a/src/input/input_tssrt.cpp ++++ b/src/input/input_tssrt.cpp +@@ -241,6 +241,7 @@ namespace Mist{ + } + // If we are here: we have a proper connection (either accepted or pull input) and should start parsing it as such + Input::streamMainLoop(); ++ srtConn.close(); + } + + bool inputTSSRT::needsLock(){return false;} +-- +2.25.1 + + +From 267a74f0f6d44b928b99312b356a3080579fdc1c Mon Sep 17 00:00:00 2001 +From: Thulinma +Date: Thu, 4 Aug 2022 16:43:04 +0200 +Subject: [PATCH 09/38] Fix track selector logic when multiple selections are + considered + +--- + lib/stream.cpp | 33 +++++++++++++++++++++++++++++++-- + 1 file changed, 31 insertions(+), 2 deletions(-) + +diff --git a/lib/stream.cpp b/lib/stream.cpp +index 6036a8fd..715015d6 100644 +--- a/lib/stream.cpp ++++ b/lib/stream.cpp +@@ -1249,7 +1249,7 @@ std::set Util::wouldSelect(const DTSC::Meta &M, const std::map result; + + /*LTS-START*/ +- bool noSelAudio = false, noSelVideo = false, noSelSub = false; ++ bool noSelAudio = false, noSelVideo = false, noSelSub = false, noSelMeta = false; + // Then, select the tracks we've been asked to select. + if (targetParams.count("audio") && targetParams.at("audio").size()){ + if (targetParams.at("audio") != "-1" && targetParams.at("audio") != "none"){ +@@ -1270,6 +1270,11 @@ std::set Util::wouldSelect(const DTSC::Meta &M, const std::map tracks = Util::findTracks(M, capa, "meta", targetParams.at("meta"), UA); ++ result.insert(tracks.begin(), tracks.end()); ++ noSelMeta = true; ++ } + /*LTS-END*/ + + std::set validTracks = M.getValidTracks(); +@@ -1291,6 +1296,7 @@ std::set Util::wouldSelect(const DTSC::Meta &M, const std::map Util::wouldSelect(const DTSC::Meta &M, const std::map Util::wouldSelect(const DTSC::Meta &M, const std::map 0){ + jsonForEachConst((*it), itb){ + if ((*itb).size() > 0){ +@@ -1360,6 +1368,26 @@ std::set Util::wouldSelect(const DTSC::Meta &M, const std::map::iterator itd = validTracks.begin(); itd != validTracks.end(); itd++){ ++ if ((!byType && M.getCodec(*itd) == strRef.substr(shift)) || ++ (byType && M.getType(*itd) == strRef.substr(shift)) || strRef.substr(shift) == "*"){ ++ // user-agent-check ++ bool problems = false; ++ if (capa.isMember("exceptions") && capa["exceptions"].isObject() && ++ capa["exceptions"].size()){ ++ jsonForEachConst(capa["exceptions"], ex){ ++ if (ex.key() == "codec:" + strRef.substr(shift)){ ++ problems = !Util::checkException(*ex, UA); ++ break; ++ } ++ } ++ } ++ if (problems){break;} ++ extraCounter++; + if (!multiSel){break;} + } + } +@@ -1367,8 +1395,9 @@ std::set Util::wouldSelect(const DTSC::Meta &M, const std::map bestSoFarCount){ ++ if (selCounter > bestSoFarCount || (selCounter == bestSoFarCount && extraCounter > bestSoFarCountExtra)){ + bestSoFarCount = selCounter; ++ bestSoFarCountExtra = extraCounter; + bestSoFar = index; + HIGH_MSG("Matched %u: %s", selCounter, (*it).toString().c_str()); + } +-- +2.25.1 + + +From 3734c90544a68d03a1a69438f00f215c517fec01 Mon Sep 17 00:00:00 2001 +From: Phencys +Date: Sun, 20 Sep 2020 20:31:17 +0200 +Subject: [PATCH 10/38] Added support for raw passthrough of MPEG2-TS data + +--- + src/input/input_ts.cpp | 73 ++++++++++++++++++++++++++++++++--- + src/input/input_ts.h | 5 +++ + src/input/input_tsrist.cpp | 54 +++++++++++++++++++++++++- + src/input/input_tsrist.h | 6 +++ + src/input/input_tssrt.cpp | 35 +++++++++++++++++ + src/input/input_tssrt.h | 4 ++ + src/io.cpp | 7 ++++ + src/io.h | 1 + + src/output/output_httpts.cpp | 1 + + src/output/output_ts.cpp | 1 + + src/output/output_ts_base.cpp | 5 +++ + src/output/output_tsrist.cpp | 1 + + src/output/output_tssrt.cpp | 1 + + 13 files changed, 187 insertions(+), 7 deletions(-) + +diff --git a/src/input/input_ts.cpp b/src/input/input_ts.cpp +index 98300411..cfe3f7cd 100644 +--- a/src/input/input_ts.cpp ++++ b/src/input/input_ts.cpp +@@ -134,6 +134,13 @@ void parseThread(void *mistIn){ + } + } + } ++ ++ //On shutdown, make sure to clean up stream buffer ++ if (idx != INVALID_TRACK_ID){ ++ tthread::lock_guard guard(threadClaimMutex); ++ input->liveFinalize(idx); ++ } ++ + std::string reason = "unknown reason"; + if (!(Util::bootSecs() - threadTimer[tid] < THREAD_TIMEOUT)){reason = "thread timeout";} + if (!cfgPointer->is_active){reason = "input shutting down";} +@@ -155,6 +162,9 @@ namespace Mist{ + /// Constructor of TS Input + /// \arg cfg Util::Config that contains all current configurations. + inputTS::inputTS(Util::Config *cfg) : Input(cfg){ ++ rawMode = false; ++ rawIdx = INVALID_TRACK_ID; ++ lastRawPacket = 0; + capa["name"] = "TS"; + capa["desc"] = + "This input allows you to stream MPEG2-TS data from static files (/*.ts), streamed files " +@@ -188,6 +198,7 @@ namespace Mist{ + capa["codecs"][0u][1u].append("AC3"); + capa["codecs"][0u][1u].append("MP2"); + capa["codecs"][0u][1u].append("opus"); ++ capa["codecs"][1u][0u].append("rawts"); + inFile = NULL; + inputProcess = 0; + isFinished = false; +@@ -232,6 +243,16 @@ namespace Mist{ + "Alternative stream to load for playback when there is no active broadcast"; + capa["optional"]["fallback_stream"]["type"] = "str"; + capa["optional"]["fallback_stream"]["default"] = ""; ++ ++ capa["optional"]["raw"]["name"] = "Raw input mode"; ++ capa["optional"]["raw"]["help"] = "Enable raw MPEG-TS passthrough mode"; ++ capa["optional"]["raw"]["option"] = "--raw"; ++ ++ JSON::Value option; ++ option["long"] = "raw"; ++ option["short"] = "R"; ++ option["help"] = "Enable raw MPEG-TS passthrough mode"; ++ config->addOption("raw", option); + } + + inputTS::~inputTS(){ +@@ -257,6 +278,10 @@ namespace Mist{ + /// Live Setup of TS Input + bool inputTS::preRun(){ + INFO_MSG("Prerun: %s", config->getString("input").c_str()); ++ ++ rawMode = config->getBool("raw"); ++ if (rawMode){INFO_MSG("Entering raw mode");} ++ + // streamed standard input + if (config->getString("input") == "-"){ + standAlone = false; +@@ -520,9 +545,28 @@ namespace Mist{ + } + if (tcpCon.Received().available(188) && tcpCon.Received().get()[0] == 0x47){ + std::string newData = tcpCon.Received().remove(188); +- tsBuf.FromPointer(newData.data()); +- liveStream.add(tsBuf); +- if (!liveStream.isDataTrack(tsBuf.getPID())){liveStream.parse(tsBuf.getPID());} ++ if (rawMode){ ++ keepAlive(); ++ rawBuffer.append(newData); ++ if (rawBuffer.size() >= 1316 && (lastRawPacket == 0 || lastRawPacket != Util::bootMS())){ ++ if (rawIdx == INVALID_TRACK_ID){ ++ rawIdx = meta.addTrack(); ++ meta.setType(rawIdx, "meta"); ++ meta.setCodec(rawIdx, "rawts"); ++ meta.setID(rawIdx, 1); ++ userSelect[rawIdx].reload(streamName, rawIdx, COMM_STATUS_SOURCE); ++ } ++ uint64_t packetTime = Util::bootMS(); ++ thisPacket.genericFill(packetTime, 0, 1, rawBuffer, rawBuffer.size(), 0, 0); ++ bufferLivePacket(thisPacket); ++ lastRawPacket = packetTime; ++ rawBuffer.truncate(0); ++ } ++ }else { ++ tsBuf.FromPointer(newData.data()); ++ liveStream.add(tsBuf); ++ if (!liveStream.isDataTrack(tsBuf.getPID())){liveStream.parse(tsBuf.getPID());} ++ } + } + } + noDataSince = Util::bootSecs(); +@@ -543,7 +587,26 @@ namespace Mist{ + gettingData = true; + INFO_MSG("Now receiving UDP data..."); + } +- assembler.assemble(liveStream, udpCon.data, udpCon.data.size()); ++ if (rawMode){ ++ keepAlive(); ++ rawBuffer.append(udpCon.data, udpCon.data.size()); ++ if (rawBuffer.size() >= 1316 && (lastRawPacket == 0 || lastRawPacket != Util::bootMS())){ ++ if (rawIdx == INVALID_TRACK_ID){ ++ rawIdx = meta.addTrack(); ++ meta.setType(rawIdx, "meta"); ++ meta.setCodec(rawIdx, "rawts"); ++ meta.setID(rawIdx, 1); ++ userSelect[rawIdx].reload(streamName, rawIdx, COMM_STATUS_SOURCE); ++ } ++ uint64_t packetTime = Util::bootMS(); ++ thisPacket.genericFill(packetTime, 0, 1, rawBuffer, rawBuffer.size(), 0, 0); ++ bufferLivePacket(thisPacket); ++ lastRawPacket = packetTime; ++ rawBuffer.truncate(0); ++ } ++ }else{ ++ assembler.assemble(liveStream, udpCon.data, udpCon.data.size()); ++ } + } + if (!received){ + Util::sleep(100); +@@ -578,7 +641,7 @@ namespace Mist{ + } + + std::set activeTracks = liveStream.getActiveTracks(); +- { ++ if (!rawMode){ + tthread::lock_guard guard(threadClaimMutex); + if (hasStarted && !threadTimer.size()){ + if (!isAlwaysOn()){ +diff --git a/src/input/input_ts.h b/src/input/input_ts.h +index e810ddb4..3116725d 100644 +--- a/src/input/input_ts.h ++++ b/src/input/input_ts.h +@@ -41,6 +41,11 @@ namespace Mist{ + pid_t inputProcess; + size_t tmpIdx; + bool isFinished; ++ ++ bool rawMode; ++ Util::ResizeablePointer rawBuffer; ++ size_t rawIdx; ++ uint64_t lastRawPacket; + }; + }// namespace Mist + +diff --git a/src/input/input_tsrist.cpp b/src/input/input_tsrist.cpp +index 18b05c9d..5b95aa9f 100644 +--- a/src/input/input_tsrist.cpp ++++ b/src/input/input_tsrist.cpp +@@ -66,6 +66,10 @@ namespace Mist{ + /// Constructor of TS Input + /// \arg cfg Util::Config that contains all current configurations. + inputTSRIST::inputTSRIST(Util::Config *cfg) : Input(cfg){ ++ rawMode = false; ++ rawIdx = INVALID_TRACK_ID; ++ lastRawPacket = 0; ++ hasRaw = false; + connPtr = this; + cnfPtr = config; + +@@ -96,6 +100,7 @@ namespace Mist{ + capa["codecs"][0u][1u].append("AC3"); + capa["codecs"][0u][1u].append("MP2"); + capa["codecs"][0u][1u].append("opus"); ++ capa["codecs"][1u][0u].append("rawts"); + + JSON::Value option; + option["arg"] = "integer"; +@@ -132,6 +137,15 @@ namespace Mist{ + capa["optional"]["profile"]["type"] = "select"; + capa["optional"]["profile"]["option"] = "--profile"; + ++ capa["optional"]["raw"]["name"] = "Raw input mode"; ++ capa["optional"]["raw"]["help"] = "Enable raw MPEG-TS passthrough mode"; ++ capa["optional"]["raw"]["option"] = "--raw"; ++ ++ option["long"] = "raw"; ++ option["short"] = "R"; ++ option["help"] = "Enable raw MPEG-TS passthrough mode"; ++ config->addOption("raw", option); ++ + lastTimeStamp = 0; + timeStampOffset = 0; + receiver_ctx = 0; +@@ -146,6 +160,9 @@ namespace Mist{ + + /// Live Setup of SRT Input. Runs only if we are the "main" thread + bool inputTSRIST::preRun(){ ++ rawMode = config->getBool("raw"); ++ if (rawMode){INFO_MSG("Entering raw mode");} ++ + std::string source = config->getString("input"); + standAlone = false; + HTTP::URL u(source); +@@ -161,6 +178,20 @@ namespace Mist{ + // Retrieve the next packet to be played from the srt connection. + void inputTSRIST::getNext(size_t idx){ + thisPacket.null(); ++ if (rawMode){ ++ //Set to false so the other thread knows its safe to fill ++ hasRaw = false; ++ while (!hasRaw && config->is_active){ ++ Util::sleep(50); ++ if (!bufferActive()){ ++ Util::logExitReason("Buffer shut down"); ++ return; ++ } ++ } ++ //if hasRaw, thisPacket has been filled by the other thread ++ return; ++ } ++ + while (!thisPacket && config->is_active){ + if (tsStream.hasPacket()){ + tsStream.getEarliestPacket(thisPacket); +@@ -228,8 +259,27 @@ namespace Mist{ + } + + void inputTSRIST::addData(const char * ptr, size_t len){ +- for (size_t o = 0; o <= len-188; o += 188){ +- tsStream.parse((char*)ptr+o, 0); ++ for (size_t o = 0; o+188 <= len; o += 188){ ++ if (rawMode){ ++ rawBuffer.append(ptr+o, 188); ++ if (!hasRaw && rawBuffer.size() >= 1316 && (lastRawPacket == 0 || lastRawPacket != Util::bootMS())){ ++ if (rawIdx == INVALID_TRACK_ID){ ++ rawIdx = meta.addTrack(); ++ meta.setType(rawIdx, "meta"); ++ meta.setCodec(rawIdx, "rawts"); ++ meta.setID(rawIdx, 1); ++ userSelect[rawIdx].reload(streamName, rawIdx, COMM_STATUS_SOURCE); ++ } ++ thisTime = Util::bootMS(); ++ thisIdx = rawIdx; ++ thisPacket.genericFill(thisTime, 0, 1, rawBuffer, rawBuffer.size(), 0, 0); ++ lastRawPacket = thisTime; ++ rawBuffer.truncate(0); ++ hasRaw = true; ++ } ++ }else{ ++ tsStream.parse((char*)ptr+o, 0); ++ } + } + } + +diff --git a/src/input/input_tsrist.h b/src/input/input_tsrist.h +index 3bc29e1c..731f9b04 100644 +--- a/src/input/input_tsrist.h ++++ b/src/input/input_tsrist.h +@@ -33,6 +33,12 @@ namespace Mist{ + virtual void connStats(Comms::Statistics &statComm); + + struct rist_ctx *receiver_ctx; ++ ++ bool rawMode; ++ Util::ResizeablePointer rawBuffer; ++ size_t rawIdx; ++ uint64_t lastRawPacket; ++ bool hasRaw; + }; + }// namespace Mist + +diff --git a/src/input/input_tssrt.cpp b/src/input/input_tssrt.cpp +index 6cd614e7..da69be29 100644 +--- a/src/input/input_tssrt.cpp ++++ b/src/input/input_tssrt.cpp +@@ -25,6 +25,7 @@ + Util::Config *cfgPointer = NULL; + std::string baseStreamName; + Socket::SRTServer sSock; ++bool rawMode = false; + + void (*oldSignal)(int, siginfo_t *,void *) = 0; + +@@ -49,6 +50,8 @@ namespace Mist{ + /// Constructor of TS Input + /// \arg cfg Util::Config that contains all current configurations. + inputTSSRT::inputTSSRT(Util::Config *cfg, SRTSOCKET s) : Input(cfg){ ++ rawIdx = INVALID_TRACK_ID; ++ lastRawPacket = 0; + capa["name"] = "TSSRT"; + capa["desc"] = "This input allows for processing MPEG2-TS-based SRT streams. Use mode=listener " + "for push input."; +@@ -66,6 +69,7 @@ namespace Mist{ + capa["codecs"][0u][1u].append("AC3"); + capa["codecs"][0u][1u].append("MP2"); + capa["codecs"][0u][1u].append("opus"); ++ capa["codecs"][1u][0u].append("rawts"); + + JSON::Value option; + option["arg"] = "integer"; +@@ -103,7 +107,16 @@ namespace Mist{ + capa["optional"]["acceptable"]["select"][2u][0u] = 2; + capa["optional"]["acceptable"]["select"][2u][1u] = "Disallow non-matching streamid"; + ++ capa["optional"]["raw"]["name"] = "Raw input mode"; ++ capa["optional"]["raw"]["help"] = "Enable raw MPEG-TS passthrough mode"; ++ capa["optional"]["raw"]["option"] = "--raw"; + ++ option.null(); ++ option["long"] = "raw"; ++ option["short"] = "R"; ++ option["help"] = "Enable raw MPEG-TS passthrough mode"; ++ config->addOption("raw", option); ++ + // Setup if we are called form with a thread for push-based input. + if (s != -1){ + srtConn = Socket::SRTConnection(s); +@@ -131,6 +144,8 @@ namespace Mist{ + + /// Live Setup of SRT Input. Runs only if we are the "main" thread + bool inputTSSRT::preRun(){ ++ rawMode = config->getBool("raw"); ++ if (rawMode){INFO_MSG("Entering raw mode");} + if (srtConn.getSocket() == -1){ + std::string source = config->getString("input"); + standAlone = false; +@@ -183,6 +198,26 @@ namespace Mist{ + + size_t recvSize = srtConn.RecvNow(); + if (recvSize){ ++ if (rawMode){ ++ keepAlive(); ++ rawBuffer.append(srtConn.recvbuf, recvSize); ++ if (rawBuffer.size() >= 1316 && (lastRawPacket == 0 || lastRawPacket != Util::bootMS())){ ++ if (rawIdx == INVALID_TRACK_ID){ ++ rawIdx = meta.addTrack(); ++ meta.setType(rawIdx, "meta"); ++ meta.setCodec(rawIdx, "rawts"); ++ meta.setID(rawIdx, 1); ++ userSelect[rawIdx].reload(streamName, rawIdx, COMM_STATUS_SOURCE); ++ } ++ uint64_t packetTime = Util::bootMS(); ++ thisPacket.genericFill(packetTime, 0, 1, rawBuffer, rawBuffer.size(), 0, 0); ++ bufferLivePacket(thisPacket); ++ lastRawPacket = packetTime; ++ rawBuffer.truncate(0); ++ return; ++ } ++ continue; ++ } + if (assembler.assemble(tsStream, srtConn.recvbuf, recvSize, true)){hasPacket = tsStream.hasPacket();} + }else if (srtConn){ + // This should not happen as the SRT socket is read blocking and won't return until there is +diff --git a/src/input/input_tssrt.h b/src/input/input_tssrt.h +index 40fa05c1..4f337b48 100644 +--- a/src/input/input_tssrt.h ++++ b/src/input/input_tssrt.h +@@ -41,6 +41,10 @@ namespace Mist{ + Socket::SRTConnection srtConn; + bool singularFlag; + virtual void connStats(Comms::Statistics &statComm); ++ ++ Util::ResizeablePointer rawBuffer; ++ size_t rawIdx; ++ uint64_t lastRawPacket; + }; + }// namespace Mist + +diff --git a/src/io.cpp b/src/io.cpp +index 6a5844c4..fea32193 100644 +--- a/src/io.cpp ++++ b/src/io.cpp +@@ -292,6 +292,13 @@ namespace Mist{ + tPages.setInt("avail", pageOffset + packDataLen, pageIdx); + } + ++ /// Wraps up the buffering of a shared memory data page ++ /// \param idx The track index of the page to finalize ++ void InOutBase::liveFinalize(size_t idx){ ++ if (!livePage.count(idx)){return;} ++ bufferFinalize(idx, livePage[idx]); ++ } ++ + /// Wraps up the buffering of a shared memory data page + /// \param idx The track index of the page to finalize + void InOutBase::bufferFinalize(size_t idx, IPC::sharedPage & page){ +diff --git a/src/io.h b/src/io.h +index d839952a..e6545929 100644 +--- a/src/io.h ++++ b/src/io.h +@@ -21,6 +21,7 @@ namespace Mist{ + + bool bufferStart(size_t idx, uint32_t pageNumber, IPC::sharedPage & page, DTSC::Meta & aMeta); + void bufferFinalize(size_t idx, IPC::sharedPage & page); ++ void liveFinalize(size_t idx); + bool isCurrentLivePage(size_t idx, uint32_t pageNumber); + void bufferRemove(size_t idx, uint32_t pageNumber); + void bufferLivePacket(const DTSC::Packet &packet); +diff --git a/src/output/output_httpts.cpp b/src/output/output_httpts.cpp +index 6aab2b9b..9daa9dc3 100644 +--- a/src/output/output_httpts.cpp ++++ b/src/output/output_httpts.cpp +@@ -135,6 +135,7 @@ namespace Mist{ + capa["codecs"][0u][1u].append("+AC3"); + capa["codecs"][0u][1u].append("+MP2"); + capa["codecs"][0u][1u].append("+opus"); ++ capa["codecs"][1u][0u].append("rawts"); + capa["methods"][0u]["handler"] = "http"; + capa["methods"][0u]["type"] = "html5/video/mpeg"; + capa["methods"][0u]["hrn"] = "TS HTTP progressive"; +diff --git a/src/output/output_ts.cpp b/src/output/output_ts.cpp +index d2addc24..561bf9aa 100644 +--- a/src/output/output_ts.cpp ++++ b/src/output/output_ts.cpp +@@ -179,6 +179,7 @@ namespace Mist{ + capa["codecs"][0u][1u].append("+AC3"); + capa["codecs"][0u][1u].append("+MP2"); + capa["codecs"][0u][1u].append("+opus"); ++ capa["codecs"][1u][0u].append("rawts"); + cfg->addConnectorOptions(8888, capa); + config = cfg; + capa["push_urls"].append("tsudp://*"); +diff --git a/src/output/output_ts_base.cpp b/src/output/output_ts_base.cpp +index 17b2a8ce..c4163e7f 100644 +--- a/src/output/output_ts_base.cpp ++++ b/src/output/output_ts_base.cpp +@@ -75,6 +75,11 @@ namespace Mist{ + size_t dataLen = 0; + thisPacket.getString("data", dataPointer, dataLen); // data + ++ if (codec == "rawts"){ ++ for (size_t i = 0; i+188 <= dataLen; i+=188){sendTS(dataPointer+i, 188);} ++ return; ++ } ++ + packTime *= 90; + std::string bs; + // prepare bufferstring +diff --git a/src/output/output_tsrist.cpp b/src/output/output_tsrist.cpp +index ee57d8ee..c4e03f80 100644 +--- a/src/output/output_tsrist.cpp ++++ b/src/output/output_tsrist.cpp +@@ -202,6 +202,7 @@ namespace Mist{ + capa["codecs"][0u][1u].append("+AC3"); + capa["codecs"][0u][1u].append("+MP2"); + capa["codecs"][0u][1u].append("+opus"); ++ capa["codecs"][1u][0u].append("rawts"); + + capa["optional"]["profile"]["name"] = "RIST profile"; + capa["optional"]["profile"]["help"] = "RIST profile to use"; +diff --git a/src/output/output_tssrt.cpp b/src/output/output_tssrt.cpp +index fb07b11e..db07dc91 100644 +--- a/src/output/output_tssrt.cpp ++++ b/src/output/output_tssrt.cpp +@@ -200,6 +200,7 @@ namespace Mist{ + capa["codecs"][0u][1u].append("AC3"); + capa["codecs"][0u][1u].append("MP2"); + capa["codecs"][0u][1u].append("opus"); ++ capa["codecs"][1u][0u].append("rawts"); + cfg->addConnectorOptions(8889, capa); + config = cfg; + capa["push_urls"].append("srt://*"); +-- +2.25.1 + + +From 6c117b63cfe85a204b48dfddddb355f416b39364 Mon Sep 17 00:00:00 2001 +From: Alex Kordic +Date: Thu, 4 Aug 2022 14:49:57 +0200 +Subject: [PATCH 11/38] Add s3 protocol to `URIReader` + +--- + lib/timing.cpp | 12 +++++ + lib/timing.h | 1 + + lib/urireader.cpp | 114 +++++++++++++++++++++++++++++++++++++++++++++- + 3 files changed, 125 insertions(+), 2 deletions(-) + +diff --git a/lib/timing.cpp b/lib/timing.cpp +index 036f0b29..f339f4fe 100644 +--- a/lib/timing.cpp ++++ b/lib/timing.cpp +@@ -114,3 +114,15 @@ std::string Util::getUTCString(uint64_t epoch){ + ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec); + return std::string(result); + } ++ ++std::string Util::getDateString(uint64_t epoch){ ++ char buffer[80]; ++ time_t rawtime = epoch; ++ if (!epoch) { ++ time(&rawtime); ++ } ++ struct tm * timeinfo; ++ timeinfo = localtime(&rawtime); ++ strftime(buffer, sizeof(buffer), "%a, %d %b %Y %H:%M:%S %z", timeinfo); ++ return std::string(buffer); ++} +diff --git a/lib/timing.h b/lib/timing.h +index 107ccee5..b2a4d612 100644 +--- a/lib/timing.h ++++ b/lib/timing.h +@@ -17,4 +17,5 @@ namespace Util{ + uint64_t getNTP(); + uint64_t epoch(); ///< Gets the amount of seconds since 01/01/1970. + std::string getUTCString(uint64_t epoch = 0); ++ std::string getDateString(uint64_t epoch = 0); + }// namespace Util +diff --git a/lib/urireader.cpp b/lib/urireader.cpp +index 80f204c8..ad85b052 100644 +--- a/lib/urireader.cpp ++++ b/lib/urireader.cpp +@@ -3,11 +3,86 @@ + #include "timing.h" + #include "urireader.h" + #include "util.h" ++#include "encode.h" + #include + #include ++#include ++#include + + namespace HTTP{ + ++ // When another protocol needs this, rename struct to HeaderOverride or similar ++ struct HTTPHeadThenGet { ++ bool continueOperation; ++ std::string date, headAuthorization, getAuthorization; ++ ++ HTTPHeadThenGet() : continueOperation(false) {} ++ ++ void prepareHeadHeaders(HTTP::Downloader& downloader) { ++ if(!continueOperation) return; ++ downloader.setHeader("Date", date); ++ downloader.setHeader("Authorization", headAuthorization); ++ } ++ ++ void prepareGetHeaders(HTTP::Downloader& downloader) { ++ if(!continueOperation) return; ++ // .setHeader() overwrites existing header value ++ downloader.setHeader("Date", date); ++ downloader.setHeader("Authorization", getAuthorization); ++ } ++ }; ++ ++#ifndef NOSSL ++ inline bool s3CalculateSignature(std::string& signature, const std::string method, const std::string date, const std::string& requestPath, const std::string& accessKey, const std::string& secret) { ++ std::string toSign = method + "\n\n\n" + date + "\n" + requestPath; ++ unsigned char signatureBytes[MBEDTLS_MD_MAX_SIZE]; ++ const int sha1Size = 20; ++ mbedtls_md_context_t md_ctx = {0}; ++ // TODO: When we use MBEDTLS_MD_SHA512 ? Consult documentation/code ++ const mbedtls_md_info_t *md_info = mbedtls_md_info_from_type(MBEDTLS_MD_SHA1); ++ if (!md_info){ FAIL_MSG("error s3 MBEDTLS_MD_SHA1 unavailable"); return false; } ++ int status = mbedtls_md_setup(&md_ctx, md_info, 1); ++ if(status != 0) { FAIL_MSG("error s3 mbedtls_md_setup error %d", status); return false; } ++ status = mbedtls_md_hmac_starts(&md_ctx, (const unsigned char *)secret.c_str(), secret.size()); ++ if(status != 0) { FAIL_MSG("error s3 mbedtls_md_hmac_starts error %d", status); return false; } ++ status = mbedtls_md_hmac_update(&md_ctx, (const unsigned char *)toSign.c_str(), toSign.size()); ++ if(status != 0) { FAIL_MSG("error s3 mbedtls_md_hmac_update error %d", status); return false; } ++ status = mbedtls_md_hmac_finish(&md_ctx, signatureBytes); ++ if(status != 0) { FAIL_MSG("error s3 mbedtls_md_hmac_finish error %d", status); return false; } ++ std::string base64encoded = Encodings::Base64::encode(std::string((const char*)signatureBytes, sha1Size)); ++ signature = "AWS " + accessKey + ":" + base64encoded; ++ return true; ++ } ++ // Input url == s3+https://s3_key:secret@storage.googleapis.com/alexk-dms-upload-test/testvideo.ts ++ // Transform to: ++ // url=https://storage.googleapis.com/alexk-dms-upload-test/testvideo.ts ++ // header Date: ${Util::getDateString(()} ++ // header Authorization: AWS ${url.user}:${signature} ++ inline HTTPHeadThenGet s3TransformToHttp(HTTP::URL& url) { ++ HTTPHeadThenGet result; ++ result.date = Util::getDateString(); ++ // remove "s3+" prefix ++ url.protocol = url.protocol.erase(0, 3); ++ // Use user and pass to create signature and remove from HTTP request ++ std::string accessKey(url.user), secret(url.pass); ++ url.user = ""; ++ url.pass = ""; ++ std::string requestPath = "/" + Encodings::URL::encode(url.path, "/:=@[]#?&"); ++ if(url.args.size()) requestPath += "?" + url.args; ++ // Calculate Authorization data ++ if(!s3CalculateSignature(result.headAuthorization, "HEAD", result.date, requestPath, accessKey, secret)) { ++ result.continueOperation = false; ++ return result; ++ } ++ if(!s3CalculateSignature(result.getAuthorization, "GET", result.date, requestPath, accessKey, secret)) { ++ result.continueOperation = false; ++ return result; ++ } ++ result.continueOperation = true; ++ return result; ++ } ++#endif // ifndef NOSSL ++ + void URIReader::init(){ + handle = -1; + mapped = 0; +@@ -97,12 +172,45 @@ namespace HTTP{ + } + } + ++ // prepare for s3 and http ++ HTTPHeadThenGet httpHeaderOverride; ++ ++#ifndef NOSSL ++ // In case of s3 URI we prepare HTTP request with AWS authorization and rely on HTTP logic below ++ if (myURI.protocol == "s3+https" || myURI.protocol == "s3+http"){ ++ // Check fallback to global credentials in env vars ++ bool haveCredentials = myURI.user.size() && myURI.pass.size(); ++ if(!haveCredentials) { ++ // Use environment variables ++ char * envValue = std::getenv("S3_ACCESS_KEY_ID"); ++ if(envValue == NULL) { ++ FAIL_MSG("error s3 uri without credentials. Consider setting S3_ACCESS_KEY_ID env var"); ++ return false; ++ } ++ myURI.user = envValue; ++ envValue = std::getenv("S3_SECRET_ACCESS_KEY"); ++ if(envValue == NULL) { ++ FAIL_MSG("error s3 uri without credentials. Consider setting S3_SECRET_ACCESS_KEY env var"); ++ return false; ++ } ++ myURI.pass = envValue; ++ } ++ // Transform s3 url to HTTP request: ++ httpHeaderOverride = s3TransformToHttp(myURI); ++ bool errorInSignatureCalculation = !httpHeaderOverride.continueOperation; ++ if(errorInSignatureCalculation) return false; ++ // Do not return, continue to HTTP case ++ } ++#endif // ifndef NOSSL ++ + // HTTP, stream or regular download? + if (myURI.protocol == "http" || myURI.protocol == "https"){ +- stateType = HTTP::HTTP; ++ stateType = HTTP; ++ downer.clearHeaders(); + ++ // One set of headers specified for HEAD request ++ httpHeaderOverride.prepareHeadHeaders(downer); + // Send HEAD request to determine range request is supported, and get total length +- downer.clearHeaders(); + if (userAgentOverride.size()){downer.setHeader("User-Agent", userAgentOverride);} + if (!downer.head(myURI) || !downer.isOk()){ + FAIL_MSG("Error getting URI info for '%s': %" PRIu32 " %s", myURI.getUrl().c_str(), +@@ -120,6 +228,8 @@ namespace HTTP{ + myURI = downer.lastURL(); + } + ++ // Other set of headers specified for GET request ++ httpHeaderOverride.prepareGetHeaders(downer); + // streaming mode when size is unknown + if (!supportRangeRequest){ + MEDIUM_MSG("URI get without range request: %s, totalsize: %zu", myURI.getUrl().c_str(), totalSize); +-- +2.25.1 + + +From df4076a06eeb6fbc0fc20d9ace0091eb476420a2 Mon Sep 17 00:00:00 2001 +From: Thulinma +Date: Wed, 17 Aug 2022 14:57:29 +0200 +Subject: [PATCH 12/38] Added ResizeablePointer::shift operator to shift data + forward in buffer + +--- + lib/util.cpp | 11 +++++++++++ + lib/util.h | 1 + + 2 files changed, 12 insertions(+) + +diff --git a/lib/util.cpp b/lib/util.cpp +index 8dd75727..d4ff380f 100644 +--- a/lib/util.cpp ++++ b/lib/util.cpp +@@ -191,6 +191,17 @@ namespace Util{ + maxSize = 0; + } + ++ void ResizeablePointer::shift(size_t byteCount){ ++ //Shifting the entire buffer is easy, we do nothing and set size to zero ++ if (byteCount >= currSize){ ++ currSize = 0; ++ return; ++ } ++ //Shifting partial needs a memmove and size change ++ memmove(ptr, ((char*)ptr)+byteCount, currSize-byteCount); ++ currSize -= byteCount; ++ } ++ + bool ResizeablePointer::assign(const void *p, uint32_t l){ + if (!allocate(l)){return false;} + memcpy(ptr, p, l); +diff --git a/lib/util.h b/lib/util.h +index 84b0d56c..d28a7f76 100644 +--- a/lib/util.h ++++ b/lib/util.h +@@ -45,6 +45,7 @@ namespace Util{ + bool append(const void *p, uint32_t l); + bool append(const std::string &str); + bool allocate(uint32_t l); ++ void shift(size_t byteCount); + uint32_t rsize(); + void truncate(const size_t newLen); + inline operator char *(){return (char *)ptr;} +-- +2.25.1 + + +From 44c28097332b0a137c4674bcfd8d243b593b3cee Mon Sep 17 00:00:00 2001 +From: Thulinma +Date: Thu, 18 Aug 2022 02:44:11 +0200 +Subject: [PATCH 13/38] Decreased verbosity of harmless data offset warnings, + increased verbosity of harmful ones + +--- + src/input/input.cpp | 16 +++++++++++----- + 1 file changed, 11 insertions(+), 5 deletions(-) + +diff --git a/src/input/input.cpp b/src/input/input.cpp +index 942a55f8..0a062392 100644 +--- a/src/input/input.cpp ++++ b/src/input/input.cpp +@@ -1382,10 +1382,12 @@ namespace Mist{ + size_t currPos = tPages.getInt("avail", pageIdx); + if (currPos){ + size_t keySize = keys.getSize(keyNum); +- if (currPos-prevPos != keySize){ +- INFO_MSG("Key %" PRIu32 " was %zu bytes but should've been %zu bytes! (differs %d)", keyNum, currPos-prevPos, keySize, (int)(currPos-prevPos-keySize)); +- }else{ ++ if (currPos-prevPos == keySize){ + VERYHIGH_MSG("Key %" PRIu32 " was %zu bytes", keyNum, currPos-prevPos); ++ }else if (currPos-prevPos > keySize){ ++ FAIL_MSG("Key %" PRIu32 " was %zu bytes but should've been %zu bytes! (differs %d)", keyNum, currPos-prevPos, keySize, (int)(currPos-prevPos-keySize)); ++ }else{ ++ MEDIUM_MSG("Key %" PRIu32 " was %zu bytes but should've been %zu bytes! (differs %d)", keyNum, currPos-prevPos, keySize, (int)(currPos-prevPos-keySize)); + } + ++keyNum; + prevPos = currPos; +@@ -1412,8 +1414,12 @@ namespace Mist{ + size_t currPos = tPages.getInt("avail", pageIdx); + if (currPos){ + size_t keySize = keys.getSize(keyNum); +- if (currPos-prevPos != keySize){ +- INFO_MSG("Key %" PRIu32 " was %zu bytes but should've been %zu bytes! (differs %d)", keyNum, currPos-prevPos, keySize, (int)(currPos-prevPos-keySize)); ++ if (currPos-prevPos == keySize){ ++ VERYHIGH_MSG("Key %" PRIu32 " was %zu bytes", keyNum, currPos-prevPos); ++ }else if (currPos-prevPos > keySize){ ++ FAIL_MSG("Key %" PRIu32 " was %zu bytes but should've been %zu bytes! (differs %d)", keyNum, currPos-prevPos, keySize, (int)(currPos-prevPos-keySize)); ++ }else{ ++ MEDIUM_MSG("Key %" PRIu32 " was %zu bytes but should've been %zu bytes! (differs %d)", keyNum, currPos-prevPos, keySize, (int)(currPos-prevPos-keySize)); + } + ++keyNum; + prevPos = currPos; +-- +2.25.1 + + +From 747438746c068f9217f804fab4d58bdf57156a43 Mon Sep 17 00:00:00 2001 +From: Thulinma +Date: Thu, 18 Aug 2022 15:45:46 +0200 +Subject: [PATCH 14/38] Change VoD data page logic to use wallclock seconds + rather than loop iterations for timeouts + +--- + src/input/input.cpp | 30 ++++++++++++++++++++++-------- + src/input/input.h | 2 +- + 2 files changed, 23 insertions(+), 9 deletions(-) + +diff --git a/src/input/input.cpp b/src/input/input.cpp +index 0a062392..a86fe3cd 100644 +--- a/src/input/input.cpp ++++ b/src/input/input.cpp +@@ -1022,28 +1022,42 @@ namespace Mist{ + } + + void Input::removeUnused(){ ++ uint64_t cTime = Util::bootSecs(); + std::set validTracks = M.getValidTracks(); ++ std::map > checkedPages; + for (std::set::iterator it = validTracks.begin(); it != validTracks.end(); ++it){ + Util::RelAccX &tPages = meta.pages(*it); + for (size_t i = tPages.getDeleted(); i < tPages.getEndPos(); i++){ + uint64_t pageNum = tPages.getInt("firstkey", i); ++ checkedPages[*it].insert(pageNum); + if (pageCounter[*it].count(pageNum)){ + // If the page is still being written to, reset the counter rather than potentially unloading it + if (isCurrentLivePage(*it, pageNum)){ +- pageCounter[*it][pageNum] = DEFAULT_PAGE_TIMEOUT; ++ pageCounter[*it][pageNum] = cTime; + continue; + } +- --pageCounter[*it][pageNum]; +- if (!pageCounter[*it][pageNum]){ ++ if (cTime > pageCounter[*it][pageNum] + DEFAULT_PAGE_TIMEOUT){ + pageCounter[*it].erase(pageNum); + bufferRemove(*it, pageNum); + } ++ }else{ ++ pageCounter[*it][pageNum] = cTime; + } +- else{ +- pageCounter[*it][pageNum] = DEFAULT_PAGE_TIMEOUT; ++ } ++ } ++ //Check pages we buffered but forgot about ++ for (std::map >::iterator it = pageCounter.begin(); ++ it != pageCounter.end(); it++){ ++ for (std::map::iterator it2 = it->second.begin(); it2 != it->second.end(); it2++){ ++ if (!checkedPages.count(it->first) || !checkedPages[it->first].count(it2->first)){ ++ INFO_MSG("Deleting forgotten page %zu:%" PRIu32, it->first, it2->first); ++ bufferRemove(it->first, it2->first); ++ it->second.erase(it2); ++ it2 = it->second.begin(); + } + } + } ++ + } + + std::string formatGUID(const std::string &val){ +@@ -1290,8 +1304,8 @@ namespace Mist{ + } + uint32_t pageNumber = tPages.getInt("firstkey", pageIdx); + if (isBuffered(idx, pageNumber, meta)){ +- // Mark the page for removal after 15 seconds of no one watching it +- pageCounter[idx][pageNumber] = DEFAULT_PAGE_TIMEOUT; ++ // Mark the page as still actively requested ++ pageCounter[idx][pageNumber] = Util::bootSecs(); + DONTEVEN_MSG("Track %zu, key %" PRIu32 " is already buffered in page %" PRIu32 + ". Cancelling bufferFrame", + idx, keyNum, pageNumber); +@@ -1432,7 +1446,7 @@ namespace Mist{ + idx, pageNumber, tPages.getInt("firsttime", pageIdx), thisTime, bufferTimer); + INFO_MSG(" (%" PRIu32 "/%" PRIu64 " parts, %" PRIu64 " bytes)", packCounter, + tPages.getInt("parts", pageIdx), byteCounter); +- pageCounter[idx][pageNumber] = DEFAULT_PAGE_TIMEOUT; ++ pageCounter[idx][pageNumber] = Util::bootSecs(); + return true; + } + +diff --git a/src/input/input.h b/src/input/input.h +index 28880337..8d7e8891 100644 +--- a/src/input/input.h ++++ b/src/input/input.h +@@ -87,7 +87,7 @@ namespace Mist{ + + IPC::sharedPage streamStatus; + +- std::map > pageCounter; ++ std::map > pageCounter; + + static Input *singleton; + +-- +2.25.1 + + +From b210b4f5afc98b3cb4ea46e607267e12cf868831 Mon Sep 17 00:00:00 2001 +From: Thulinma +Date: Wed, 17 Aug 2022 17:06:25 +0200 +Subject: [PATCH 15/38] Fixed seek-related bugs in URIReader for HTTP sources + +--- + lib/urireader.cpp | 40 ++++++++++++++++++++++------------------ + 1 file changed, 22 insertions(+), 18 deletions(-) + +diff --git a/lib/urireader.cpp b/lib/urireader.cpp +index ad85b052..d4dd0e31 100644 +--- a/lib/urireader.cpp ++++ b/lib/urireader.cpp +@@ -255,20 +255,30 @@ namespace HTTP{ + + // seek to pos, return true if succeeded. + bool URIReader::seek(const uint64_t pos){ +- if (isSeekable()){ +- allData.truncate(0); +- bufPos = 0; +- if (stateType == HTTP::File){ +- curPos = pos; +- return true; +- }else if (stateType == HTTP::HTTP && supportRangeRequest){ +- INFO_MSG("SEEK: RangeRequest to %" PRIu64, pos); +- if (!downer.getRangeNonBlocking(myURI.getUrl(), pos, 0)){ +- FAIL_MSG("error loading request"); +- } +- } ++ //Seeking in a non-seekable source? No-op, always fails. ++ if (!isSeekable()){return false;} ++ ++ //Reset internal buffers ++ allData.truncate(0); ++ bufPos = 0; ++ ++ //Files always succeed because we use memmap ++ if (stateType == HTTP::File){ ++ curPos = pos; ++ return true; + } + ++ //HTTP-based needs to do a range request ++ if (stateType == HTTP::HTTP && supportRangeRequest){ ++ downer.getSocket().close(); ++ downer.getSocket().Received().clear(); ++ if (!downer.getRangeNonBlocking(myURI.getUrl(), pos, 0)){ ++ FAIL_MSG("Error making range request"); ++ return false; ++ } ++ curPos = pos; ++ return true; ++ } + return false; + } + +@@ -323,19 +333,13 @@ namespace HTTP{ + + }else if (stateType == HTTP::HTTP){ + downer.continueNonBlocking(cb); +- if (curPos == downer.const_data().size()){ +- Util::sleep(50); +- } +- curPos = downer.const_data().size(); + }else{// streaming mode + int s; +- static int totaal = 0; + if ((downer.getSocket() && downer.getSocket().spool())){// || downer.getSocket().Received().size() > 0){ + s = downer.getSocket().Received().bytes(wantedLen); + std::string buf = downer.getSocket().Received().remove(s); + + cb.dataCallback(buf.data(), s); +- totaal += s; + }else{ + Util::sleep(50); + } +-- +2.25.1 + + +From 01a2ff54edb0cb544495bff57d8cc2a61440db2a Mon Sep 17 00:00:00 2001 +From: Thulinma +Date: Wed, 17 Aug 2022 14:57:48 +0200 +Subject: [PATCH 16/38] Converted MP4 input to use URIReader + +--- + src/input/input_mp4.cpp | 518 ++++++++++++++++++++++------------------ + src/input/input_mp4.h | 15 +- + 2 files changed, 298 insertions(+), 235 deletions(-) + +diff --git a/src/input/input_mp4.cpp b/src/input/input_mp4.cpp +index 2e1c7923..f85911fb 100644 +--- a/src/input/input_mp4.cpp ++++ b/src/input/input_mp4.cpp +@@ -159,11 +159,14 @@ namespace Mist{ + } + + inputMP4::inputMP4(Util::Config *cfg) : Input(cfg){ +- malSize = 4; // initialise data read buffer to 0; +- data = (char *)malloc(malSize); + capa["name"] = "MP4"; + capa["desc"] = "This input allows streaming of MP4 files as Video on Demand."; +- capa["source_match"] = "/*.mp4"; ++ capa["source_match"].append("/*.mp4"); ++ capa["source_match"].append("http://*.mp4"); ++ capa["source_match"].append("https://*.mp4"); ++ capa["source_match"].append("s3+http://*.mp4"); ++ capa["source_match"].append("s3+https://*.mp4"); ++ capa["source_match"].append("mp4:*"); + capa["source_file"] = "$source"; + capa["priority"] = 9; + capa["codecs"][0u][0u].append("HEVC"); +@@ -173,10 +176,9 @@ namespace Mist{ + capa["codecs"][0u][1u].append("AAC"); + capa["codecs"][0u][1u].append("AC3"); + capa["codecs"][0u][1u].append("MP3"); ++ readPos = 0; + } + +- inputMP4::~inputMP4(){free(data);} +- + bool inputMP4::checkArguments(){ + if (config->getString("input") == "-"){ + std::cerr << "Input from stdin not yet supported" << std::endl; +@@ -199,255 +201,292 @@ namespace Mist{ + + bool inputMP4::preRun(){ + // open File +- inFile = fopen(config->getString("input").c_str(), "r"); ++ std::string inUrl = config->getString("input"); ++ if (inUrl.size() > 4 && inUrl.substr(0, 4) == "mp4:"){inUrl.erase(0, 4);} ++ inFile.open(inUrl); + if (!inFile){return false;} ++ if (!inFile.isSeekable()){ ++ FAIL_MSG("MP4 input only supports seekable data sources, for now, and this source is not seekable: %s", config->getString("input").c_str()); ++ return false; ++ } + return true; + } + ++ void inputMP4::dataCallback(const char *ptr, size_t size){readBuffer.append(ptr, size);} ++ + bool inputMP4::readHeader(){ + if (!inFile){ + INFO_MSG("inFile failed!"); + return false; + } ++ bool hasMoov = false; ++ readBuffer.truncate(0); ++ readPos = 0; + + // first we get the necessary header parts + size_t tNumber = 0; +- while (!feof(inFile)){ +- std::string boxType = MP4::readBoxType(inFile); +- if (boxType == "erro"){break;} ++ activityCounter = Util::bootSecs(); ++ while (inFile && keepRunning()){ ++ //Read box header if needed ++ while (readBuffer.size() < 16 && inFile && keepRunning()){inFile.readSome(16, *this);} ++ //Failed? Abort. ++ if (readBuffer.size() < 16){ ++ FAIL_MSG("Could not read box header from input!"); ++ break; ++ } ++ //Box type is always on bytes 5-8 from the start of a box ++ std::string boxType = std::string(readBuffer+4, 4); ++ uint64_t boxSize = MP4::calcBoxSize(readBuffer); + if (boxType == "moov"){ +- MP4::MOOV moovBox; +- moovBox.read(inFile); +- // for all box in moov ++ while (readBuffer.size() < boxSize && inFile && keepRunning()){inFile.readSome(boxSize-readBuffer.size(), *this);} ++ if (readBuffer.size() < boxSize){ ++ FAIL_MSG("Could not read entire MOOV box into memory"); ++ break; ++ } ++ MP4::Box moovBox(readBuffer, false); + +- std::deque trak = moovBox.getChildren(); ++ // for all box in moov ++ std::deque trak = ((MP4::MOOV*)&moovBox)->getChildren(); + for (std::deque::iterator trakIt = trak.begin(); trakIt != trak.end(); trakIt++){ + trackHeaders.push_back(mp4TrackHeader()); + trackHeaders.rbegin()->read(*trakIt); + } +- continue; ++ hasMoov = true; ++ break; + } +- if (!MP4::skipBox(inFile)){// moving on to next box +- FAIL_MSG("Error in skipping box, exiting"); +- return false; ++ activityCounter = Util::bootSecs(); ++ //Skip to next box ++ if (readBuffer.size() > boxSize){ ++ readBuffer.shift(boxSize); ++ readPos += boxSize; ++ }else{ ++ readBuffer.truncate(0); ++ if (!inFile.seek(readPos + boxSize)){ ++ FAIL_MSG("Seek to %" PRIu64 " failed! Aborting load", readPos+boxSize); ++ } ++ readPos = inFile.getPos(); + } + } +- fseeko(inFile, 0, SEEK_SET); + + // See whether a separate header file exists. +- if (readExistingHeader()){return true;} +- HIGH_MSG("Not read existing header"); ++ if (readExistingHeader()){ ++ bps = 0; ++ std::set tracks = M.getValidTracks(); ++ for (std::set::iterator it = tracks.begin(); it != tracks.end(); it++){bps += M.getBps(*it);} ++ return true; ++ } ++ INFO_MSG("Not read existing header"); + + meta.reInit(isSingular() ? streamName : ""); ++ if (!hasMoov){ ++ FAIL_MSG("No MOOV box found; aborting header creation!"); ++ return false; ++ } + + tNumber = 0; + // Create header file from MP4 data +- while (!feof(inFile)){ +- std::string boxType = MP4::readBoxType(inFile); +- if (boxType == "erro"){break;} +- if (boxType == "moov"){ +- MP4::MOOV moovBox; +- moovBox.read(inFile); ++ MP4::Box moovBox(readBuffer, false); + +- std::deque trak = moovBox.getChildren(); +- HIGH_MSG("Obtained %zu trak Boxes", trak.size()); ++ std::deque trak = ((MP4::MOOV*)&moovBox)->getChildren(); ++ HIGH_MSG("Obtained %zu trak Boxes", trak.size()); + +- for (std::deque::iterator trakIt = trak.begin(); trakIt != trak.end(); trakIt++){ +- MP4::MDIA mdiaBox = trakIt->getChild(); ++ for (std::deque::iterator trakIt = trak.begin(); trakIt != trak.end(); trakIt++){ ++ MP4::MDIA mdiaBox = trakIt->getChild(); + +- std::string hdlrType = mdiaBox.getChild().getHandlerType(); +- if (hdlrType != "vide" && hdlrType != "soun" && hdlrType != "sbtl"){ +- INFO_MSG("Unsupported handler: %s", hdlrType.c_str()); +- continue; +- } ++ std::string hdlrType = mdiaBox.getChild().getHandlerType(); ++ if (hdlrType != "vide" && hdlrType != "soun" && hdlrType != "sbtl"){ ++ INFO_MSG("Unsupported handler: %s", hdlrType.c_str()); ++ continue; ++ } + +- tNumber = meta.addTrack(); ++ tNumber = meta.addTrack(); + +- MP4::TKHD tkhdBox = trakIt->getChild(); +- if (tkhdBox.getWidth() > 0){ +- meta.setWidth(tNumber, tkhdBox.getWidth()); +- meta.setHeight(tNumber, tkhdBox.getHeight()); +- } +- meta.setID(tNumber, tkhdBox.getTrackID()); +- +- MP4::MDHD mdhdBox = mdiaBox.getChild(); +- uint64_t timescale = mdhdBox.getTimeScale(); +- meta.setLang(tNumber, mdhdBox.getLanguage()); +- +- MP4::STBL stblBox = mdiaBox.getChild().getChild(); +- +- MP4::STSD stsdBox = stblBox.getChild(); +- MP4::Box sEntryBox = stsdBox.getEntry(0); +- std::string sType = sEntryBox.getType(); +- HIGH_MSG("Found track %zu of type %s", tNumber, sType.c_str()); +- +- if (sType == "avc1" || sType == "h264" || sType == "mp4v"){ +- MP4::VisualSampleEntry &vEntryBox = (MP4::VisualSampleEntry &)sEntryBox; +- meta.setType(tNumber, "video"); +- meta.setCodec(tNumber, "H264"); +- if (!meta.getWidth(tNumber)){ +- meta.setWidth(tNumber, vEntryBox.getWidth()); +- meta.setHeight(tNumber, vEntryBox.getHeight()); +- } +- MP4::Box initBox = vEntryBox.getCLAP(); +- if (initBox.isType("avcC")){ +- meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); +- } +- initBox = vEntryBox.getPASP(); +- if (initBox.isType("avcC")){ +- meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); +- } +- /// this is a hacky way around invalid FLV data (since it gets ignored nearly +- /// everywhere, but we do need correct data... +- if (!meta.getWidth(tNumber)){ +- h264::sequenceParameterSet sps; +- sps.fromDTSCInit(meta.getInit(tNumber)); +- h264::SPSMeta spsChar = sps.getCharacteristics(); +- meta.setWidth(tNumber, spsChar.width); +- meta.setHeight(tNumber, spsChar.height); +- } +- } +- if (sType == "hev1" || sType == "hvc1"){ +- MP4::VisualSampleEntry &vEntryBox = (MP4::VisualSampleEntry &)sEntryBox; +- meta.setType(tNumber, "video"); +- meta.setCodec(tNumber, "HEVC"); +- if (!meta.getWidth(tNumber)){ +- meta.setWidth(tNumber, vEntryBox.getWidth()); +- meta.setHeight(tNumber, vEntryBox.getHeight()); +- } +- MP4::Box initBox = vEntryBox.getCLAP(); +- if (initBox.isType("hvcC")){ +- meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); +- } +- initBox = vEntryBox.getPASP(); +- if (initBox.isType("hvcC")){ +- meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); +- } +- } +- if (sType == "mp4a" || sType == "aac " || sType == "ac-3"){ +- MP4::AudioSampleEntry &aEntryBox = (MP4::AudioSampleEntry &)sEntryBox; +- meta.setType(tNumber, "audio"); +- meta.setChannels(tNumber, aEntryBox.getChannelCount()); +- meta.setRate(tNumber, aEntryBox.getSampleRate()); +- +- if (sType == "ac-3"){ +- meta.setCodec(tNumber, "AC3"); +- }else{ +- MP4::ESDS esdsBox = (MP4::ESDS &)(aEntryBox.getCodecBox()); +- meta.setCodec(tNumber, esdsBox.getCodec()); +- meta.setInit(tNumber, esdsBox.getInitData()); +- } +- meta.setSize(tNumber, 16); ///\todo this might be nice to calculate from mp4 file; +- } +- if (sType == "tx3g"){// plain text subtitles +- meta.setType(tNumber, "meta"); +- meta.setCodec(tNumber, "subtitle"); ++ MP4::TKHD tkhdBox = trakIt->getChild(); ++ if (tkhdBox.getWidth() > 0){ ++ meta.setWidth(tNumber, tkhdBox.getWidth()); ++ meta.setHeight(tNumber, tkhdBox.getHeight()); ++ } ++ meta.setID(tNumber, tkhdBox.getTrackID()); ++ ++ MP4::MDHD mdhdBox = mdiaBox.getChild(); ++ uint64_t timescale = mdhdBox.getTimeScale(); ++ meta.setLang(tNumber, mdhdBox.getLanguage()); ++ ++ MP4::STBL stblBox = mdiaBox.getChild().getChild(); ++ ++ MP4::STSD stsdBox = stblBox.getChild(); ++ MP4::Box sEntryBox = stsdBox.getEntry(0); ++ std::string sType = sEntryBox.getType(); ++ HIGH_MSG("Found track %zu of type %s", tNumber, sType.c_str()); ++ ++ if (sType == "avc1" || sType == "h264" || sType == "mp4v"){ ++ MP4::VisualSampleEntry &vEntryBox = (MP4::VisualSampleEntry &)sEntryBox; ++ meta.setType(tNumber, "video"); ++ meta.setCodec(tNumber, "H264"); ++ if (!meta.getWidth(tNumber)){ ++ meta.setWidth(tNumber, vEntryBox.getWidth()); ++ meta.setHeight(tNumber, vEntryBox.getHeight()); ++ } ++ MP4::Box initBox = vEntryBox.getCLAP(); ++ if (initBox.isType("avcC")){ ++ meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); ++ } ++ initBox = vEntryBox.getPASP(); ++ if (initBox.isType("avcC")){ ++ meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); ++ } ++ /// this is a hacky way around invalid FLV data (since it gets ignored nearly ++ /// everywhere, but we do need correct data... ++ if (!meta.getWidth(tNumber)){ ++ h264::sequenceParameterSet sps; ++ sps.fromDTSCInit(meta.getInit(tNumber)); ++ h264::SPSMeta spsChar = sps.getCharacteristics(); ++ meta.setWidth(tNumber, spsChar.width); ++ meta.setHeight(tNumber, spsChar.height); ++ } ++ } ++ if (sType == "hev1" || sType == "hvc1"){ ++ MP4::VisualSampleEntry &vEntryBox = (MP4::VisualSampleEntry &)sEntryBox; ++ meta.setType(tNumber, "video"); ++ meta.setCodec(tNumber, "HEVC"); ++ if (!meta.getWidth(tNumber)){ ++ meta.setWidth(tNumber, vEntryBox.getWidth()); ++ meta.setHeight(tNumber, vEntryBox.getHeight()); ++ } ++ MP4::Box initBox = vEntryBox.getCLAP(); ++ if (initBox.isType("hvcC")){ ++ meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); ++ } ++ initBox = vEntryBox.getPASP(); ++ if (initBox.isType("hvcC")){ ++ meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); ++ } ++ } ++ if (sType == "mp4a" || sType == "aac " || sType == "ac-3"){ ++ MP4::AudioSampleEntry &aEntryBox = (MP4::AudioSampleEntry &)sEntryBox; ++ meta.setType(tNumber, "audio"); ++ meta.setChannels(tNumber, aEntryBox.getChannelCount()); ++ meta.setRate(tNumber, aEntryBox.getSampleRate()); ++ ++ if (sType == "ac-3"){ ++ meta.setCodec(tNumber, "AC3"); ++ }else{ ++ MP4::ESDS esdsBox = (MP4::ESDS &)(aEntryBox.getCodecBox()); ++ meta.setCodec(tNumber, esdsBox.getCodec()); ++ meta.setInit(tNumber, esdsBox.getInitData()); ++ } ++ meta.setSize(tNumber, 16); ///\todo this might be nice to calculate from mp4 file; ++ } ++ if (sType == "tx3g"){// plain text subtitles ++ meta.setType(tNumber, "meta"); ++ meta.setCodec(tNumber, "subtitle"); ++ } ++ ++ MP4::STSS stssBox = stblBox.getChild(); ++ MP4::STTS sttsBox = stblBox.getChild(); ++ MP4::STSZ stszBox = stblBox.getChild(); ++ MP4::STCO stcoBox = stblBox.getChild(); ++ MP4::CO64 co64Box = stblBox.getChild(); ++ MP4::STSC stscBox = stblBox.getChild(); ++ MP4::CTTS cttsBox = stblBox.getChild(); // optional ctts box ++ ++ bool stco64 = co64Box.isType("co64"); ++ bool hasCTTS = cttsBox.isType("ctts"); ++ ++ uint64_t totaldur = 0; ///\todo note: set this to begin time ++ mp4PartBpos BsetPart; ++ ++ uint64_t entryNo = 0; ++ uint64_t sampleNo = 0; ++ ++ uint64_t stssIndex = 0; ++ uint64_t stcoIndex = 0; ++ uint64_t stscIndex = 0; ++ uint64_t cttsIndex = 0; // current ctts Index we are reading ++ uint64_t cttsEntryRead = 0; // current part of ctts we are reading ++ ++ uint64_t stssCount = stssBox.getEntryCount(); ++ uint64_t stscCount = stscBox.getEntryCount(); ++ uint64_t stszCount = stszBox.getSampleCount(); ++ uint64_t stcoCount = (stco64 ? co64Box.getEntryCount() : stcoBox.getEntryCount()); ++ ++ MP4::STTSEntry sttsEntry = sttsBox.getSTTSEntry(0); ++ ++ uint32_t fromSTCOinSTSC = 0; ++ uint64_t tmpOffset = (stco64 ? co64Box.getChunkOffset(0) : stcoBox.getChunkOffset(0)); ++ ++ uint64_t nextFirstChunk = (stscCount > 1 ? stscBox.getSTSCEntry(1).firstChunk - 1 : stcoCount); ++ ++ for (uint64_t stszIndex = 0; stszIndex < stszCount; ++stszIndex){ ++ if (stcoIndex >= nextFirstChunk){ ++ ++stscIndex; ++ nextFirstChunk = ++ (stscIndex + 1 < stscCount ? stscBox.getSTSCEntry(stscIndex + 1).firstChunk - 1 : stcoCount); ++ } ++ BsetPart.keyframe = (meta.getType(tNumber) == "video" && stssIndex < stssCount && ++ stszIndex + 1 == stssBox.getSampleNumber(stssIndex)); ++ if (BsetPart.keyframe){++stssIndex;} ++ // in bpos set ++ BsetPart.stcoNr = stcoIndex; ++ // bpos = chunkoffset[samplenr] in stco ++ BsetPart.bpos = tmpOffset; ++ ++fromSTCOinSTSC; ++ if (fromSTCOinSTSC < stscBox.getSTSCEntry(stscIndex).samplesPerChunk){// as long as we are still in this chunk ++ tmpOffset += stszBox.getEntrySize(stszIndex); ++ }else{ ++ ++stcoIndex; ++ fromSTCOinSTSC = 0; ++ tmpOffset = (stco64 ? co64Box.getChunkOffset(stcoIndex) : stcoBox.getChunkOffset(stcoIndex)); ++ } ++ BsetPart.time = (totaldur * 1000) / timescale; ++ totaldur += sttsEntry.sampleDelta; ++ sampleNo++; ++ if (sampleNo >= sttsEntry.sampleCount){ ++ ++entryNo; ++ sampleNo = 0; ++ if (entryNo < sttsBox.getEntryCount()){sttsEntry = sttsBox.getSTTSEntry(entryNo);} ++ } ++ ++ if (hasCTTS){ ++ MP4::CTTSEntry cttsEntry = cttsBox.getCTTSEntry(cttsIndex); ++ cttsEntryRead++; ++ if (cttsEntryRead >= cttsEntry.sampleCount){ ++ ++cttsIndex; ++ cttsEntryRead = 0; + } ++ BsetPart.timeOffset = (cttsEntry.sampleOffset * 1000) / timescale; ++ }else{ ++ BsetPart.timeOffset = 0; ++ } + +- MP4::STSS stssBox = stblBox.getChild(); +- MP4::STTS sttsBox = stblBox.getChild(); +- MP4::STSZ stszBox = stblBox.getChild(); +- MP4::STCO stcoBox = stblBox.getChild(); +- MP4::CO64 co64Box = stblBox.getChild(); +- MP4::STSC stscBox = stblBox.getChild(); +- MP4::CTTS cttsBox = stblBox.getChild(); // optional ctts box +- +- bool stco64 = co64Box.isType("co64"); +- bool hasCTTS = cttsBox.isType("ctts"); +- +- uint64_t totaldur = 0; ///\todo note: set this to begin time +- mp4PartBpos BsetPart; +- +- uint64_t entryNo = 0; +- uint64_t sampleNo = 0; +- +- uint64_t stssIndex = 0; +- uint64_t stcoIndex = 0; +- uint64_t stscIndex = 0; +- uint64_t cttsIndex = 0; // current ctts Index we are reading +- uint64_t cttsEntryRead = 0; // current part of ctts we are reading +- +- uint64_t stssCount = stssBox.getEntryCount(); +- uint64_t stscCount = stscBox.getEntryCount(); +- uint64_t stszCount = stszBox.getSampleCount(); +- uint64_t stcoCount = (stco64 ? co64Box.getEntryCount() : stcoBox.getEntryCount()); +- +- MP4::STTSEntry sttsEntry = sttsBox.getSTTSEntry(0); +- +- uint32_t fromSTCOinSTSC = 0; +- uint64_t tmpOffset = (stco64 ? co64Box.getChunkOffset(0) : stcoBox.getChunkOffset(0)); +- +- uint64_t nextFirstChunk = (stscCount > 1 ? stscBox.getSTSCEntry(1).firstChunk - 1 : stcoCount); +- +- for (uint64_t stszIndex = 0; stszIndex < stszCount; ++stszIndex){ +- if (stcoIndex >= nextFirstChunk){ +- ++stscIndex; +- nextFirstChunk = +- (stscIndex + 1 < stscCount ? stscBox.getSTSCEntry(stscIndex + 1).firstChunk - 1 : stcoCount); +- } +- BsetPart.keyframe = (meta.getType(tNumber) == "video" && stssIndex < stssCount && +- stszIndex + 1 == stssBox.getSampleNumber(stssIndex)); +- if (BsetPart.keyframe){++stssIndex;} +- // in bpos set +- BsetPart.stcoNr = stcoIndex; +- // bpos = chunkoffset[samplenr] in stco +- BsetPart.bpos = tmpOffset; +- ++fromSTCOinSTSC; +- if (fromSTCOinSTSC < stscBox.getSTSCEntry(stscIndex).samplesPerChunk){// as long as we are still in this chunk +- tmpOffset += stszBox.getEntrySize(stszIndex); +- }else{ +- ++stcoIndex; +- fromSTCOinSTSC = 0; +- tmpOffset = (stco64 ? co64Box.getChunkOffset(stcoIndex) : stcoBox.getChunkOffset(stcoIndex)); +- } +- BsetPart.time = (totaldur * 1000) / timescale; +- totaldur += sttsEntry.sampleDelta; +- sampleNo++; +- if (sampleNo >= sttsEntry.sampleCount){ +- ++entryNo; +- sampleNo = 0; +- if (entryNo < sttsBox.getEntryCount()){sttsEntry = sttsBox.getSTTSEntry(entryNo);} +- } +- +- if (hasCTTS){ +- MP4::CTTSEntry cttsEntry = cttsBox.getCTTSEntry(cttsIndex); +- cttsEntryRead++; +- if (cttsEntryRead >= cttsEntry.sampleCount){ +- ++cttsIndex; +- cttsEntryRead = 0; +- } +- BsetPart.timeOffset = (cttsEntry.sampleOffset * 1000) / timescale; +- }else{ +- BsetPart.timeOffset = 0; +- } +- +- if (sType == "tx3g"){ +- if (stszBox.getEntrySize(stszIndex) <= 2 && false){ +- FAIL_MSG("size <=2"); +- }else{ +- long long packSendSize = 0; +- packSendSize = 24 + (BsetPart.timeOffset ? 17 : 0) + (BsetPart.bpos ? 15 : 0) + 19 + +- stszBox.getEntrySize(stszIndex) + 11 - 2 + 19; +- meta.update(BsetPart.time, BsetPart.timeOffset, tNumber, +- stszBox.getEntrySize(stszIndex) - 2, BsetPart.bpos, true, packSendSize); +- } +- }else{ +- meta.update(BsetPart.time, BsetPart.timeOffset, tNumber, +- stszBox.getEntrySize(stszIndex), BsetPart.bpos, BsetPart.keyframe); +- } ++ if (sType == "tx3g"){ ++ if (stszBox.getEntrySize(stszIndex) <= 2 && false){ ++ FAIL_MSG("size <=2"); ++ }else{ ++ long long packSendSize = 0; ++ packSendSize = 24 + (BsetPart.timeOffset ? 17 : 0) + (BsetPart.bpos ? 15 : 0) + 19 + ++ stszBox.getEntrySize(stszIndex) + 11 - 2 + 19; ++ meta.update(BsetPart.time, BsetPart.timeOffset, tNumber, ++ stszBox.getEntrySize(stszIndex) - 2, BsetPart.bpos, true, packSendSize); + } ++ }else{ ++ meta.update(BsetPart.time, BsetPart.timeOffset, tNumber, ++ stszBox.getEntrySize(stszIndex), BsetPart.bpos, BsetPart.keyframe); + } +- continue; +- } +- if (!MP4::skipBox(inFile)){// moving on to next box +- FAIL_MSG("Error in Skipping box, exiting"); +- return false; + } + } +- clearerr(inFile); + + // outputting dtsh file +- M.toFile(config->getString("input") + ".dtsh"); ++ std::string inUrl = config->getString("input"); ++ if (inUrl.size() > 4 && inUrl.substr(0, 4) == "mp4:"){inUrl.erase(0, 4);} ++ if (inUrl != "-" && HTTP::URL(inUrl).isLocalPath()){ ++ M.toFile(inUrl + ".dtsh"); ++ }else{ ++ INFO_MSG("Skipping header write, as the source is not a local file"); ++ } ++ bps = 0; ++ std::set tracks = M.getValidTracks(); ++ for (std::set::iterator it = tracks.begin(); it != tracks.end(); it++){bps += M.getBps(*it);} + return true; + } + +@@ -474,23 +513,48 @@ namespace Mist{ + ++nextKeyNum; + } + } +- if (fseeko(inFile, curPart.bpos, SEEK_SET)){ +- FAIL_MSG("seek unsuccessful @bpos %" PRIu64 ": %s", curPart.bpos, strerror(errno)); +- thisPacket.null(); +- return; ++ if (curPart.bpos < readPos || curPart.bpos > readPos + readBuffer.size() + 512*1024 + bps){ ++ INFO_MSG("Buffer contains %" PRIu64 "-%" PRIu64 ", but we need %" PRIu64 "; seeking!", readPos, readPos + readBuffer.size(), curPart.bpos); ++ readBuffer.truncate(0); ++ if (!inFile.seek(curPart.bpos)){ ++ FAIL_MSG("seek unsuccessful @bpos %" PRIu64 ": %s", curPart.bpos, strerror(errno)); ++ thisPacket.null(); ++ return; ++ } ++ readPos = inFile.getPos(); ++ }else{ ++ //If we have more than 5MiB buffered and are more than 5MiB into the buffer, shift the first 4MiB off the buffer. ++ //This prevents infinite growth of the read buffer for large files ++ if (readBuffer.size() >= 5*1024*1024 && curPart.bpos > readPos + 5*1024*1024 + bps){ ++ readBuffer.shift(4*1024*1024); ++ readPos += 4*1024*1024; ++ } + } +- if (curPart.size > malSize){ +- data = (char *)realloc(data, curPart.size); +- malSize = curPart.size; ++ ++ while (readPos+readBuffer.size() < curPart.bpos+curPart.size && inFile && keepRunning()){ ++ inFile.readSome((curPart.bpos+curPart.size) - (readPos+readBuffer.size()), *this); + } +- if (fread(data, curPart.size, 1, inFile) != 1){ +- FAIL_MSG("read unsuccessful at %ld", ftell(inFile)); +- thisPacket.null(); +- return; ++ if (readPos+readBuffer.size() < curPart.bpos+curPart.size){ ++ FAIL_MSG("Read unsuccessful at %" PRIu64 ", seeking to retry...", readPos+readBuffer.size()); ++ readBuffer.truncate(0); ++ if (!inFile.seek(curPart.bpos)){ ++ FAIL_MSG("seek unsuccessful @bpos %" PRIu64 ": %s", curPart.bpos, strerror(errno)); ++ thisPacket.null(); ++ return; ++ } ++ readPos = inFile.getPos(); ++ while (readPos+readBuffer.size() < curPart.bpos+curPart.size && inFile && keepRunning()){ ++ inFile.readSome((curPart.bpos+curPart.size) - (readPos+readBuffer.size()), *this); ++ } ++ if (readPos+readBuffer.size() < curPart.bpos+curPart.size){ ++ FAIL_MSG("Read retry unsuccessful at %" PRIu64 ", aborting", readPos+readBuffer.size()); ++ thisPacket.null(); ++ return; ++ } + } + + if (M.getCodec(curPart.trackID) == "subtitle"){ +- unsigned int txtLen = Bit::btohs(data); ++ unsigned int txtLen = Bit::btohs(readBuffer + (curPart.bpos-readPos)); + if (!txtLen && false){ + curPart.index++; + return getNext(idx); +@@ -499,14 +563,14 @@ namespace Mist{ + thisPack.null(); + thisPack["trackid"] = curPart.trackID; + thisPack["bpos"] = curPart.bpos; //(long long)fileSource.tellg(); +- thisPack["data"] = std::string(data + 2, txtLen); ++ thisPack["data"] = std::string(readBuffer + (curPart.bpos-readPos) + 2, txtLen); + thisPack["time"] = curPart.time; + if (curPart.duration){thisPack["duration"] = curPart.duration;} + thisPack["keyframe"] = true; + std::string tmpStr = thisPack.toNetPacked(); + thisPacket.reInit(tmpStr.data(), tmpStr.size()); + }else{ +- thisPacket.genericFill(curPart.time, curPart.offset, curPart.trackID, data, curPart.size, 0, isKeyframe); ++ thisPacket.genericFill(curPart.time, curPart.offset, curPart.trackID, readBuffer + (curPart.bpos-readPos), curPart.size, 0, isKeyframe); + } + thisTime = curPart.time; + thisIdx = curPart.trackID; +diff --git a/src/input/input_mp4.h b/src/input/input_mp4.h +index b5987d3b..4ab9f36e 100644 +--- a/src/input/input_mp4.h ++++ b/src/input/input_mp4.h +@@ -1,5 +1,6 @@ + #include "input.h" + #include ++#include + #include + #include + namespace Mist{ +@@ -70,10 +71,10 @@ namespace Mist{ + bool stco64; + }; + +- class inputMP4 : public Input{ ++ class inputMP4 : public Input, public Util::DataCallback { + public: + inputMP4(Util::Config *cfg); +- ~inputMP4(); ++ void dataCallback(const char *ptr, size_t size); + + protected: + // Private Functions +@@ -85,7 +86,10 @@ namespace Mist{ + void seek(uint64_t seekTime, size_t idx = INVALID_TRACK_ID); + void handleSeek(uint64_t seekTime, size_t idx); + +- FILE *inFile; ++ HTTP::URIReader inFile; ++ Util::ResizeablePointer readBuffer; ++ uint64_t readPos; ++ uint64_t bps; + + mp4TrackHeader &headerData(size_t trackID); + +@@ -94,11 +98,6 @@ namespace Mist{ + + // remember last seeked keyframe; + std::map nextKeyframe; +- +- // these next two variables keep a buffer for reading from filepointer inFile; +- uint64_t malSize; +- char *data; ///\todo rename this variable to a more sensible name, it is a temporary piece of +- /// memory to read from files + }; + }// namespace Mist + +-- +2.25.1 + + +From 2740d65a0f5715c42eb6de24cbf610ba6271b85a Mon Sep 17 00:00:00 2001 +From: Thulinma +Date: Wed, 17 Aug 2022 10:54:31 +0200 +Subject: [PATCH 17/38] Improved RTP timestamp logging, fixed bug related to + firstTime value + +--- + lib/rtp.cpp | 7 ++++--- + 1 file changed, 4 insertions(+), 3 deletions(-) + +diff --git a/lib/rtp.cpp b/lib/rtp.cpp +index 3ed9b7c6..5a5ed829 100644 +--- a/lib/rtp.cpp ++++ b/lib/rtp.cpp +@@ -877,7 +877,7 @@ namespace RTP{ + if (!firstTime){ + milliSync = Util::bootMS(); + firstTime = pTime + 1; +- INFO_MSG("RTP timestamp rollover expected in " PRETTY_PRINT_TIME, ++ INFO_MSG("RTP timestamp rollover for %" PRIu64 " (%s) expected in " PRETTY_PRINT_TIME, trackId, codec.c_str(), + PRETTY_ARG_TIME((0xFFFFFFFFul - firstTime) / multiplier / 1000)); + }else{ + if (recentWrap){ +@@ -886,14 +886,15 @@ namespace RTP{ + }else{ + if (prevTime > pTime && pTime < 0x40000000lu && prevTime > 0x80000000lu){ + ++wrapArounds; ++ INFO_MSG("RTP timestamp rollover %" PRIu32 " for %" PRIu64 " (%s) happened; next should be in " PRETTY_PRINT_TIME, wrapArounds, trackId, codec.c_str(), PRETTY_ARG_TIME((0xFFFFFFFFul) / multiplier / 1000)); + recentWrap = true; + } + } + } + // When there are B-frames, the firstTime can be higher than the current time + // causing msTime to become negative and thus overflow +- if (firstTime > pTime + 1){ +- WARN_MSG("firstTime was higher than current packet time. Readjusting firsTime..."); ++ if (!wrapArounds && firstTime > pTime + 1){ ++ WARN_MSG("firstTime was higher than current packet time. Readjusting firstTime..."); + firstTime = pTime + 1; + } + prevTime = pkt.getTimeStamp(); +-- +2.25.1 + + +From 2870ae1cedd202755bbe9d0e6808086d40656157 Mon Sep 17 00:00:00 2001 +From: Thulinma +Date: Wed, 24 Aug 2022 15:02:19 +0200 +Subject: [PATCH 18/38] =?UTF-8?q?SRT=20RAW=20fixup=20=F0=9F=A4=A6?= +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +--- + src/input/input_tssrt.cpp | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/src/input/input_tssrt.cpp b/src/input/input_tssrt.cpp +index da69be29..4219ebbc 100644 +--- a/src/input/input_tssrt.cpp ++++ b/src/input/input_tssrt.cpp +@@ -211,7 +211,6 @@ namespace Mist{ + } + uint64_t packetTime = Util::bootMS(); + thisPacket.genericFill(packetTime, 0, 1, rawBuffer, rawBuffer.size(), 0, 0); +- bufferLivePacket(thisPacket); + lastRawPacket = packetTime; + rawBuffer.truncate(0); + return; +-- +2.25.1 + + +From a9ddc377895f5feb898c162de0f6b0363cf94276 Mon Sep 17 00:00:00 2001 +From: Thulinma +Date: Thu, 25 Aug 2022 10:26:38 +0200 +Subject: [PATCH 19/38] =?UTF-8?q?Why,=20Haivision,=20why=3F=20=F0=9F=98=AD?= +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +--- + lib/socket_srt.cpp | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/lib/socket_srt.cpp b/lib/socket_srt.cpp +index 3e03bce4..6d1d5695 100644 +--- a/lib/socket_srt.cpp ++++ b/lib/socket_srt.cpp +@@ -392,6 +392,10 @@ namespace Socket{ + } + } + params["payloadsize"] = asString(chunkTransmitSize); ++ //This line forces the transmission type to live if unset. ++ //Live is actually the default, but not explicitly setting the option means ++ //that all other defaults do not get applied either, which is bad. ++ if (!params.count("transtype")){params["transtype"] = "live";} + } + + int SRTConnection::preConfigureSocket(){ +-- +2.25.1 + + +From 8aceff951d3e320786bf33c9ff558a506df372c6 Mon Sep 17 00:00:00 2001 +From: Marco van Dijk +Date: Mon, 29 Aug 2022 14:36:00 +0200 +Subject: [PATCH 20/38] Fix connections getting closed on interrupted system + call + +--- + lib/socket.cpp | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/lib/socket.cpp b/lib/socket.cpp +index 865c3be7..6d63af00 100644 +--- a/lib/socket.cpp ++++ b/lib/socket.cpp +@@ -1032,6 +1032,7 @@ unsigned int Socket::Connection::iwrite(const void *buffer, int len){ + case MBEDTLS_ERR_SSL_WANT_WRITE: return 0; break; + case MBEDTLS_ERR_SSL_WANT_READ: return 0; break; + case EWOULDBLOCK: return 0; break; ++ case EINTR: return 0; break; + default: + Error = true; + lastErr = strerror(errno); +@@ -1071,6 +1072,7 @@ unsigned int Socket::Connection::iwrite(const void *buffer, int len){ + if (r < 0){ + switch (errno){ + case EWOULDBLOCK: return 0; break; ++ case EINTR: return 0; break; + default: + Error = true; + lastErr = strerror(errno); +-- +2.25.1 + + +From 54a46146c2527004a96b78fe8b7341eab4de6f62 Mon Sep 17 00:00:00 2001 +From: Thulinma +Date: Thu, 11 Aug 2022 13:17:13 +0200 +Subject: [PATCH 21/38] Fix TS inputs taking more and more memory over time in + some cases + +--- + lib/ts_stream.cpp | 12 +++++++++--- + 1 file changed, 9 insertions(+), 3 deletions(-) + +diff --git a/lib/ts_stream.cpp b/lib/ts_stream.cpp +index 4ab20c59..8da72493 100644 +--- a/lib/ts_stream.cpp ++++ b/lib/ts_stream.cpp +@@ -190,7 +190,8 @@ namespace TS{ + uint32_t tid = newPack.getPID(); + bool unitStart = newPack.getUnitStart(); + static uint32_t wantPrev = 0; +- bool wantTrack = ((wantPrev == tid) || (tid == 0 || newPack.isPMT(pmtTracks) || pidToCodec.count(tid))); ++ bool isData = pidToCodec.count(tid); ++ bool wantTrack = ((wantPrev == tid) || (tid == 0 || newPack.isPMT(pmtTracks) || isData)); + if (!wantTrack){return;} + if (psCacheTid != tid || !psCache){ + psCache = &(pesStreams[tid]); +@@ -199,7 +200,7 @@ namespace TS{ + if (unitStart || !psCache->empty()){ + wantPrev = tid; + psCache->push_back(newPack); +- if (unitStart){ ++ if (unitStart && isData){ + pesPositions[tid].push_back(bytePos); + ++(seenUnitStart[tid]); + } +@@ -210,7 +211,7 @@ namespace TS{ + if (tid == 0){return false;} + { + tthread::lock_guard guard(tMutex); +- return !pmtTracks.count(tid); ++ return pidToCodec.count(tid); + } + } + +@@ -280,6 +281,11 @@ namespace TS{ + } + + if (!pidToCodec.count(tid)){ ++ pesStreams.erase(tid); ++ pesPositions.erase(tid); ++ seenUnitStart.erase(tid); ++ psCacheTid = 0; ++ psCache = 0; + return; // skip unknown codecs + } + +-- +2.25.1 + + +From c10d5a7ec1fdc71cc4af2329a9030b422735fd34 Mon Sep 17 00:00:00 2001 +From: Marco van Dijk +Date: Wed, 16 Mar 2022 13:45:37 +0100 +Subject: [PATCH 22/38] onFail on a WS connection, send the error back using + the websocket request handler + +--- + src/output/output_http_internal.cpp | 20 ++++++++++++++++++-- + src/output/output_http_internal.h | 1 + + 2 files changed, 19 insertions(+), 2 deletions(-) + +diff --git a/src/output/output_http_internal.cpp b/src/output/output_http_internal.cpp +index c8ca2e6c..97ca455f 100644 +--- a/src/output/output_http_internal.cpp ++++ b/src/output/output_http_internal.cpp +@@ -36,6 +36,7 @@ namespace Mist{ + + OutHTTP::OutHTTP(Socket::Connection &conn) : HTTPOutput(conn){ + stayConnected = false; ++ thisError = ""; + // If this connection is a socket and not already connected to stdio, connect it to stdio. + if (myConn.getPureSocket() != -1 && myConn.getSocket() != STDIN_FILENO && myConn.getSocket() != STDOUT_FILENO){ + std::string host = getConnectedHost(); +@@ -63,6 +64,11 @@ namespace Mist{ + bool OutHTTP::listenMode(){return !(config->getString("ip").size());} + + void OutHTTP::onFail(const std::string &msg, bool critical){ ++ // If we are connected through WS, the websockethandler should return the error message ++ if (stayConnected){ ++ thisError = msg; ++ return; ++ } + if (responded){ + HTTPOutput::onFail(msg, critical); + return; +@@ -78,7 +84,6 @@ namespace Mist{ + return; + } + if (H.url.size() >= 3 && H.url.substr(H.url.size() - 3) == ".js"){ +- if (websocketHandler()){return;} + JSON::Value json_resp; + json_resp["error"] = "Could not retrieve stream. Sorry."; + json_resp["error_guru"] = msg; +@@ -1149,12 +1154,23 @@ namespace Mist{ + if (meta){meta.reloadReplacedPagesIfNeeded();} + if (newState != prevState || (newState == STRMSTAT_READY && M.getValidTracks() != prevTracks)){ + if (newState == STRMSTAT_READY){ ++ thisError = ""; + reconnect(); + prevTracks = M.getValidTracks(); + }else{ + disconnect(); + } +- JSON::Value resp = getStatusJSON(reqHost, useragent); ++ JSON::Value resp; ++ // Check if we have an error message set ++ if (thisError == ""){ ++ resp = getStatusJSON(reqHost, useragent); ++ }else{ ++ resp["error"] = "Could not retrieve stream. Sorry."; ++ resp["error_guru"] = thisError; ++ if (config->getString("nostreamtext") != ""){ ++ resp["on_error"] = config->getString("nostreamtext"); ++ } ++ } + if (currStreamName != streamName){ + currStreamName = streamName; + snprintf(pageName, NAME_BUFFER_SIZE, SHM_STREAM_STATE, streamName.c_str()); +diff --git a/src/output/output_http_internal.h b/src/output/output_http_internal.h +index 0f939e78..774eb186 100644 +--- a/src/output/output_http_internal.h ++++ b/src/output/output_http_internal.h +@@ -21,6 +21,7 @@ namespace Mist{ + private: + std::string origStreamName; + std::string mistPath; ++ std::string thisError; + }; + }// namespace Mist + +-- +2.25.1 + + +From f418fed81c0c46fd338e238b558085f408456ed9 Mon Sep 17 00:00:00 2001 +From: Thulinma +Date: Thu, 24 Mar 2022 15:20:52 +0100 +Subject: [PATCH 23/38] Fix HLS output not setting "responded" value + appropriately + +--- + src/output/output_hls.cpp | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/src/output/output_hls.cpp b/src/output/output_hls.cpp +index 19d083e6..ad810aa7 100644 +--- a/src/output/output_hls.cpp ++++ b/src/output/output_hls.cpp +@@ -232,6 +232,7 @@ namespace Mist{ + H.setCORSHeaders(); + if (H.method == "OPTIONS" || H.method == "HEAD"){ + H.SendResponse("200", "OK", myConn); ++ responded = true; + return; + } + H.SetBody(""); + H.SendResponse("200", "OK", myConn); ++ responded = true; + return; + }// crossdomain.xml + +@@ -262,6 +264,7 @@ namespace Mist{ + } + H.SetBody(""); + H.SendResponse("200", "OK", myConn); ++ responded = true; + return; + } + +@@ -311,6 +314,7 @@ namespace Mist{ + targetTime = HLS::getPartTargetTime(M, idx, mTrack, startTime, msn, part); + if (!targetTime){ + H.SendResponse("404", "Partial fragment does not exist", myConn); ++ responded = true; + return; + } + startTime += part * HLS::partDurationMaxMs; +@@ -372,6 +376,7 @@ namespace Mist{ + "served.\n"); + myConn.SendNow(H.BuildResponse("404", "Fragment out of range")); + WARN_MSG("Fragment @ %" PRIu64 " too old", startTime); ++ responded = true; + return; + } + +@@ -389,10 +394,12 @@ namespace Mist{ + } + if (H.method == "OPTIONS" || H.method == "HEAD"){ + H.SendResponse("200", "OK", myConn); ++ responded = true; + return; + } + + H.StartResponse(H, myConn, VLCworkaround || config->getBool("nonchunked")); ++ responded = true; + // we assume whole fragments - but timestamps may be altered at will + contPAT = fragmentIndex; // PAT continuity counter + contPMT = fragmentIndex; // PMT continuity counter +@@ -419,6 +426,7 @@ namespace Mist{ + // Strip /hls// from url + std::string url = H.url.substr(H.url.find('/', 5) + 1); + sendHlsManifest(url); ++ responded = true; + } + } + +-- +2.25.1 + + +From f3c003481d572c140643e9bbfb292942d8e2ed38 Mon Sep 17 00:00:00 2001 +From: Thulinma +Date: Mon, 26 Sep 2022 12:44:30 +0200 +Subject: [PATCH 24/38] Made FLV memory-based loader functions use const + pointers + +--- + lib/flv_tag.cpp | 6 +++--- + lib/flv_tag.h | 6 +++--- + 2 files changed, 6 insertions(+), 6 deletions(-) + +diff --git a/lib/flv_tag.cpp b/lib/flv_tag.cpp +index 9ccb6c41..780c6c5a 100644 +--- a/lib/flv_tag.cpp ++++ b/lib/flv_tag.cpp +@@ -49,7 +49,7 @@ bool FLV::check_header(char *header){ + + /// Checks the first 3 bytes for the string "FLV". Implementing a basic FLV header check, + /// returning true if it is, false if not. +-bool FLV::is_header(char *header){ ++bool FLV::is_header(const char *header){ + if (header[0] != 'F') return false; + if (header[1] != 'L') return false; + if (header[2] != 'V') return false; +@@ -624,7 +624,7 @@ bool FLV::Tag::ChunkLoader(const RTMPStream::Chunk &O){ + /// \param S The size of the data buffer. + /// \param P The current position in the data buffer. Will be updated to reflect new position. + /// \return True if count bytes are read succesfully, false otherwise. +-bool FLV::Tag::MemReadUntil(char *buffer, unsigned int count, unsigned int &sofar, char *D, ++bool FLV::Tag::MemReadUntil(char *buffer, unsigned int count, unsigned int &sofar, const char *D, + unsigned int S, unsigned int &P){ + if (sofar >= count){return true;} + int r = 0; +@@ -646,7 +646,7 @@ bool FLV::Tag::MemReadUntil(char *buffer, unsigned int count, unsigned int &sofa + /// location of the data buffer. \param S The size of the data buffer. \param P The current position + /// in the data buffer. Will be updated to reflect new position. \return True if a whole tag is + /// succesfully read, false otherwise. +-bool FLV::Tag::MemLoader(char *D, unsigned int S, unsigned int &P){ ++bool FLV::Tag::MemLoader(const char *D, unsigned int S, unsigned int &P){ + if (len < 15){len = 15;} + if (!checkBufferSize()){return false;} + if (done){ +diff --git a/lib/flv_tag.h b/lib/flv_tag.h +index ca27f472..10f616b9 100644 +--- a/lib/flv_tag.h ++++ b/lib/flv_tag.h +@@ -22,7 +22,7 @@ namespace FLV{ + + // functions + bool check_header(char *header); ///< Checks a FLV Header for validness. +- bool is_header(char *header); ///< Checks the first 3 bytes for the string "FLV". ++ bool is_header(const char *header); ///< Checks the first 3 bytes for the string "FLV". + + /// Helper function that can quickly skip through a file looking for a particular tag type + bool seekToTagType(FILE *f, uint8_t type); +@@ -55,7 +55,7 @@ namespace FLV{ + bool DTSCMetaInit(const DTSC::Meta &M, std::set &selTracks); + void toMeta(DTSC::Meta &meta, AMF::Object &amf_storage); + void toMeta(DTSC::Meta &meta, AMF::Object &amf_storage, size_t &reTrack, const std::map &targetParams); +- bool MemLoader(char *D, unsigned int S, unsigned int &P); ++ bool MemLoader(const char *D, unsigned int S, unsigned int &P); + bool FileLoader(FILE *f); + unsigned int getTrackID(); + char *getData(); +@@ -68,7 +68,7 @@ namespace FLV{ + void setLen(); + bool checkBufferSize(); + // loader helper functions +- bool MemReadUntil(char *buffer, unsigned int count, unsigned int &sofar, char *D, ++ bool MemReadUntil(char *buffer, unsigned int count, unsigned int &sofar, const char *D, + unsigned int S, unsigned int &P); + bool FileReadUntil(char *buffer, unsigned int count, unsigned int &sofar, FILE *f); + }; +-- +2.25.1 + + +From 3e85da2afd1b3d36290763eebf84dd66ad08636c Mon Sep 17 00:00:00 2001 +From: Thulinma +Date: Mon, 19 Sep 2022 17:26:15 +0200 +Subject: [PATCH 25/38] Match libav's RTMP receive window rollover behaviour + +--- + lib/rtmpchunks.cpp | 5 +++++ + src/output/output_rtmp.cpp | 2 +- + 2 files changed, 6 insertions(+), 1 deletion(-) + +diff --git a/lib/rtmpchunks.cpp b/lib/rtmpchunks.cpp +index e7c151f0..5f9d33f0 100644 +--- a/lib/rtmpchunks.cpp ++++ b/lib/rtmpchunks.cpp +@@ -475,6 +475,11 @@ bool RTMPStream::Chunk::Parse(Socket::Buffer &buffer){ + } + lastrecv[cs_id] = *this; + RTMPStream::rec_cnt += i + real_len; ++ if (RTMPStream::rec_cnt >= 0xf0000000){ ++ INFO_MSG("Resetting receive window due to impending rollover"); ++ RTMPStream::rec_cnt -= 0xf0000000; ++ RTMPStream::rec_window_at = 0; ++ } + if (len_left == 0){ + return true; + }else{ +diff --git a/src/output/output_rtmp.cpp b/src/output/output_rtmp.cpp +index 16a400d0..23194f4a 100644 +--- a/src/output/output_rtmp.cpp ++++ b/src/output/output_rtmp.cpp +@@ -1599,7 +1599,7 @@ namespace Mist{ + while (next.Parse(inputBuffer)){ + + // send ACK if we received a whole window +- if ((RTMPStream::rec_cnt - RTMPStream::rec_window_at > RTMPStream::rec_window_size) || Util::bootSecs() > lastAck+15){ ++ if ((RTMPStream::rec_cnt - RTMPStream::rec_window_at > RTMPStream::rec_window_size / 4) || Util::bootSecs() > lastAck+15){ + lastAck = Util::bootSecs(); + RTMPStream::rec_window_at = RTMPStream::rec_cnt; + myConn.SendNow(RTMPStream::SendCTL(3, RTMPStream::rec_cnt)); // send ack (msg 3) +-- +2.25.1 + + +From 074e7570284a094c12b00b88aa5cdf4a8f7fe5e4 Mon Sep 17 00:00:00 2001 +From: Ramkoemar +Date: Mon, 18 Oct 2021 14:29:13 +0200 +Subject: [PATCH 26/38] Sessions rework + +--- + CMakeLists.txt | 11 + + lib/comms.cpp | 428 +++++--- + lib/comms.h | 50 +- + lib/defines.h | 7 +- + src/controller/controller.cpp | 1 - + src/controller/controller_api.cpp | 1 + + src/controller/controller_statistics.cpp | 1185 ++++++++-------------- + src/controller/controller_statistics.h | 61 +- + src/controller/controller_storage.cpp | 22 +- + src/input/input.cpp | 12 +- + src/input/input.h | 2 +- + src/input/input_rtsp.cpp | 5 +- + src/input/input_sdp.cpp | 5 +- + src/input/input_ts.cpp | 5 +- + src/input/input_tssrt.cpp | 2 +- + src/input/input_tssrt.h | 2 +- + src/output/output.cpp | 125 +-- + src/output/output.h | 9 +- + src/output/output_cmaf.cpp | 2 +- + src/output/output_cmaf.h | 3 +- + src/output/output_http.cpp | 28 +- + src/output/output_tssrt.cpp | 2 +- + src/output/output_tssrt.h | 2 +- + src/output/output_webrtc.cpp | 2 +- + src/output/output_webrtc.h | 2 +- + src/process/process_exec.cpp | 4 +- + src/session.cpp | 367 +++++++ + 27 files changed, 1192 insertions(+), 1153 deletions(-) + create mode 100644 src/session.cpp + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 0e9bd644..6af16808 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -592,6 +592,17 @@ makeOutput(RTSP rtsp)#LTS + makeOutput(WAV wav)#LTS + makeOutput(SDP sdp http) + ++add_executable(MistSession ++ ${BINARY_DIR}/mist/.headers ++ src/session.cpp ++) ++install( ++ TARGETS MistSession ++ DESTINATION bin ++) ++target_link_libraries(MistSession mist) ++ ++ + add_executable(MistProcFFMPEG + ${BINARY_DIR}/mist/.headers + src/process/process_ffmpeg.cpp +diff --git a/lib/comms.cpp b/lib/comms.cpp +index 108c686a..85f1e6d7 100644 +--- a/lib/comms.cpp ++++ b/lib/comms.cpp +@@ -7,6 +7,7 @@ + #include "timing.h" + #include + #include ++#include "config.h" + + namespace Comms{ + Comms::Comms(){ +@@ -141,22 +142,197 @@ namespace Comms{ + } + } + +- Statistics::Statistics() : Comms(){sem.open(SEM_STATISTICS, O_CREAT | O_RDWR, ACCESSPERMS, 1);} ++ Sessions::Sessions() : Connections(){sem.open(SEM_STATISTICS, O_CREAT | O_RDWR, ACCESSPERMS, 1);} + +- void Statistics::unload(){ +- if (index != INVALID_RECORD_INDEX){ +- setStatus(COMM_STATUS_DISCONNECT | getStatus()); ++ void Sessions::reload(bool _master, bool reIssue){ ++ Comms::reload(COMMS_STATISTICS, COMMS_STATISTICS_INITSIZE, _master, reIssue); ++ } ++ ++ std::string Sessions::getSessId() const{return sessId.string(index);} ++ std::string Sessions::getSessId(size_t idx) const{return (master ? sessId.string(idx) : 0);} ++ void Sessions::setSessId(std::string _sid){sessId.set(_sid, index);} ++ void Sessions::setSessId(std::string _sid, size_t idx){ ++ if (!master){return;} ++ sessId.set(_sid, idx); ++ } ++ ++ bool Sessions::sessIdExists(std::string _sid){ ++ for (size_t i = 0; i < recordCount(); i++){ ++ if (getStatus(i) == COMM_STATUS_INVALID || (getStatus(i) & COMM_STATUS_DISCONNECT)){continue;} ++ if (getSessId(i) == _sid){ ++ if (Util::Procs::isRunning(getPid(i))){ ++ return true; ++ } ++ } + } +- index = INVALID_RECORD_INDEX; ++ return false; + } + +- void Statistics::reload(bool _master, bool reIssue){ +- Comms::reload(COMMS_STATISTICS, COMMS_STATISTICS_INITSIZE, _master, reIssue); ++ void Sessions::addFields(){ ++ Connections::addFields(); ++ dataAccX.addField("sessid", RAX_STRING, 80); + } + +- void Statistics::addFields(){ ++ void Sessions::nullFields(){ ++ Connections::nullFields(); ++ setSessId(""); ++ } ++ ++ void Sessions::fieldAccess(){ ++ Connections::fieldAccess(); ++ sessId = dataAccX.getFieldAccX("sessid"); ++ } ++ ++ Users::Users() : Comms(){} ++ ++ Users::Users(const Users &rhs) : Comms(){ ++ if (rhs){ ++ reload(rhs.streamName, (size_t)rhs.getTrack()); ++ if (*this){ ++ setKeyNum(rhs.getKeyNum()); ++ setTrack(rhs.getTrack()); ++ } ++ } ++ } ++ ++ void Users::reload(const std::string &_streamName, bool _master, bool reIssue){ ++ streamName = _streamName; ++ ++ char semName[NAME_BUFFER_SIZE]; ++ snprintf(semName, NAME_BUFFER_SIZE, SEM_USERS, streamName.c_str()); ++ sem.open(semName, O_CREAT | O_RDWR, ACCESSPERMS, 1); ++ ++ char userPageName[NAME_BUFFER_SIZE]; ++ snprintf(userPageName, NAME_BUFFER_SIZE, COMMS_USERS, streamName.c_str()); ++ ++ Comms::reload(userPageName, COMMS_USERS_INITSIZE, _master, reIssue); ++ } ++ ++ void Users::addFields(){ ++ Comms::addFields(); ++ dataAccX.addField("track", RAX_64UINT); ++ dataAccX.addField("keynum", RAX_64UINT); ++ } ++ ++ void Users::nullFields(){ ++ Comms::nullFields(); ++ setTrack(0); ++ setKeyNum(0); ++ } ++ ++ void Users::fieldAccess(){ ++ Comms::fieldAccess(); ++ track = dataAccX.getFieldAccX("track"); ++ keyNum = dataAccX.getFieldAccX("keynum"); ++ } ++ ++ void Users::reload(const std::string &_streamName, size_t idx, uint8_t initialState){ ++ reload(_streamName); ++ if (dataPage){ ++ setTrack(idx); ++ setKeyNum(0); ++ setStatus(initialState); ++ } ++ } ++ ++ uint32_t Users::getTrack() const{return track.uint(index);} ++ uint32_t Users::getTrack(size_t idx) const{return (master ? track.uint(idx) : 0);} ++ void Users::setTrack(uint32_t _track){track.set(_track, index);} ++ void Users::setTrack(uint32_t _track, size_t idx){ ++ if (!master){return;} ++ track.set(_track, idx); ++ } ++ ++ size_t Users::getKeyNum() const{return keyNum.uint(index);} ++ size_t Users::getKeyNum(size_t idx) const{return (master ? keyNum.uint(idx) : 0);} ++ void Users::setKeyNum(size_t _keyNum){keyNum.set(_keyNum, index);} ++ void Users::setKeyNum(size_t _keyNum, size_t idx){ ++ if (!master){return;} ++ keyNum.set(_keyNum, idx); ++ } ++ ++ /// \brief Claims a spot on the connections page for the input/output which calls this function ++ /// Starts the MistSession binary for each session, which handles the statistics ++ /// and the USER_NEW and USER_END triggers ++ /// \param streamName: Name of the stream the input is providing or an output is making available to viewers ++ /// \param ip: IP address of the viewer which wants to access streamName. For inputs this value can be set to any value ++ /// \param sid: Session ID given by the player or randomly generated ++ /// \param protocol: Protocol currently in use for this connection ++ /// \param sessionMode: Determines how a viewer session is defined: ++ // If set to 0, all connections with the same viewer IP and stream name are bundled. ++ // If set to 1, all connections with the same viewer IP and player ID are bundled. ++ // If set to 2, all connections with the same player ID and stream name are bundled. ++ // If set to 3, all connections with the same viewer IP, player ID and stream name are bundled. ++ /// \param _master: If True, we are reading from this page. If False, we are writing (to our entry) on this page ++ /// \param reIssue: If True, claim a new entry on this page ++ void Connections::reload(std::string streamName, std::string ip, std::string sid, std::string protocol, std::string reqUrl, uint64_t sessionMode, bool _master, bool reIssue){ ++ if (sessionMode == 0xFFFFFFFFFFFFFFFFull){ ++ FAIL_MSG("The session mode was not initialised properly. Assuming default behaviour of bundling by viewer IP, stream name and player id"); ++ sessionMode = SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID; ++ } ++ // Generate a unique session ID for each viewer, input or output ++ sessionId = generateSession(streamName, ip, sid, protocol, sessionMode); ++ if (protocol.size() >= 6 && protocol.substr(0, 6) == "INPUT:"){ ++ sessionId = "I" + sessionId; ++ }else if (protocol.size() >= 7 && protocol.substr(0, 7) == "OUTPUT:"){ ++ sessionId = "O" + sessionId; ++ } ++ char userPageName[NAME_BUFFER_SIZE]; ++ snprintf(userPageName, NAME_BUFFER_SIZE, COMMS_SESSIONS, sessionId.c_str()); ++ // Check if the page exists, if not, spawn new session process ++ if (!_master){ ++ dataPage.init(userPageName, 0, false, false); ++ if (!dataPage){ ++ pid_t thisPid; ++ std::deque args; ++ args.push_back(Util::getMyPath() + "MistSession"); ++ args.push_back(sessionId); ++ args.push_back("--sessionmode"); ++ args.push_back(JSON::Value(sessionMode).asString()); ++ args.push_back("--streamname"); ++ args.push_back(streamName); ++ args.push_back("--ip"); ++ args.push_back(ip); ++ args.push_back("--sid"); ++ args.push_back(sid); ++ args.push_back("--protocol"); ++ args.push_back(protocol); ++ args.push_back("--requrl"); ++ args.push_back(reqUrl); ++ int err = fileno(stderr); ++ thisPid = Util::Procs::StartPiped(args, 0, 0, &err); ++ Util::Procs::forget(thisPid); ++ HIGH_MSG("Spawned new session executeable (pid %u) for sessionId '%s', corresponding to host %s and stream %s", thisPid, sessionId.c_str(), ip.c_str(), streamName.c_str()); ++ } ++ } ++ // Open SEM_SESSION ++ if(!sem){ ++ char semName[NAME_BUFFER_SIZE]; ++ snprintf(semName, NAME_BUFFER_SIZE, SEM_SESSION, sessionId.c_str()); ++ sem.open(semName, O_RDWR, ACCESSPERMS, 1); ++ } ++ Comms::reload(userPageName, COMMS_SESSIONS_INITSIZE, _master, reIssue); ++ VERYHIGH_MSG("Reloading connection. Claimed record %lu", index); ++ } ++ ++ /// \brief Marks the data page as closed, so that we longer write any new data to is ++ void Connections::setExit(){ ++ if (!master){return;} ++ dataAccX.setExit(); ++ } ++ ++ bool Connections::getExit(){ ++ return dataAccX.isExit(); ++ } ++ ++ void Connections::unload(){ ++ if (index != INVALID_RECORD_INDEX){ ++ setStatus(COMM_STATUS_DISCONNECT | getStatus()); ++ } ++ index = INVALID_RECORD_INDEX; ++ } ++ void Connections::addFields(){ + Comms::addFields(); +- dataAccX.addField("sync", RAX_UINT); + dataAccX.addField("now", RAX_64UINT); + dataAccX.addField("time", RAX_64UINT); + dataAccX.addField("lastsecond", RAX_64UINT); +@@ -165,15 +341,15 @@ namespace Comms{ + dataAccX.addField("host", RAX_RAW, 16); + dataAccX.addField("stream", RAX_STRING, 100); + dataAccX.addField("connector", RAX_STRING, 20); +- dataAccX.addField("crc", RAX_32UINT); ++ dataAccX.addField("tags", RAX_STRING, 512); + dataAccX.addField("pktcount", RAX_64UINT); + dataAccX.addField("pktloss", RAX_64UINT); + dataAccX.addField("pktretrans", RAX_64UINT); + } + +- void Statistics::nullFields(){ ++ void Connections::nullFields(){ + Comms::nullFields(); +- setCRC(0); ++ setTags(""); + setConnector(""); + setStream(""); + setHost(""); +@@ -182,15 +358,13 @@ namespace Comms{ + setLastSecond(0); + setTime(0); + setNow(0); +- setSync(0); + setPacketCount(0); + setPacketLostCount(0); + setPacketRetransmitCount(0); + } + +- void Statistics::fieldAccess(){ ++ void Connections::fieldAccess(){ + Comms::fieldAccess(); +- sync = dataAccX.getFieldAccX("sync"); + now = dataAccX.getFieldAccX("now"); + time = dataAccX.getFieldAccX("time"); + lastSecond = dataAccX.getFieldAccX("lastsecond"); +@@ -199,209 +373,159 @@ namespace Comms{ + host = dataAccX.getFieldAccX("host"); + stream = dataAccX.getFieldAccX("stream"); + connector = dataAccX.getFieldAccX("connector"); +- crc = dataAccX.getFieldAccX("crc"); ++ tags = dataAccX.getFieldAccX("tags"); + pktcount = dataAccX.getFieldAccX("pktcount"); + pktloss = dataAccX.getFieldAccX("pktloss"); + pktretrans = dataAccX.getFieldAccX("pktretrans"); + } + +- uint8_t Statistics::getSync() const{return sync.uint(index);} +- uint8_t Statistics::getSync(size_t idx) const{return (master ? sync.uint(idx) : 0);} +- void Statistics::setSync(uint8_t _sync){sync.set(_sync, index);} +- void Statistics::setSync(uint8_t _sync, size_t idx){ +- if (!master){return;} +- sync.set(_sync, idx); +- } +- +- uint64_t Statistics::getNow() const{return now.uint(index);} +- uint64_t Statistics::getNow(size_t idx) const{return (master ? now.uint(idx) : 0);} +- void Statistics::setNow(uint64_t _now){now.set(_now, index);} +- void Statistics::setNow(uint64_t _now, size_t idx){ ++ uint64_t Connections::getNow() const{return now.uint(index);} ++ uint64_t Connections::getNow(size_t idx) const{return (master ? now.uint(idx) : 0);} ++ void Connections::setNow(uint64_t _now){now.set(_now, index);} ++ void Connections::setNow(uint64_t _now, size_t idx){ + if (!master){return;} + now.set(_now, idx); + } + +- uint64_t Statistics::getTime() const{return time.uint(index);} +- uint64_t Statistics::getTime(size_t idx) const{return (master ? time.uint(idx) : 0);} +- void Statistics::setTime(uint64_t _time){time.set(_time, index);} +- void Statistics::setTime(uint64_t _time, size_t idx){ ++ uint64_t Connections::getTime() const{return time.uint(index);} ++ uint64_t Connections::getTime(size_t idx) const{return (master ? time.uint(idx) : 0);} ++ void Connections::setTime(uint64_t _time){time.set(_time, index);} ++ void Connections::setTime(uint64_t _time, size_t idx){ + if (!master){return;} + time.set(_time, idx); + } + +- uint64_t Statistics::getLastSecond() const{return lastSecond.uint(index);} +- uint64_t Statistics::getLastSecond(size_t idx) const{ ++ uint64_t Connections::getLastSecond() const{return lastSecond.uint(index);} ++ uint64_t Connections::getLastSecond(size_t idx) const{ + return (master ? lastSecond.uint(idx) : 0); + } +- void Statistics::setLastSecond(uint64_t _lastSecond){lastSecond.set(_lastSecond, index);} +- void Statistics::setLastSecond(uint64_t _lastSecond, size_t idx){ ++ void Connections::setLastSecond(uint64_t _lastSecond){lastSecond.set(_lastSecond, index);} ++ void Connections::setLastSecond(uint64_t _lastSecond, size_t idx){ + if (!master){return;} + lastSecond.set(_lastSecond, idx); + } + +- uint64_t Statistics::getDown() const{return down.uint(index);} +- uint64_t Statistics::getDown(size_t idx) const{return (master ? down.uint(idx) : 0);} +- void Statistics::setDown(uint64_t _down){down.set(_down, index);} +- void Statistics::setDown(uint64_t _down, size_t idx){ ++ uint64_t Connections::getDown() const{return down.uint(index);} ++ uint64_t Connections::getDown(size_t idx) const{return (master ? down.uint(idx) : 0);} ++ void Connections::setDown(uint64_t _down){down.set(_down, index);} ++ void Connections::setDown(uint64_t _down, size_t idx){ + if (!master){return;} + down.set(_down, idx); + } + +- uint64_t Statistics::getUp() const{return up.uint(index);} +- uint64_t Statistics::getUp(size_t idx) const{return (master ? up.uint(idx) : 0);} +- void Statistics::setUp(uint64_t _up){up.set(_up, index);} +- void Statistics::setUp(uint64_t _up, size_t idx){ ++ uint64_t Connections::getUp() const{return up.uint(index);} ++ uint64_t Connections::getUp(size_t idx) const{return (master ? up.uint(idx) : 0);} ++ void Connections::setUp(uint64_t _up){up.set(_up, index);} ++ void Connections::setUp(uint64_t _up, size_t idx){ + if (!master){return;} + up.set(_up, idx); + } + +- std::string Statistics::getHost() const{return std::string(host.ptr(index), 16);} +- std::string Statistics::getHost(size_t idx) const{ ++ std::string Connections::getHost() const{return std::string(host.ptr(index), 16);} ++ std::string Connections::getHost(size_t idx) const{ + if (!master){return std::string((size_t)16, (char)'\000');} + return std::string(host.ptr(idx), 16); + } +- void Statistics::setHost(std::string _host){host.set(_host, index);} +- void Statistics::setHost(std::string _host, size_t idx){ ++ void Connections::setHost(std::string _host){host.set(_host, index);} ++ void Connections::setHost(std::string _host, size_t idx){ + if (!master){return;} + host.set(_host, idx); + } + +- std::string Statistics::getStream() const{return stream.string(index);} +- std::string Statistics::getStream(size_t idx) const{return (master ? stream.string(idx) : "");} +- void Statistics::setStream(std::string _stream){stream.set(_stream, index);} +- void Statistics::setStream(std::string _stream, size_t idx){ ++ std::string Connections::getStream() const{return stream.string(index);} ++ std::string Connections::getStream(size_t idx) const{return (master ? stream.string(idx) : "");} ++ void Connections::setStream(std::string _stream){stream.set(_stream, index);} ++ void Connections::setStream(std::string _stream, size_t idx){ + if (!master){return;} + stream.set(_stream, idx); + } + +- std::string Statistics::getConnector() const{return connector.string(index);} +- std::string Statistics::getConnector(size_t idx) const{ ++ std::string Connections::getConnector() const{return connector.string(index);} ++ std::string Connections::getConnector(size_t idx) const{ + return (master ? connector.string(idx) : ""); + } +- void Statistics::setConnector(std::string _connector){connector.set(_connector, index);} +- void Statistics::setConnector(std::string _connector, size_t idx){ ++ void Connections::setConnector(std::string _connector){connector.set(_connector, index);} ++ void Connections::setConnector(std::string _connector, size_t idx){ + if (!master){return;} + connector.set(_connector, idx); + } + +- uint32_t Statistics::getCRC() const{return crc.uint(index);} +- uint32_t Statistics::getCRC(size_t idx) const{return (master ? crc.uint(idx) : 0);} +- void Statistics::setCRC(uint32_t _crc){crc.set(_crc, index);} +- void Statistics::setCRC(uint32_t _crc, size_t idx){ ++ bool Connections::hasConnector(size_t idx, std::string protocol){ ++ std::stringstream sstream(connector.string(idx)); ++ std::string _conn; ++ while (std::getline(sstream, _conn, ',')){ ++ if (_conn == protocol){ ++ return true; ++ } ++ } ++ return false; ++ } ++ ++ std::string Connections::getTags() const{return tags.string(index);} ++ std::string Connections::getTags(size_t idx) const{return (master ? tags.string(idx) : 0);} ++ void Connections::setTags(std::string _sid){tags.set(_sid, index);} ++ void Connections::setTags(std::string _sid, size_t idx){ + if (!master){return;} +- crc.set(_crc, idx); ++ tags.set(_sid, idx); + } + +- uint64_t Statistics::getPacketCount() const{return pktcount.uint(index);} +- uint64_t Statistics::getPacketCount(size_t idx) const{ ++ uint64_t Connections::getPacketCount() const{return pktcount.uint(index);} ++ uint64_t Connections::getPacketCount(size_t idx) const{ + return (master ? pktcount.uint(idx) : 0); + } +- void Statistics::setPacketCount(uint64_t _count){pktcount.set(_count, index);} +- void Statistics::setPacketCount(uint64_t _count, size_t idx){ ++ void Connections::setPacketCount(uint64_t _count){pktcount.set(_count, index);} ++ void Connections::setPacketCount(uint64_t _count, size_t idx){ + if (!master){return;} + pktcount.set(_count, idx); + } + +- uint64_t Statistics::getPacketLostCount() const{return pktloss.uint(index);} +- uint64_t Statistics::getPacketLostCount(size_t idx) const{ ++ uint64_t Connections::getPacketLostCount() const{return pktloss.uint(index);} ++ uint64_t Connections::getPacketLostCount(size_t idx) const{ + return (master ? pktloss.uint(idx) : 0); + } +- void Statistics::setPacketLostCount(uint64_t _lost){pktloss.set(_lost, index);} +- void Statistics::setPacketLostCount(uint64_t _lost, size_t idx){ ++ void Connections::setPacketLostCount(uint64_t _lost){pktloss.set(_lost, index);} ++ void Connections::setPacketLostCount(uint64_t _lost, size_t idx){ + if (!master){return;} + pktloss.set(_lost, idx); + } + +- uint64_t Statistics::getPacketRetransmitCount() const{return pktretrans.uint(index);} +- uint64_t Statistics::getPacketRetransmitCount(size_t idx) const{ ++ uint64_t Connections::getPacketRetransmitCount() const{return pktretrans.uint(index);} ++ uint64_t Connections::getPacketRetransmitCount(size_t idx) const{ + return (master ? pktretrans.uint(idx) : 0); + } +- void Statistics::setPacketRetransmitCount(uint64_t _retrans){pktretrans.set(_retrans, index);} +- void Statistics::setPacketRetransmitCount(uint64_t _retrans, size_t idx){ ++ void Connections::setPacketRetransmitCount(uint64_t _retrans){pktretrans.set(_retrans, index);} ++ void Connections::setPacketRetransmitCount(uint64_t _retrans, size_t idx){ + if (!master){return;} + pktretrans.set(_retrans, idx); + } + +- std::string Statistics::getSessId() const{return getSessId(index);} +- +- std::string Statistics::getSessId(size_t idx) const{ +- char res[140]; +- memset(res, 0, 140); +- std::string tmp = host.string(idx); +- memcpy(res, tmp.c_str(), (tmp.size() > 16 ? 16 : tmp.size())); +- tmp = stream.string(idx); +- memcpy(res + 16, tmp.c_str(), (tmp.size() > 100 ? 100 : tmp.size())); +- tmp = connector.string(idx); +- memcpy(res + 116, tmp.c_str(), (tmp.size() > 20 ? 20 : tmp.size())); +- Bit::htobl(res + 136, crc.uint(idx)); +- return Secure::md5(res, 140); +- } +- +- Users::Users() : Comms(){} +- +- Users::Users(const Users &rhs) : Comms(){ +- if (rhs){ +- reload(rhs.streamName, (size_t)rhs.getTrack()); +- if (*this){ +- setKeyNum(rhs.getKeyNum()); +- setTrack(rhs.getTrack()); +- } ++ /// \brief Generates a session ID which is unique per viewer ++ /// \return generated session ID as string ++ std::string Connections::generateSession(std::string streamName, std::string ip, std::string sid, std::string connector, uint64_t sessionMode){ ++ std::string concat; ++ // First bit defines whether to include stream name ++ if (sessionMode > 7){ ++ concat += streamName; ++ sessionMode -= 8; + } +- } +- +- void Users::reload(const std::string &_streamName, bool _master, bool reIssue){ +- streamName = _streamName; +- +- char semName[NAME_BUFFER_SIZE]; +- snprintf(semName, NAME_BUFFER_SIZE, SEM_USERS, streamName.c_str()); +- sem.open(semName, O_CREAT | O_RDWR, ACCESSPERMS, 1); +- +- char userPageName[NAME_BUFFER_SIZE]; +- snprintf(userPageName, NAME_BUFFER_SIZE, COMMS_USERS, streamName.c_str()); +- +- Comms::reload(userPageName, COMMS_USERS_INITSIZE, _master, reIssue); +- } +- +- void Users::addFields(){ +- Comms::addFields(); +- dataAccX.addField("track", RAX_64UINT); +- dataAccX.addField("keynum", RAX_64UINT); +- } +- +- void Users::nullFields(){ +- Comms::nullFields(); +- setTrack(0); +- setKeyNum(0); +- } +- +- void Users::fieldAccess(){ +- Comms::fieldAccess(); +- track = dataAccX.getFieldAccX("track"); +- keyNum = dataAccX.getFieldAccX("keynum"); +- } +- +- void Users::reload(const std::string &_streamName, size_t idx, uint8_t initialState){ +- reload(_streamName); +- if (dataPage){ +- setTrack(idx); +- setKeyNum(0); +- setStatus(initialState); ++ // Second bit defines whether to include viewer ip ++ if (sessionMode > 3){ ++ concat += ip; ++ sessionMode -= 4; + } +- } +- +- uint32_t Users::getTrack() const{return track.uint(index);} +- uint32_t Users::getTrack(size_t idx) const{return (master ? track.uint(idx) : 0);} +- void Users::setTrack(uint32_t _track){track.set(_track, index);} +- void Users::setTrack(uint32_t _track, size_t idx){ +- if (!master){return;} +- track.set(_track, idx); +- } +- +- size_t Users::getKeyNum() const{return keyNum.uint(index);} +- size_t Users::getKeyNum(size_t idx) const{return (master ? keyNum.uint(idx) : 0);} +- void Users::setKeyNum(size_t _keyNum){keyNum.set(_keyNum, index);} +- void Users::setKeyNum(size_t _keyNum, size_t idx){ +- if (!master){return;} +- keyNum.set(_keyNum, idx); ++ // Third bit defines whether to include player ip ++ if (sessionMode > 1){ ++ concat += sid; ++ sessionMode -= 2; ++ } ++ // Fourth bit defines whether to include protocol ++ if (sessionMode == 1){ ++ concat += connector; ++ sessionMode = 0; ++ } ++ if (sessionMode > 0){ ++ WARN_MSG("Could not resolve session mode of value %lu", sessionMode); ++ } ++ return Secure::sha256(concat.c_str(), concat.length()); + } + }// namespace Comms +diff --git a/lib/comms.h b/lib/comms.h +index 9f459422..9a5c0ea9 100644 +--- a/lib/comms.h ++++ b/lib/comms.h +@@ -64,21 +64,21 @@ namespace Comms{ + Util::FieldAccX pid; + }; + +- class Statistics : public Comms{ ++ class Connections : public Comms{ + public: +- Statistics(); +- operator bool() const{return dataPage.mapped && (master || index != INVALID_RECORD_INDEX);} ++ void reload(std::string streamName, std::string ip, std::string sid, std::string protocol, std::string reqUrl, uint64_t sessionMode, bool _master = false, bool reIssue = false); + void unload(); +- void reload(bool _master = false, bool reIssue = false); ++ operator bool() const{return dataPage.mapped && (master || index != INVALID_RECORD_INDEX);} ++ std::string generateSession(std::string streamName, std::string ip, std::string sid, std::string connector, uint64_t sessionMode); ++ std::string sessionId; ++ ++ void setExit(); ++ bool getExit(); ++ + virtual void addFields(); + virtual void nullFields(); + virtual void fieldAccess(); + +- uint8_t getSync() const; +- uint8_t getSync(size_t idx) const; +- void setSync(uint8_t _sync); +- void setSync(uint8_t _sync, size_t idx); +- + uint64_t getNow() const; + uint64_t getNow(size_t idx) const; + void setNow(uint64_t _now); +@@ -118,11 +118,12 @@ namespace Comms{ + std::string getConnector(size_t idx) const; + void setConnector(std::string _connector); + void setConnector(std::string _connector, size_t idx); ++ bool hasConnector(size_t idx, std::string protocol); + +- uint32_t getCRC() const; +- uint32_t getCRC(size_t idx) const; +- void setCRC(uint32_t _crc); +- void setCRC(uint32_t _crc, size_t idx); ++ std::string getTags() const; ++ std::string getTags(size_t idx) const; ++ void setTags(std::string _sid); ++ void setTags(std::string _sid, size_t idx); + + uint64_t getPacketCount() const; + uint64_t getPacketCount(size_t idx) const; +@@ -139,11 +140,7 @@ namespace Comms{ + void setPacketRetransmitCount(uint64_t _retransmit); + void setPacketRetransmitCount(uint64_t _retransmit, size_t idx); + +- std::string getSessId() const; +- std::string getSessId(size_t index) const; +- +- private: +- Util::FieldAccX sync; ++ protected: + Util::FieldAccX now; + Util::FieldAccX time; + Util::FieldAccX lastSecond; +@@ -152,7 +149,8 @@ namespace Comms{ + Util::FieldAccX host; + Util::FieldAccX stream; + Util::FieldAccX connector; +- Util::FieldAccX crc; ++ Util::FieldAccX sessId; ++ Util::FieldAccX tags; + Util::FieldAccX pktcount; + Util::FieldAccX pktloss; + Util::FieldAccX pktretrans; +@@ -186,4 +184,18 @@ namespace Comms{ + Util::FieldAccX track; + Util::FieldAccX keyNum; + }; ++ ++ class Sessions : public Connections{ ++ public: ++ Sessions(); ++ void reload(bool _master = false, bool reIssue = false); ++ std::string getSessId() const; ++ std::string getSessId(size_t idx) const; ++ void setSessId(std::string _sid); ++ void setSessId(std::string _sid, size_t idx); ++ bool sessIdExists(std::string _sid); ++ virtual void addFields(); ++ virtual void nullFields(); ++ virtual void fieldAccess(); ++ }; + }// namespace Comms +diff --git a/lib/defines.h b/lib/defines.h +index 10c4b9b1..203b75db 100644 +--- a/lib/defines.h ++++ b/lib/defines.h +@@ -196,11 +196,14 @@ static inline void show_stackframe(){} + #define TRACK_PAGE_RECORDSIZE 36 + + #define COMMS_STATISTICS "MstStat" +-#define COMMS_STATISTICS_INITSIZE 8 * 1024 * 1024 ++#define COMMS_STATISTICS_INITSIZE 16 * 1024 * 1024 + + #define COMMS_USERS "MstUser%s" //%s stream name + #define COMMS_USERS_INITSIZE 512 * 1024 + ++#define COMMS_SESSIONS "MstSession%s" ++#define COMMS_SESSIONS_INITSIZE 8 * 1024 * 1024 ++ + #define SEM_STATISTICS "/MstStat" + #define SEM_USERS "/MstUser%s" //%s stream name + +@@ -226,7 +229,9 @@ static inline void show_stackframe(){} + #define SEM_LIVE "/MstLIVE%s" //%s stream name + #define SEM_INPUT "/MstInpt%s" //%s stream name + #define SEM_TRACKLIST "/MstTRKS%s" //%s stream name ++#define SEM_SESSION "MstSess%s" + #define SEM_SESSCACHE "/MstSessCacheLock" ++#define SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID 14 + #define SHM_CAPA "MstCapa" + #define SHM_PROTO "MstProt" + #define SHM_PROXY "MstProx" +diff --git a/src/controller/controller.cpp b/src/controller/controller.cpp +index a0ff2ee8..9e17ca6b 100644 +--- a/src/controller/controller.cpp ++++ b/src/controller/controller.cpp +@@ -306,7 +306,6 @@ int main_loop(int argc, char **argv){ + if (Controller::Storage["config"].isMember("accesslog")){ + Controller::conf.getOption("accesslog", true)[0u] = Controller::Storage["config"]["accesslog"]; + } +- Controller::maxConnsPerIP = Controller::conf.getInteger("maxconnsperip"); + Controller::Storage["config"]["prometheus"] = Controller::conf.getString("prometheus"); + Controller::Storage["config"]["accesslog"] = Controller::conf.getString("accesslog"); + Controller::normalizeTrustedProxies(Controller::Storage["config"]["trustedproxy"]); +diff --git a/src/controller/controller_api.cpp b/src/controller/controller_api.cpp +index 4d94bb11..02491065 100644 +--- a/src/controller/controller_api.cpp ++++ b/src/controller/controller_api.cpp +@@ -594,6 +594,7 @@ void Controller::handleAPICommands(JSON::Value &Request, JSON::Value &Response){ + out["prometheus"] = in["prometheus"]; + Controller::prometheus = out["prometheus"].asStringRef(); + } ++ if (in.isMember("sessionMode")){out["sessionMode"] = in["sessionMode"];} + if (in.isMember("defaultStream")){out["defaultStream"] = in["defaultStream"];} + if (in.isMember("location") && in["location"].isObject()){ + out["location"]["lat"] = in["location"]["lat"].asDouble(); +diff --git a/src/controller/controller_statistics.cpp b/src/controller/controller_statistics.cpp +index c5b7f643..6e51f4ee 100644 +--- a/src/controller/controller_statistics.cpp ++++ b/src/controller/controller_statistics.cpp +@@ -15,6 +15,7 @@ + #include + #include //for fstatvfs + #include ++#include + + #ifndef KILL_ON_EXIT + #define KILL_ON_EXIT false +@@ -30,7 +31,6 @@ + #define STAT_CLI_UP 64 + #define STAT_CLI_BPS_DOWN 128 + #define STAT_CLI_BPS_UP 256 +-#define STAT_CLI_CRC 512 + #define STAT_CLI_SESSID 1024 + #define STAT_CLI_PKTCOUNT 2048 + #define STAT_CLI_PKTLOST 4096 +@@ -46,59 +46,18 @@ + #define STAT_TOT_PERCRETRANS 64 + #define STAT_TOT_ALL 0xFF + +-#define COUNTABLE_BYTES 128 * 1024 +- +-std::map Controller::sessions; ///< list of sessions that have statistics data available +-std::map Controller::connToSession; ///< Map of socket IDs to session info. ++// Mapping of sessId -> session statistics ++std::map sessions; + + std::map Controller::triggerStats; ///< Holds prometheus stats for trigger executions + bool Controller::killOnExit = KILL_ON_EXIT; + tthread::mutex Controller::statsMutex; +-unsigned int Controller::maxConnsPerIP = 0; + uint64_t Controller::statDropoff = 0; + static uint64_t cpu_use = 0; + + char noBWCountMatches[1717]; + uint64_t bwLimit = 128 * 1024 * 1024; // gigabit default limit + +-/// Session cache shared memory page +-IPC::sharedPage *shmSessions = 0; +-/// Lock for the session cache shared memory page +-IPC::semaphore *cacheLock = 0; +- +-/// Convert bandwidth config into memory format +-void Controller::updateBandwidthConfig(){ +- size_t offset = 0; +- bwLimit = 128 * 1024 * 1024; // gigabit default limit +- memset(noBWCountMatches, 0, 1717); +- if (Storage.isMember("bandwidth")){ +- if (Storage["bandwidth"].isMember("limit")){bwLimit = Storage["bandwidth"]["limit"].asInt();} +- if (Storage["bandwidth"].isMember("exceptions")){ +- jsonForEach(Storage["bandwidth"]["exceptions"], j){ +- std::string newbins = Socket::getBinForms(j->asStringRef()); +- if (offset + newbins.size() < 1700){ +- memcpy(noBWCountMatches + offset, newbins.data(), newbins.size()); +- offset += newbins.size(); +- } +- } +- } +- } +- //Localhost is always excepted from counts +- { +- std::string newbins = Socket::getBinForms("::1"); +- if (offset + newbins.size() < 1700){ +- memcpy(noBWCountMatches + offset, newbins.data(), newbins.size()); +- offset += newbins.size(); +- } +- } +- { +- std::string newbins = Socket::getBinForms("127.0.0.1/8"); +- if (offset + newbins.size() < 1700){ +- memcpy(noBWCountMatches + offset, newbins.data(), newbins.size()); +- offset += newbins.size(); +- } +- } +-} + + // For server-wide totals. Local to this file only. + struct streamTotals{ +@@ -116,11 +75,30 @@ struct streamTotals{ + uint64_t packLoss; + uint64_t packRetrans; + }; ++ ++Comms::Sessions statComm; ++bool statCommActive = false; ++// Global server wide statistics ++static uint64_t servUpBytes = 0; ++static uint64_t servDownBytes = 0; ++static uint64_t servUpOtherBytes = 0; ++static uint64_t servDownOtherBytes = 0; ++static uint64_t servInputs = 0; ++static uint64_t servOutputs = 0; ++static uint64_t servViewers = 0; ++static uint64_t servSeconds = 0; ++static uint64_t servPackSent = 0; ++static uint64_t servPackLoss = 0; ++static uint64_t servPackRetrans = 0; ++// Total time watched for all sessions which are no longer active ++static uint64_t viewSecondsTotal = 0; ++// Mapping of streamName -> summary of stream-wide statistics + static std::map streamStats; + +-static void createEmptyStatsIfNeeded(const std::string & strm){ +- if (streamStats.count(strm)){return;} +- streamTotals & sT = streamStats[strm]; ++// If sessId does not exist yet in streamStats, create and init an entry for it ++static void createEmptyStatsIfNeeded(const std::string & sessId){ ++ if (streamStats.count(sessId)){return;} ++ streamTotals & sT = streamStats[sessId]; + sT.upBytes = 0; + sT.downBytes = 0; + sT.inputs = 0; +@@ -136,68 +114,38 @@ static void createEmptyStatsIfNeeded(const std::string & strm){ + sT.packRetrans = 0; + } + +- +-static uint64_t servUpBytes = 0; +-static uint64_t servDownBytes = 0; +-static uint64_t servUpOtherBytes = 0; +-static uint64_t servDownOtherBytes = 0; +-static uint64_t servInputs = 0; +-static uint64_t servOutputs = 0; +-static uint64_t servViewers = 0; +-static uint64_t servSeconds = 0; +-static uint64_t servPackSent = 0; +-static uint64_t servPackLoss = 0; +-static uint64_t servPackRetrans = 0; +- +-Controller::sessIndex::sessIndex(){ +- crc = 0; +-} +- +-/// Initializes a sessIndex from a statistics object + index, converting binary format IP addresses +-/// into strings. This extracts the host, stream name, connector and crc field, ignoring everything +-/// else. +-Controller::sessIndex::sessIndex(const Comms::Statistics &statComm, size_t id){ +- Socket::hostBytesToStr(statComm.getHost(id).data(), 16, host); +- streamName = statComm.getStream(id); +- connector = statComm.getConnector(id); +- crc = statComm.getCRC(id); +- ID = statComm.getSessId(id); +-} +- +-std::string Controller::sessIndex::toStr(){ +- std::stringstream s; +- s << ID << "(" << host << " " << crc << " " << streamName << " " << connector << ")"; +- return s.str(); +-} +- +-bool Controller::sessIndex::operator==(const Controller::sessIndex &b) const{ +- return (host == b.host && crc == b.crc && streamName == b.streamName && connector == b.connector); +-} +- +-bool Controller::sessIndex::operator!=(const Controller::sessIndex &b) const{ +- return !(*this == b); +-} +- +-bool Controller::sessIndex::operator>(const Controller::sessIndex &b) const{ +- return host > b.host || +- (host == b.host && +- (crc > b.crc || (crc == b.crc && (streamName > b.streamName || +- (streamName == b.streamName && connector > b.connector))))); +-} +- +-bool Controller::sessIndex::operator<(const Controller::sessIndex &b) const{ +- return host < b.host || +- (host == b.host && +- (crc < b.crc || (crc == b.crc && (streamName < b.streamName || +- (streamName == b.streamName && connector < b.connector))))); +-} +- +-bool Controller::sessIndex::operator<=(const Controller::sessIndex &b) const{ +- return !(*this > b); +-} +- +-bool Controller::sessIndex::operator>=(const Controller::sessIndex &b) const{ +- return !(*this < b); ++/// Convert bandwidth config into memory format ++void Controller::updateBandwidthConfig(){ ++ size_t offset = 0; ++ bwLimit = 128 * 1024 * 1024; // gigabit default limit ++ memset(noBWCountMatches, 0, 1717); ++ if (Storage.isMember("bandwidth")){ ++ if (Storage["bandwidth"].isMember("limit")){bwLimit = Storage["bandwidth"]["limit"].asInt();} ++ if (Storage["bandwidth"].isMember("exceptions")){ ++ jsonForEach(Storage["bandwidth"]["exceptions"], j){ ++ std::string newbins = Socket::getBinForms(j->asStringRef()); ++ if (offset + newbins.size() < 1700){ ++ memcpy(noBWCountMatches + offset, newbins.data(), newbins.size()); ++ offset += newbins.size(); ++ } ++ } ++ } ++ } ++ //Localhost is always excepted from counts ++ { ++ std::string newbins = Socket::getBinForms("::1"); ++ if (offset + newbins.size() < 1700){ ++ memcpy(noBWCountMatches + offset, newbins.data(), newbins.size()); ++ offset += newbins.size(); ++ } ++ } ++ { ++ std::string newbins = Socket::getBinForms("127.0.0.1/8"); ++ if (offset + newbins.size() < 1700){ ++ memcpy(noBWCountMatches + offset, newbins.data(), newbins.size()); ++ offset += newbins.size(); ++ } ++ } + } + + /// This function is ran whenever a stream becomes active. +@@ -211,9 +159,6 @@ void Controller::streamStopped(std::string stream){ + INFO_MSG("Stream %s became inactive", stream.c_str()); + } + +-Comms::Statistics statComm; +-bool statCommActive = false; +- + /// Invalidates all current sessions for the given streamname + /// Updates the session cache, afterwards. + void Controller::sessions_invalidate(const std::string &streamname){ +@@ -221,18 +166,17 @@ void Controller::sessions_invalidate(const std::string &streamname){ + FAIL_MSG("In shutdown procedure - cannot invalidate sessions."); + return; + } +- unsigned int invalidated = 0; + unsigned int sessCount = 0; +- tthread::lock_guard guard(statsMutex); +- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ +- if (it->first.streamName == streamname){ ++ // Find all matching streams in statComm ++ for (size_t i = 0; i < statComm.recordCount(); i++){ ++ if (statComm.getStatus(i) == COMM_STATUS_INVALID || (statComm.getStatus(i) & COMM_STATUS_DISCONNECT)){continue;} ++ if (statComm.getStream(i) == streamname){ + sessCount++; +- invalidated += it->second.invalidate(); ++ // Re-trigger USER_NEW trigger for this session ++ kill(statComm.getPid(i), SIGUSR1); + } + } +- Controller::writeSessionCache(); +- INFO_MSG("Invalidated %u connections in %u sessions for stream %s", invalidated, sessCount, +- streamname.c_str()); ++ INFO_MSG("Invalidated %u session(s) for stream %s", sessCount, streamname.c_str()); + } + + /// Shuts down all current sessions for the given streamname +@@ -256,18 +200,8 @@ void Controller::sessId_shutdown(const std::string &sessId){ + FAIL_MSG("In controller shutdown procedure - cannot shutdown sessions."); + return; + } +- unsigned int murdered = 0; +- unsigned int sessCount = 0; +- tthread::lock_guard guard(statsMutex); +- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ +- if (it->first.ID == sessId){ +- sessCount++; +- murdered += it->second.kill(); +- break; +- } +- } +- Controller::writeSessionCache(); +- INFO_MSG("Shut down %u connections in %u session(s) for ID %s", murdered, sessCount, sessId.c_str()); ++ killConnections(sessId); ++ INFO_MSG("Shut down session with session ID %s", sessId.c_str()); + } + + /// Tags the given session +@@ -277,8 +211,8 @@ void Controller::sessId_tag(const std::string &sessId, const std::string &tag){ + return; + } + tthread::lock_guard guard(statsMutex); +- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ +- if (it->first.ID == sessId){ ++ for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ ++ if (it->first == sessId){ + it->second.tags.insert(tag); + return; + } +@@ -295,17 +229,15 @@ void Controller::tag_shutdown(const std::string &tag){ + FAIL_MSG("In controller shutdown procedure - cannot shutdown sessions."); + return; + } +- unsigned int murdered = 0; + unsigned int sessCount = 0; + tthread::lock_guard guard(statsMutex); +- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ ++ for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ + if (it->second.tags.count(tag)){ + sessCount++; +- murdered += it->second.kill(); ++ killConnections(it->first); + } + } +- Controller::writeSessionCache(); +- INFO_MSG("Shut down %u connections in %u session(s) for tag %s", murdered, sessCount, tag.c_str()); ++ INFO_MSG("Shut down %u session(s) for tag %s", sessCount, tag.c_str()); + } + + /// Shuts down all current sessions for the given streamname +@@ -315,48 +247,23 @@ void Controller::sessions_shutdown(const std::string &streamname, const std::str + FAIL_MSG("In controller shutdown procedure - cannot shutdown sessions."); + return; + } +- unsigned int murdered = 0; + unsigned int sessCount = 0; + tthread::lock_guard guard(statsMutex); +- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ +- if ((!streamname.size() || it->first.streamName == streamname) && +- (!protocol.size() || it->first.connector == protocol)){ ++ // Find all matching streams in statComm and get their sessId ++ for (size_t i = 0; i < statComm.recordCount(); i++){ ++ if (statComm.getStatus(i) == COMM_STATUS_INVALID || (statComm.getStatus(i) & COMM_STATUS_DISCONNECT)){continue;} ++ if ((!streamname.size() || statComm.getStream(i) == streamname) && ++ (!protocol.size() || statComm.hasConnector(i, protocol))){ ++ uint32_t pid = statComm.getPid(i); + sessCount++; +- murdered += it->second.kill(); +- } +- } +- Controller::writeSessionCache(); +- INFO_MSG("Shut down %u connections in %u sessions for stream %s/%s", murdered, sessCount, +- streamname.c_str(), protocol.c_str()); +-} +- +-/// Writes the session cache to shared memory. +-/// Assumes the config mutex, stats mutex and session cache semaphore are already locked. +-/// Does nothing if the session cache could not be initialized on the first try +-/// Does no error checking after first open attempt (fails silently)! +-void Controller::writeSessionCache(){ +- uint32_t shmOffset = 0; +- if (shmSessions && shmSessions->mapped){ +- if (cacheLock){cacheLock->wait(16);} +- if (sessions.size()){ +- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ +- if (it->second.hasData()){ +- // store an entry in the shmSessions page, if it fits +- if (it->second.sync > 2 && shmOffset + SHM_SESSIONS_ITEM < SHM_SESSIONS_SIZE){ +- *((uint32_t *)(shmSessions->mapped + shmOffset)) = it->first.crc; +- strncpy(shmSessions->mapped + shmOffset + 4, it->first.streamName.c_str(), 100); +- strncpy(shmSessions->mapped + shmOffset + 104, it->first.connector.c_str(), 20); +- strncpy(shmSessions->mapped + shmOffset + 124, it->first.host.c_str(), 40); +- shmSessions->mapped[shmOffset + 164] = it->second.sync; +- shmOffset += SHM_SESSIONS_ITEM; +- } +- } ++ if (pid > 1){ ++ Util::Procs::Stop(pid); ++ INFO_MSG("Killing PID %" PRIu32, pid); + } + } +- // set a final shmSessions entry to all zeroes +- memset(shmSessions->mapped + shmOffset, 0, SHM_SESSIONS_ITEM); +- if (cacheLock){cacheLock->post(16);} + } ++ INFO_MSG("Shut down %u sessions for stream %s/%s", sessCount, ++ streamname.c_str(), protocol.c_str()); + } + + /// This function runs as a thread and roughly once per second retrieves +@@ -366,14 +273,6 @@ void Controller::SharedMemStats(void *config){ + HIGH_MSG("Starting stats thread"); + statComm.reload(true); + statCommActive = true; +- shmSessions = new IPC::sharedPage(SHM_SESSIONS, SHM_SESSIONS_SIZE, false, false); +- if (!shmSessions || !shmSessions->mapped){ +- if (shmSessions){delete shmSessions;} +- shmSessions = new IPC::sharedPage(SHM_SESSIONS, SHM_SESSIONS_SIZE, true); +- } +- cacheLock = new IPC::semaphore(SEM_SESSCACHE, O_CREAT | O_RDWR, ACCESSPERMS, 16); +- cacheLock->unlink(); +- cacheLock->open(SEM_SESSCACHE, O_CREAT | O_RDWR, ACCESSPERMS, 16); + std::set inactiveStreams; + Controller::initState(); + bool shiftWrites = true; +@@ -401,7 +300,6 @@ void Controller::SharedMemStats(void *config){ + } + } + { +- + tthread::lock_guard guard(Controller::configMutex); + tthread::lock_guard guard2(statsMutex); + // parse current users +@@ -429,29 +327,47 @@ void Controller::SharedMemStats(void *config){ + it->second.packRetrans = 0; + } + } +- // wipe old statistics ++ unsigned int tOut = Util::bootSecs() - STATS_DELAY; ++ unsigned int tIn = Util::bootSecs() - STATS_INPUT_DELAY; ++ if (streamStats.size()){ ++ for (std::map::iterator it = streamStats.begin(); ++ it != streamStats.end(); ++it){ ++ it->second.currViews = 0; ++ it->second.currIns = 0; ++ it->second.currOuts = 0; ++ } ++ } ++ // wipe old statistics and set session type counters + if (sessions.size()){ +- std::list mustWipe; ++ std::list mustWipe; + uint64_t cutOffPoint = Util::bootSecs() - STAT_CUTOFF; +- uint64_t disconnectPointIn = Util::bootSecs() - STATS_INPUT_DELAY; +- uint64_t disconnectPointOut = Util::bootSecs() - STATS_DELAY; +- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ +- uint64_t dPoint = it->second.getSessType() == SESS_INPUT ? disconnectPointIn : disconnectPointOut; +- if (it->second.sync == 100){ +- // Denied entries are connection-entry-wiped as soon as they become boring +- it->second.wipeOld(dPoint); +- }else{ +- // Normal entries are summarized after STAT_CUTOFF seconds +- it->second.wipeOld(cutOffPoint); +- } ++ for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ + // This part handles ending sessions, keeping them in cache for now +- if (it->second.isTracked() && !it->second.isConnected() && it->second.getEnd() < dPoint){ +- it->second.dropSession(it->first); +- } +- // This part handles wiping from the session cache +- if (!it->second.hasData()){ +- it->second.dropSession(it->first); // End the session, just in case it wasn't yet ++ if (it->second.getEnd() < cutOffPoint && it->second.newestDataPoint() < cutOffPoint){ ++ viewSecondsTotal += it->second.getConnTime(); + mustWipe.push_back(it->first); ++ // Don't count this session as a viewer ++ continue; ++ } ++ // Recount input, output and viewer type sessions ++ switch (it->second.getSessType()){ ++ case SESS_UNSET: break; ++ case SESS_VIEWER: ++ if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){ ++ streamStats[it->first].currViews++; ++ } ++ servSeconds += it->second.getConnTime(); ++ break; ++ case SESS_INPUT: ++ if (it->second.hasDataFor(tIn) && it->second.isViewerOn(tIn)){ ++ streamStats[it->first].currIns++; ++ } ++ break; ++ case SESS_OUTPUT: ++ if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){ ++ streamStats[it->first].currOuts++; ++ } ++ break; + } + } + while (mustWipe.size()){ +@@ -513,7 +429,6 @@ void Controller::SharedMemStats(void *config){ + shiftWrites = true; + } + /*LTS-START*/ +- Controller::writeSessionCache(); + Controller::checkServerLimits(); + /*LTS-END*/ + } +@@ -523,7 +438,6 @@ void Controller::SharedMemStats(void *config){ + HIGH_MSG("Stopping stats thread"); + if (Util::Config::is_restarting){ + statComm.setMaster(false); +- shmSessions->master = false; + }else{/*LTS-START*/ + if (Controller::killOnExit){ + WARN_MSG("Killing all connected clients to force full shutdown"); +@@ -532,10 +446,6 @@ void Controller::SharedMemStats(void *config){ + /*LTS-END*/ + } + Controller::deinitState(Util::Config::is_restarting); +- delete shmSessions; +- shmSessions = 0; +- delete cacheLock; +- cacheLock = 0; + } + + /// Gets a complete list of all streams currently in active state, with optional prefix matching +@@ -559,97 +469,53 @@ std::set Controller::getActiveStreams(const std::string &prefix){ + return ret; + } + +-/// Forces a re-sync of the session +-/// Assumes the session cache will be updated separately - may not work correctly if this is forgotten! +-uint32_t Controller::statSession::invalidate(){ +- uint32_t ret = 0; +- sync = 1; +- if (curConns.size() && statCommActive){ +- for (std::map::iterator jt = curConns.begin(); jt != curConns.end(); ++jt){ +- if (statComm.getStatus(jt->first) != COMM_STATUS_INVALID){ +- statComm.setSync(2, jt->first); +- ret++; +- } +- } +- } +- return ret; +-} +- +-/// Kills all active connections, sets the session state to denied (sync=100). ++/// Kills all connection of a given session + /// Assumes the session cache will be updated separately - may not work correctly if this is forgotten! +-uint32_t Controller::statSession::kill(){ +- uint32_t ret = 0; +- sync = 100; +- if (curConns.size() && statCommActive){ +- for (std::map::iterator jt = curConns.begin(); jt != curConns.end(); ++jt){ +- if (statComm.getStatus(jt->first) != COMM_STATUS_INVALID){ +- statComm.setSync(100, jt->first); +- uint32_t pid = statComm.getPid(jt->first); ++void Controller::killConnections(std::string sessId){ ++ if (statCommActive){ ++ // Find a matching stream in statComm with a matching sessID and kill it ++ for (size_t i = 0; i < statComm.recordCount(); i++){ ++ if (statComm.getStatus(i) == COMM_STATUS_INVALID || (statComm.getStatus(i) & COMM_STATUS_DISCONNECT)){continue;} ++ if (statComm.getSessId(i) == sessId){ ++ uint32_t pid = statComm.getPid(i); + if (pid > 1){ + Util::Procs::Stop(pid); + INFO_MSG("Killing PID %" PRIu32, pid); + } +- ret++; + } + } + } +- return ret; + } + + /// Updates the given active connection with new stats data. +-void Controller::statSession::update(uint64_t index, Comms::Statistics &statComm){ +- std::string myHost; +- Socket::hostBytesToStr(statComm.getHost(index).data(), 16, myHost); +- std::string myStream = statComm.getStream(index); +- std::string myConnector = statComm.getConnector(index); +- // update the sync byte: 0 = requesting fill, 2 = requesting refill, 1 = needs checking, > 2 = +- // state known (100=denied, 10=accepted) +- if (!statComm.getSync(index)){ +- sessIndex tmpidx(statComm, index); +- // if we have a maximum connection count per IP, enforce it +- if (maxConnsPerIP && !statComm.getSync(index)){ +- unsigned int currConns = 1; +- long long shortly = Util::bootSecs(); +- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ +- +- if (&it->second != this && it->first.host == myHost && +- (it->second.hasDataFor(shortly - STATS_DELAY) || it->second.hasDataFor(shortly) || +- it->second.hasDataFor(shortly - 1) || it->second.hasDataFor(shortly - 2) || +- it->second.hasDataFor(shortly - 3) || it->second.hasDataFor(shortly - 4) || +- it->second.hasDataFor(shortly - 5)) && +- ++currConns > maxConnsPerIP){ +- break; +- } +- } +- if (currConns > maxConnsPerIP){ +- WARN_MSG("Disconnecting session from %s: exceeds max connection count of %u", myHost.c_str(), maxConnsPerIP); +- statComm.setSync(100, index); +- } +- } +- if (statComm.getSync(index) != 100){ +- // only set the sync if this is the first connection in the list +- // we also catch the case that there are no connections, which is an error-state +- if (!sessions[tmpidx].curConns.size() || sessions[tmpidx].curConns.begin()->first == index){ +- MEDIUM_MSG("Requesting sync to %u for %s, %s, %s, %" PRIu32, sync, myStream.c_str(), +- myConnector.c_str(), myHost.c_str(), statComm.getCRC(index) & 0xFFFFFFFFu); +- statComm.setSync(sync, index); +- } +- // and, always set the sync if it is > 2 +- if (sync > 2){ +- MEDIUM_MSG("Setting sync to %u for %s, %s, %s, %" PRIu32, sync, myStream.c_str(), +- myConnector.c_str(), myHost.c_str(), statComm.getCRC(index) & 0xFFFFFFFFu); +- statComm.setSync(sync, index); +- } ++void Controller::statSession::update(uint64_t index, Comms::Sessions &statComm){ ++ if (host == ""){ ++ Socket::hostBytesToStr(statComm.getHost(index).data(), 16, host); ++ } ++ if (streamName == ""){ ++ streamName = statComm.getStream(index); ++ } ++ if (curConnector == ""){ ++ curConnector = statComm.getConnector(index); ++ } ++ if (sessId == ""){ ++ sessId = statComm.getSessId(index); ++ } ++ // Export tags to session ++ if (tags.size()){ ++ std::stringstream tagStream; ++ for (std::set::iterator it = tags.begin(); it != tags.end(); ++it){ ++ tagStream << "[" << *it << "]"; + } +- }else{ +- if (sync < 2 && statComm.getSync(index) > 2){sync = statComm.getSync(index);} ++ statComm.setTags(tagStream.str(), index); + } ++ + long long prevDown = getDown(); + long long prevUp = getUp(); + uint64_t prevPktSent = getPktCount(); + uint64_t prevPktLost = getPktLost(); + uint64_t prevPktRetrans = getPktRetransmit(); +- curConns[index].update(statComm, index); ++ curData.update(statComm, index); + // store timestamp of first received data, if older + if (firstSec > statComm.getNow(index)){firstSec = statComm.getNow(index);} + uint64_t secIncr = 0; +@@ -671,7 +537,7 @@ void Controller::statSession::update(uint64_t index, Comms::Statistics &statComm + uint64_t currPktRetrans = getPktRetransmit(); + if (currUp - prevUp < 0 || currDown - prevDown < 0){ + INFO_MSG("Negative data usage! %lldu/%lldd (u%lld->%lld) in %s over %s, #%" PRIu64, currUp - prevUp, +- currDown - prevDown, prevUp, currUp, myStream.c_str(), myConnector.c_str(), index); ++ currDown - prevDown, prevUp, currUp, streamName.c_str(), curConnector.c_str(), index); + }else{ + if (!noBWCount){ + size_t bwMatchOffset = 0; +@@ -701,56 +567,39 @@ void Controller::statSession::update(uint64_t index, Comms::Statistics &statComm + servPackRetrans += currPktRetrans - prevPktRetrans; + } + } +- if (currDown + currUp >= COUNTABLE_BYTES){ +- if (sessionType == SESS_UNSET){ +- if (myConnector.size() >= 5 && myConnector.substr(0, 5) == "INPUT"){ +- ++servInputs; +- createEmptyStatsIfNeeded(myStream); +- streamStats[myStream].inputs++; +- streamStats[myStream].currIns++; +- sessionType = SESS_INPUT; +- }else if (myConnector.size() >= 6 && myConnector.substr(0, 6) == "OUTPUT"){ +- ++servOutputs; +- createEmptyStatsIfNeeded(myStream); +- streamStats[myStream].outputs++; +- streamStats[myStream].currOuts++; +- sessionType = SESS_OUTPUT; +- }else{ +- ++servViewers; +- createEmptyStatsIfNeeded(myStream); +- streamStats[myStream].viewers++; +- streamStats[myStream].currViews++; +- sessionType = SESS_VIEWER; +- } ++ if (sessionType == SESS_UNSET){ ++ if (curConnector.size() >= 5 && curConnector.substr(0, 5) == "INPUT"){ ++ ++servInputs; ++ createEmptyStatsIfNeeded(streamName); ++ streamStats[streamName].inputs++; ++ streamStats[streamName].currIns++; ++ sessionType = SESS_INPUT; ++ }else if (curConnector.size() >= 6 && curConnector.substr(0, 6) == "OUTPUT"){ ++ ++servOutputs; ++ createEmptyStatsIfNeeded(streamName); ++ streamStats[streamName].outputs++; ++ streamStats[streamName].currOuts++; ++ sessionType = SESS_OUTPUT; ++ }else{ ++ ++servViewers; ++ createEmptyStatsIfNeeded(streamName); ++ streamStats[streamName].viewers++; ++ streamStats[streamName].currViews++; ++ sessionType = SESS_VIEWER; + } +- // If previous < COUNTABLE_BYTES, we haven't counted any data so far. +- // We need to count all the data in that case, otherwise we only count the difference. +- if (noBWCount != 2){ //only count connections that are countable +- if (prevUp + prevDown < COUNTABLE_BYTES){ +- if (!myStream.size() || myStream[0] == 0){ +- if (streamStats.count(myStream)){streamStats.erase(myStream);} +- }else{ +- createEmptyStatsIfNeeded(myStream); +- streamStats[myStream].upBytes += currUp; +- streamStats[myStream].downBytes += currDown; +- streamStats[myStream].packSent += currPktSent; +- streamStats[myStream].packLoss += currPktLost; +- streamStats[myStream].packRetrans += currPktRetrans; +- if (sessionType == SESS_VIEWER){streamStats[myStream].viewSeconds += lastSec - firstSec;} +- } +- }else{ +- if (!myStream.size() || myStream[0] == 0){ +- if (streamStats.count(myStream)){streamStats.erase(myStream);} +- }else{ +- createEmptyStatsIfNeeded(myStream); +- streamStats[myStream].upBytes += currUp - prevUp; +- streamStats[myStream].downBytes += currDown - prevDown; +- streamStats[myStream].packSent += currPktSent - prevPktSent; +- streamStats[myStream].packLoss += currPktLost - prevPktLost; +- streamStats[myStream].packRetrans += currPktRetrans - prevPktRetrans; +- if (sessionType == SESS_VIEWER){streamStats[myStream].viewSeconds += secIncr;} +- } +- } ++ } ++ // Only count connections that are countable ++ if (noBWCount != 2){ ++ if (!streamName.size() || streamName[0] == 0){ ++ if (streamStats.count(streamName)){streamStats.erase(streamName);} ++ }else{ ++ createEmptyStatsIfNeeded(streamName); ++ streamStats[streamName].upBytes += currUp - prevUp; ++ streamStats[streamName].downBytes += currDown - prevDown; ++ streamStats[streamName].packSent += currPktSent - prevPktSent; ++ streamStats[streamName].packLoss += currPktLost - prevPktLost; ++ streamStats[streamName].packRetrans += currPktRetrans - prevPktRetrans; ++ if (sessionType == SESS_VIEWER){streamStats[streamName].viewSeconds += secIncr;} + } + } + } +@@ -759,53 +608,19 @@ Controller::sessType Controller::statSession::getSessType(){ + return sessionType; + } + +-/// Archives connection log entries older than the given cutOff point. +-void Controller::statSession::wipeOld(uint64_t cutOff){ +- if (firstSec > cutOff){return;} +- firstSec = 0xFFFFFFFFFFFFFFFFull; +- if (oldConns.size()){ +- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ +- while (it->log.size() && it->log.begin()->first < cutOff){ +- if (it->log.size() == 1){ +- wipedDown += it->log.begin()->second.down; +- wipedUp += it->log.begin()->second.up; +- wipedPktCount += it->log.begin()->second.pktCount; +- wipedPktLost += it->log.begin()->second.pktLost; +- wipedPktRetransmit += it->log.begin()->second.pktRetransmit; +- } +- it->log.erase(it->log.begin()); +- } +- if (it->log.size()){ +- if (firstSec > it->log.begin()->first){firstSec = it->log.begin()->first;} +- } +- } +- while (oldConns.size() && !oldConns.begin()->log.size()){oldConns.pop_front();} +- } +- if (curConns.size()){ +- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ +- while (it->second.log.size() > 1 && it->second.log.begin()->first < cutOff){ +- it->second.log.erase(it->second.log.begin()); +- } +- if (it->second.log.size()){ +- if (firstSec > it->second.log.begin()->first){firstSec = it->second.log.begin()->first;} +- } +- } +- } +-} +- +-void Controller::statSession::dropSession(const Controller::sessIndex &index){ +- if (!tracked || curConns.size()){return;} ++Controller::statSession::~statSession(){ ++ if (!tracked){return;} + switch (sessionType){ +- case SESS_INPUT: +- if (streamStats.count(index.streamName) && streamStats[index.streamName].currIns){streamStats[index.streamName].currIns--;} +- break; +- case SESS_OUTPUT: +- if (streamStats.count(index.streamName) && streamStats[index.streamName].currOuts){streamStats[index.streamName].currOuts--;} +- break; +- case SESS_VIEWER: +- if (streamStats.count(index.streamName) && streamStats[index.streamName].currViews){streamStats[index.streamName].currViews--;} +- break; +- default: break; ++ case SESS_INPUT: ++ if (streamStats.count(streamName) && streamStats[streamName].currIns){streamStats[streamName].currIns--;} ++ break; ++ case SESS_OUTPUT: ++ if (streamStats.count(streamName) && streamStats[streamName].currOuts){streamStats[streamName].currOuts--;} ++ break; ++ case SESS_VIEWER: ++ if (streamStats.count(streamName) && streamStats[streamName].currViews){streamStats[streamName].currViews--;} ++ break; ++ default: break; + } + uint64_t duration = lastSec - firstActive; + if (duration < 1){duration = 1;} +@@ -815,13 +630,13 @@ void Controller::statSession::dropSession(const Controller::sessIndex &index){ + tagStream << "[" << *it << "]"; + } + } +- Controller::logAccess(index.ID, index.streamName, index.connector, index.host, duration, getUp(), ++ Controller::logAccess(sessId, streamName, curConnector, host, duration, getUp(), + getDown(), tagStream.str()); + if (Controller::accesslog.size()){ + if (Controller::accesslog == "LOG"){ + std::stringstream accessStr; +- accessStr << "Session <" << index.ID << "> " << index.streamName << " (" << index.connector +- << ") from " << index.host << " ended after " << duration << "s, avg " ++ accessStr << "Session <" << sessId << "> " << streamName << " (" << curConnector ++ << ") from " << host << " ended after " << duration << "s, avg " + << getUp() / duration / 1024 << "KB/s up " << getDown() / duration / 1024 << "KB/s down."; + if (tags.size()){accessStr << " Tags: " << tagStream.str();} + Controller::Log("ACCS", accessStr.str()); +@@ -845,8 +660,8 @@ void Controller::statSession::dropSession(const Controller::sessIndex &index){ + time(&rawtime); + timeinfo = localtime_r(&rawtime, &tmptime); + strftime(buffer, 100, "%F %H:%M:%S", timeinfo); +- accLogFile << buffer << ", " << index.ID << ", " << index.streamName << ", " +- << index.connector << ", " << index.host << ", " << duration << ", " ++ accLogFile << buffer << ", " << sessId << ", " << streamName << ", " ++ << curConnector << ", " << host << ", " << duration << ", " + << getUp() / duration / 1024 << ", " << getDown() / duration / 1024 << ", "; + if (tags.size()){accLogFile << tagStream.str();} + accLogFile << std::endl; +@@ -857,77 +672,21 @@ void Controller::statSession::dropSession(const Controller::sessIndex &index){ + firstActive = 0; + firstSec = 0xFFFFFFFFFFFFFFFFull; + lastSec = 0; +- wipedUp = 0; +- wipedDown = 0; +- wipedPktCount = 0; +- wipedPktLost = 0; +- wipedPktRetransmit = 0; +- oldConns.clear(); + sessionType = SESS_UNSET; + } + +-/// Archives the given connection. +-void Controller::statSession::finish(uint64_t index){ +- oldConns.push_back(curConns[index]); +- curConns.erase(index); +-} +- + /// Constructs an empty session + Controller::statSession::statSession(){ + firstActive = 0; + tracked = false; + firstSec = 0xFFFFFFFFFFFFFFFFull; + lastSec = 0; +- sync = 1; +- wipedUp = 0; +- wipedDown = 0; +- wipedPktCount = 0; +- wipedPktLost = 0; +- wipedPktRetransmit = 0; + sessionType = SESS_UNSET; + noBWCount = 0; +-} +- +-/// Moves the given connection to the given session +-void Controller::statSession::switchOverTo(statSession &newSess, uint64_t index){ +- // add to the given session first +- newSess.curConns[index] = curConns[index]; +- // if this connection has data, update firstSec/lastSec if needed +- if (curConns[index].log.size()){ +- if (newSess.firstSec > curConns[index].log.begin()->first){ +- newSess.firstSec = curConns[index].log.begin()->first; +- } +- if (newSess.lastSec < curConns[index].log.rbegin()->first){ +- newSess.lastSec = curConns[index].log.rbegin()->first; +- } +- } +- // remove from current session +- curConns.erase(index); +- // if there was any data, recalculate this session's firstSec and lastSec. +- if (newSess.curConns[index].log.size()){ +- firstSec = 0xFFFFFFFFFFFFFFFFull; +- lastSec = 0; +- if (oldConns.size()){ +- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ +- if (it->log.size()){ +- if (firstSec > it->log.begin()->first){firstSec = it->log.begin()->first;} +- if (lastSec < it->log.rbegin()->first){lastSec = it->log.rbegin()->first;} +- } +- } +- } +- if (curConns.size()){ +- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ +- if (it->second.log.size()){ +- if (firstSec > it->second.log.begin()->first){ +- firstSec = it->second.log.begin()->first; +- } +- if (lastSec < it->second.log.rbegin()->first){ +- lastSec = it->second.log.rbegin()->first; +- } +- } +- } +- } +- } ++ streamName = ""; ++ host = ""; ++ curConnector = ""; ++ sessId = ""; + } + + /// Returns the first measured timestamp in this session. +@@ -944,39 +703,34 @@ uint64_t Controller::statSession::getEnd(){ + bool Controller::statSession::hasDataFor(uint64_t t){ + if (lastSec < t){return false;} + if (firstSec > t){return false;} +- if (oldConns.size()){ +- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ +- if (it->hasDataFor(t)){return true;} +- } +- } +- if (curConns.size()){ +- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ +- if (it->second.hasDataFor(t)){return true;} +- } +- } +- return false; +-} +- +-/// Returns true if there is any data for this session. +-bool Controller::statSession::hasData(){ +- if (!firstSec && !lastSec){return false;} +- if (curConns.size()){return true;} +- if (oldConns.size()){ +- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ +- if (it->log.size()){return true;} +- } +- } ++ if (curData.hasDataFor(t)){return true;} + return false; + } + + /// Returns true if this session should count as a viewer on the given timestamp. + bool Controller::statSession::isViewerOn(uint64_t t){ +- return getUp(t) + getDown(t) > COUNTABLE_BYTES; ++ return getUp(t) + getDown(t); ++} ++ ++std::string Controller::statSession::getStreamName(){ ++ return streamName; ++} ++ ++std::string Controller::statSession::getHost(){ ++ return host; ++} ++ ++std::string Controller::statSession::getSessId(){ ++ return sessId; ++} ++ ++std::string Controller::statSession::getCurrentProtocols(){ ++ return curConnector; + } + + /// Returns true if this session should be considered connected +-bool Controller::statSession::isConnected(){ +- return curConns.size(); ++uint64_t Controller::statSession::newestDataPoint(){ ++ return lastSec; + } + + /// Returns true if this session has started (tracked == true) but not yet ended (log entry written) +@@ -986,188 +740,103 @@ bool Controller::statSession::isTracked(){ + + /// Returns the cumulative connected time for this session at timestamp t. + uint64_t Controller::statSession::getConnTime(uint64_t t){ +- uint64_t retVal = 0; +- if (oldConns.size()){ +- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ +- if (it->hasDataFor(t)){retVal += it->getDataFor(t).time;} +- } ++ if (curData.hasDataFor(t)){ ++ return curData.getDataFor(t).time; + } +- if (curConns.size()){ +- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ +- if (it->second.hasDataFor(t)){retVal += it->second.getDataFor(t).time;} +- } ++ return 0; ++} ++ ++/// Returns the cumulative connected time for this session. ++uint64_t Controller::statSession::getConnTime(){ ++ if (curData.log.size()){ ++ return curData.log.rbegin()->second.time; + } +- return retVal; ++ return 0; + } + + /// Returns the last requested media timestamp for this session at timestamp t. + uint64_t Controller::statSession::getLastSecond(uint64_t t){ +- if (curConns.size()){ +- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ +- if (it->second.hasDataFor(t)){return it->second.getDataFor(t).lastSecond;} +- } +- } +- if (oldConns.size()){ +- for (std::deque::reverse_iterator it = oldConns.rbegin(); it != oldConns.rend(); ++it){ +- if (it->hasDataFor(t)){return it->getDataFor(t).lastSecond;} +- } ++ if (curData.hasDataFor(t)){ ++ return curData.getDataFor(t).lastSecond; + } + return 0; + } + + /// Returns the cumulative downloaded bytes for this session at timestamp t. + uint64_t Controller::statSession::getDown(uint64_t t){ +- uint64_t retVal = wipedDown; +- if (oldConns.size()){ +- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ +- if (it->hasDataFor(t)){retVal += it->getDataFor(t).down;} +- } ++ if (curData.hasDataFor(t)){ ++ return curData.getDataFor(t).down; + } +- if (curConns.size()){ +- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ +- if (it->second.hasDataFor(t)){retVal += it->second.getDataFor(t).down;} +- } +- } +- return retVal; ++ return 0; + } + + /// Returns the cumulative uploaded bytes for this session at timestamp t. + uint64_t Controller::statSession::getUp(uint64_t t){ +- uint64_t retVal = wipedUp; +- if (oldConns.size()){ +- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ +- if (it->hasDataFor(t)){retVal += it->getDataFor(t).up;} +- } +- } +- if (curConns.size()){ +- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ +- if (it->second.hasDataFor(t)){retVal += it->second.getDataFor(t).up;} +- } ++ if (curData.hasDataFor(t)){ ++ return curData.getDataFor(t).up; + } +- return retVal; ++ return 0; + } + + /// Returns the cumulative downloaded bytes for this session at timestamp t. + uint64_t Controller::statSession::getDown(){ +- uint64_t retVal = wipedDown; +- if (oldConns.size()){ +- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ +- if (it->log.size()){retVal += it->log.rbegin()->second.down;} +- } ++ if (curData.log.size()){ ++ return curData.log.rbegin()->second.down; + } +- if (curConns.size()){ +- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ +- if (it->second.log.size()){retVal += it->second.log.rbegin()->second.down;} +- } +- } +- return retVal; ++ return 0; + } + + /// Returns the cumulative uploaded bytes for this session at timestamp t. + uint64_t Controller::statSession::getUp(){ +- uint64_t retVal = wipedUp; +- if (oldConns.size()){ +- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ +- if (it->log.size()){retVal += it->log.rbegin()->second.up;} +- } +- } +- if (curConns.size()){ +- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ +- if (it->second.log.size()){retVal += it->second.log.rbegin()->second.up;} +- } ++ if (curData.log.size()){ ++ return curData.log.rbegin()->second.up; + } +- return retVal; ++ return 0; + } + + uint64_t Controller::statSession::getPktCount(uint64_t t){ +- uint64_t retVal = wipedPktCount; +- if (oldConns.size()){ +- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ +- if (it->hasDataFor(t)){retVal += it->getDataFor(t).pktCount;} +- } ++ if (curData.hasDataFor(t)){ ++ return curData.getDataFor(t).pktCount; + } +- if (curConns.size()){ +- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ +- if (it->second.hasDataFor(t)){retVal += it->second.getDataFor(t).pktCount;} +- } +- } +- return retVal; ++ return 0; + } + + /// Returns the cumulative uploaded bytes for this session at timestamp t. + uint64_t Controller::statSession::getPktCount(){ +- uint64_t retVal = wipedPktCount; +- if (oldConns.size()){ +- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ +- if (it->log.size()){retVal += it->log.rbegin()->second.pktCount;} +- } +- } +- if (curConns.size()){ +- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ +- if (it->second.log.size()){retVal += it->second.log.rbegin()->second.pktCount;} +- } ++ if (curData.log.size()){ ++ return curData.log.rbegin()->second.pktCount; + } +- return retVal; ++ return 0; + } ++ + uint64_t Controller::statSession::getPktLost(uint64_t t){ +- uint64_t retVal = wipedPktLost; +- if (oldConns.size()){ +- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ +- if (it->hasDataFor(t)){retVal += it->getDataFor(t).pktLost;} +- } ++ if (curData.hasDataFor(t)){ ++ return curData.getDataFor(t).pktLost; + } +- if (curConns.size()){ +- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ +- if (it->second.hasDataFor(t)){retVal += it->second.getDataFor(t).pktLost;} +- } +- } +- return retVal; ++ return 0; + } + + /// Returns the cumulative uploaded bytes for this session at timestamp t. + uint64_t Controller::statSession::getPktLost(){ +- uint64_t retVal = wipedPktLost; +- if (oldConns.size()){ +- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ +- if (it->log.size()){retVal += it->log.rbegin()->second.pktLost;} +- } +- } +- if (curConns.size()){ +- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ +- if (it->second.log.size()){retVal += it->second.log.rbegin()->second.pktLost;} +- } ++ if (curData.log.size()){ ++ return curData.log.rbegin()->second.pktLost; + } +- return retVal; ++ return 0; + } ++ + uint64_t Controller::statSession::getPktRetransmit(uint64_t t){ +- uint64_t retVal = wipedPktRetransmit; +- if (oldConns.size()){ +- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ +- if (it->hasDataFor(t)){retVal += it->getDataFor(t).pktRetransmit;} +- } ++ if (curData.hasDataFor(t)){ ++ return curData.getDataFor(t).pktRetransmit; + } +- if (curConns.size()){ +- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ +- if (it->second.hasDataFor(t)){retVal += it->second.getDataFor(t).pktRetransmit;} +- } +- } +- return retVal; ++ return 0; + } + + /// Returns the cumulative uploaded bytes for this session at timestamp t. + uint64_t Controller::statSession::getPktRetransmit(){ +- uint64_t retVal = wipedPktRetransmit; +- if (oldConns.size()){ +- for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ +- if (it->log.size()){retVal += it->log.rbegin()->second.pktRetransmit;} +- } +- } +- if (curConns.size()){ +- for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ +- if (it->second.log.size()){retVal += it->second.log.rbegin()->second.pktRetransmit;} +- } ++ if (curData.log.size()){ ++ return curData.log.rbegin()->second.pktRetransmit; + } +- return retVal; ++ return 0; + } + + /// Returns the cumulative downloaded bytes per second for this session at timestamp t. +@@ -1207,6 +876,7 @@ Controller::statLog &Controller::statStorage::getDataFor(unsigned long long t){ + empty.pktCount = 0; + empty.pktLost = 0; + empty.pktRetransmit = 0; ++ empty.connectors = ""; + return empty; + } + std::map::iterator it = log.upper_bound(t); +@@ -1216,7 +886,7 @@ Controller::statLog &Controller::statStorage::getDataFor(unsigned long long t){ + + /// This function is called by parseStatistics. + /// It updates the internally saved statistics data. +-void Controller::statStorage::update(Comms::Statistics &statComm, size_t index){ ++void Controller::statStorage::update(Comms::Sessions &statComm, size_t index){ + statLog tmp; + tmp.time = statComm.getTime(index); + tmp.lastSecond = statComm.getLastSecond(index); +@@ -1225,56 +895,58 @@ void Controller::statStorage::update(Comms::Statistics &statComm, size_t index){ + tmp.pktCount = statComm.getPacketCount(index); + tmp.pktLost = statComm.getPacketLostCount(index); + tmp.pktRetransmit = statComm.getPacketRetransmitCount(index); ++ tmp.connectors = statComm.getConnector(index); + log[statComm.getNow(index)] = tmp; +- // wipe data older than approx. STAT_CUTOFF seconds +- /// \todo Remove least interesting data first. +- if (log.size() > STAT_CUTOFF){log.erase(log.begin());} ++ // wipe data older than STAT_CUTOFF seconds ++ while (log.size() && log.begin()->first < Util::bootSecs() - STAT_CUTOFF){log.erase(log.begin());} + } + + void Controller::statLeadIn(){ + statDropoff = Util::bootSecs() - 3; + } +-void Controller::statOnActive(size_t id){ +- // calculate the current session index, store as idx. +- sessIndex idx(statComm, id); + ++void Controller::statOnActive(size_t id){ + if (statComm.getNow(id) >= statDropoff){ +- // if the connection was already indexed and it has changed, move it +- if (connToSession.count(id) && connToSession[id] != idx){ +- if (sessions[connToSession[id]].getSessType() != SESS_UNSET){ +- INFO_MSG("Switching connection %zu from active session %s over to %s", id, +- connToSession[id].toStr().c_str(), idx.toStr().c_str()); +- }else{ +- INFO_MSG("Switching connection %zu from inactive session %s over to %s", id, +- connToSession[id].toStr().c_str(), idx.toStr().c_str()); +- } +- sessions[connToSession[id]].switchOverTo(sessions[idx], id); +- // Destroy this session without calling dropSession, because it was merged into another. What session? We never made it. Stop asking hard questions. Go, shoo. *sprays water* +- if (!sessions[connToSession[id]].hasData()){sessions.erase(connToSession[id]);} +- } +- if (!connToSession.count(id)){ +- INSANE_MSG("New connection: %zu as %s", id, idx.toStr().c_str()); +- } +- // store the index for later comparison +- connToSession[id] = idx; + // update the session with the latest data +- sessions[idx].update(id, statComm); ++ sessions[statComm.getSessId(id)].update(id, statComm); + } + } ++ + void Controller::statOnDisconnect(size_t id){ +- sessIndex idx(statComm, id); +- INSANE_MSG("Ended connection: %zu as %s", id, idx.toStr().c_str()); +- sessions[idx].finish(id); +- connToSession.erase(id); ++ // Check to see if cleanup is required (when a Session binary fails) ++ const std::string thisSessionId = statComm.getSessId(id); ++ // Try to lock to see if the session crashed during boot ++ IPC::semaphore sessionLock; ++ char semName[NAME_BUFFER_SIZE]; ++ snprintf(semName, NAME_BUFFER_SIZE, SEM_SESSION, thisSessionId.c_str()); ++ sessionLock.open(semName, O_CREAT | O_RDWR, ACCESSPERMS, 1); ++ if (!sessionLock.tryWaitOneSecond()){ ++ // Session likely crashed during boot. Remove the session lock which was created on bootup of the session ++ sessionLock.unlink(); ++ }else if (!statComm.sessIdExists(thisSessionId)){ ++ // There is no running process managing this session, so check if the data page still exists ++ IPC::sharedPage dataPage; ++ char userPageName[NAME_BUFFER_SIZE]; ++ snprintf(userPageName, NAME_BUFFER_SIZE, COMMS_SESSIONS, thisSessionId.c_str()); ++ dataPage.init(userPageName, 1, false, false); ++ if(dataPage){ ++ // Session likely crashed while it was running ++ dataPage.init(userPageName, 1, true); ++ FAIL_MSG("Session '%s' got canceled unexpectedly. Hoovering up the left overs...", thisSessionId.c_str()); ++ } ++ // Finally remove the session lock which was created on bootup of the session ++ sessionLock.unlink(); ++ } + } ++ + void Controller::statLeadOut(){} + + /// Returns true if this stream has at least one connected client. + bool Controller::hasViewers(std::string streamName){ + if (sessions.size()){ + long long currTime = Util::bootSecs(); +- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ +- if (it->first.streamName == streamName && ++ for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ ++ if (it->second.getStreamName() == streamName && + (it->second.hasDataFor(currTime) || it->second.hasDataFor(currTime - 1))){ + return true; + } +@@ -1377,7 +1049,6 @@ void Controller::fillClients(JSON::Value &req, JSON::Value &rep){ + if (fields & STAT_CLI_UP){rep["fields"].append("up");} + if (fields & STAT_CLI_BPS_DOWN){rep["fields"].append("downbps");} + if (fields & STAT_CLI_BPS_UP){rep["fields"].append("upbps");} +- if (fields & STAT_CLI_CRC){rep["fields"].append("crc");} + if (fields & STAT_CLI_SESSID){rep["fields"].append("sessid");} + if (fields & STAT_CLI_PKTCOUNT){rep["fields"].append("pktcount");} + if (fields & STAT_CLI_PKTLOST){rep["fields"].append("pktlost");} +@@ -1386,26 +1057,25 @@ void Controller::fillClients(JSON::Value &req, JSON::Value &rep){ + rep["data"].null(); + // loop over all sessions + if (sessions.size()){ +- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ ++ for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ + unsigned long long time = reqTime; + if (now && reqTime - it->second.getEnd() < 5){time = it->second.getEnd();} + // data present and wanted? insert it! + if ((it->second.getEnd() >= time && it->second.getStart() <= time) && +- (!streams.size() || streams.count(it->first.streamName)) && +- (!protos.size() || protos.count(it->first.connector))){ ++ (!streams.size() || streams.count(it->second.getStreamName())) && ++ (!protos.size() || protos.count(it->second.getCurrentProtocols()))){ + if (it->second.hasDataFor(time)){ + JSON::Value d; +- if (fields & STAT_CLI_HOST){d.append(it->first.host);} +- if (fields & STAT_CLI_STREAM){d.append(it->first.streamName);} +- if (fields & STAT_CLI_PROTO){d.append(it->first.connector);} ++ if (fields & STAT_CLI_HOST){d.append(it->second.getHost());} ++ if (fields & STAT_CLI_STREAM){d.append(it->second.getStreamName());} ++ if (fields & STAT_CLI_PROTO){d.append(it->second.getCurrentProtocols());} + if (fields & STAT_CLI_CONNTIME){d.append(it->second.getConnTime(time));} + if (fields & STAT_CLI_POSITION){d.append(it->second.getLastSecond(time));} + if (fields & STAT_CLI_DOWN){d.append(it->second.getDown(time));} + if (fields & STAT_CLI_UP){d.append(it->second.getUp(time));} + if (fields & STAT_CLI_BPS_DOWN){d.append(it->second.getBpsDown(time));} + if (fields & STAT_CLI_BPS_UP){d.append(it->second.getBpsUp(time));} +- if (fields & STAT_CLI_CRC){d.append(it->first.crc);} +- if (fields & STAT_CLI_SESSID){d.append(it->first.ID);} ++ if (fields & STAT_CLI_SESSID){d.append(it->second.getSessId());} + if (fields & STAT_CLI_PKTCOUNT){d.append(it->second.getPktCount(time));} + if (fields & STAT_CLI_PKTLOST){d.append(it->second.getPktLost(time));} + if (fields & STAT_CLI_PKTRETRANSMIT){d.append(it->second.getPktRetransmit(time));} +@@ -1463,12 +1133,12 @@ void Controller::fillHasStats(JSON::Value &req, JSON::Value &rep){ + { + tthread::lock_guard guard(statsMutex); + if (sessions.size()){ +- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ ++ for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ + if (it->second.getSessType() == SESS_INPUT){ +- streams.insert(it->first.streamName); ++ streams.insert(it->second.getStreamName()); + }else{ +- streams.insert(it->first.streamName); +- if (it->second.getSessType() == SESS_VIEWER){clients[it->first.streamName]++;} ++ streams.insert(it->second.getStreamName()); ++ if (it->second.getSessType() == SESS_VIEWER){clients[it->second.getStreamName()]++;} + } + } + } +@@ -1742,12 +1412,12 @@ void Controller::fillTotals(JSON::Value &req, JSON::Value &rep){ + // loop over all sessions + /// \todo Make the interval configurable instead of 1 second + if (sessions.size()){ +- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ ++ for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ + // data present and wanted? insert it! + if ((it->second.getEnd() >= (unsigned long long)reqStart || + it->second.getStart() <= (unsigned long long)reqEnd) && +- (!streams.size() || streams.count(it->first.streamName)) && +- (!protos.size() || protos.count(it->first.connector))){ ++ (!streams.size() || streams.count(it->second.getStreamName())) && ++ (!protos.size() || protos.count(it->second.getCurrentProtocols()))){ + for (unsigned long long i = reqStart; i <= reqEnd; ++i){ + if (it->second.hasDataFor(i)){ + totalsCount[i].add(it->second.getBpsDown(i), it->second.getBpsUp(i), it->second.getSessType(), it->second.getPktCount(), it->second.getPktLost(), it->second.getPktRetransmit()); +@@ -1831,6 +1501,25 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int + H.SetHeader("Server", APPIDENT); + H.StartResponse("200", "OK", H, conn, true); + ++ // Counters of current active viewers, inputs and outputs of the Session stats cache ++ std::map outputs; ++ uint32_t totViewers = 0; ++ uint32_t totInputs = 0; ++ uint32_t totOutputs = 0; ++ for (uint64_t idx = 0; idx < statComm.recordCount(); idx++){ ++ if (statComm.getStatus(idx) == COMM_STATUS_INVALID || statComm.getStatus(idx) & COMM_STATUS_DISCONNECT){continue;} ++ const std::string thisSessId = statComm.getSessId(idx); ++ // Count active viewers, inputs, outputs and protocols ++ if (thisSessId[0] == 'I'){ ++ totInputs++; ++ }else if (thisSessId[0] == 'O'){ ++ totOutputs++; ++ outputs[statComm.getConnector(idx)]++; ++ }else{ ++ totViewers++; ++ } ++ } ++ + // Collect core server stats + uint64_t mem_total = 0, mem_free = 0, mem_bufcache = 0; + uint64_t bw_up_total = 0, bw_down_total = 0; +@@ -1904,109 +1593,69 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int + response << "# TYPE mist_shm_used gauge\n"; + response << "mist_shm_used " << (shm_total - shm_free) << "\n\n"; + +- if (Controller::triggerStats.size()){ +- response << "# HELP mist_trigger_count Total executions for the given trigger\n"; +- response << "# HELP mist_trigger_time Total execution time in millis for the given trigger\n"; +- response << "# HELP mist_trigger_fails Total failed executions for the given trigger\n"; +- for (std::map::iterator it = Controller::triggerStats.begin(); +- it != Controller::triggerStats.end(); it++){ +- response << "mist_trigger_count{trigger=\"" << it->first << "\"}" << it->second.totalCount << "\n"; +- response << "mist_trigger_time{trigger=\"" << it->first << "\"}" << it->second.ms << "\n"; +- response << "mist_trigger_fails{trigger=\"" << it->first << "\"}" << it->second.failCount << "\n"; ++ response << "# HELP mist_viewseconds_total Number of seconds any media was received by a viewer.\n"; ++ response << "# TYPE mist_viewseconds_total counter\n"; ++ response << "mist_viewseconds_total " << servSeconds + viewSecondsTotal << "\n"; ++ ++ response << "\n# HELP mist_sessions_count Counts of unique sessions by type since server " ++ "start.\n"; ++ response << "# TYPE mist_sessions_count counter\n"; ++ response << "mist_sessions_count{sessType=\"viewers\"}" << servViewers << "\n"; ++ response << "mist_sessions_count{sessType=\"incoming\"}" << servInputs << "\n"; ++ response << "mist_sessions_count{sessType=\"outgoing\"}" << servOutputs << "\n\n"; ++ ++ response << "# HELP mist_bw_total Count of bytes handled since server start, by direction.\n"; ++ response << "# TYPE mist_bw_total counter\n"; ++ response << "stat_bw_total{direction=\"up\"}" << bw_up_total << "\n"; ++ response << "stat_bw_total{direction=\"down\"}" << bw_down_total << "\n\n"; ++ response << "mist_bw_total{direction=\"up\"}" << servUpBytes << "\n"; ++ response << "mist_bw_total{direction=\"down\"}" << servDownBytes << "\n\n"; ++ response << "mist_bw_other{direction=\"up\"}" << servUpOtherBytes << "\n"; ++ response << "mist_bw_other{direction=\"down\"}" << servDownOtherBytes << "\n\n"; ++ response << "mist_bw_limit " << bwLimit << "\n\n"; ++ ++ response << "# HELP mist_packets_total Total number of packets sent/received/lost over lossy protocols, server-wide.\n"; ++ response << "# TYPE mist_packets_total counter\n"; ++ response << "mist_packets_total{pkttype=\"sent\"}" << servPackSent << "\n"; ++ response << "mist_packets_total{pkttype=\"lost\"}" << servPackLoss << "\n"; ++ response << "mist_packets_total{pkttype=\"retrans\"}" << servPackRetrans << "\n"; ++ ++ if (outputs.size()){ ++ response << "# HELP mist_outputs Number of viewers active right now, server-wide, by output type.\n"; ++ response << "# TYPE mist_outputs gauge\n"; ++ for (std::map::iterator it = outputs.begin(); it != outputs.end(); ++it){ ++ response << "mist_outputs{output=\"" << it->first << "\"}" << it->second << "\n"; + } + response << "\n"; + } + + {// Scope for shortest possible blocking of statsMutex + tthread::lock_guard guard(statsMutex); +- // collect the data first +- std::map outputs; +- unsigned long totViewers = 0, totInputs = 0, totOutputs = 0; +- unsigned int tOut = Util::bootSecs() - STATS_DELAY; +- unsigned int tIn = Util::bootSecs() - STATS_INPUT_DELAY; +- // check all sessions +- if (sessions.size()){ +- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ +- switch (it->second.getSessType()){ +- case SESS_UNSET: break; +- case SESS_VIEWER: +- if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){ +- outputs[it->first.connector]++; +- totViewers++; +- } +- break; +- case SESS_INPUT: +- if (it->second.hasDataFor(tIn) && it->second.isViewerOn(tIn)){totInputs++;} +- break; +- case SESS_OUTPUT: +- if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){totOutputs++;} +- break; +- } +- } +- } + +- response << "# HELP mist_sessions_total Number of sessions active right now, server-wide, by " +- "type.\n"; ++ response << "# HELP mist_sessions_total Number of sessions active right now, server-wide, by type.\n"; + response << "# TYPE mist_sessions_total gauge\n"; + response << "mist_sessions_total{sessType=\"viewers\"}" << totViewers << "\n"; + response << "mist_sessions_total{sessType=\"incoming\"}" << totInputs << "\n"; + response << "mist_sessions_total{sessType=\"outgoing\"}" << totOutputs << "\n"; +- response << "mist_sessions_total{sessType=\"cached\"}" << sessions.size() << "\n\n"; +- +- response << "# HELP mist_viewseconds_total Number of seconds any media was received by a viewer.\n"; +- response << "# TYPE mist_viewseconds_total counter\n"; +- response << "mist_viewseconds_total " << servSeconds << "\n"; ++ response << "mist_sessions_total{sessType=\"cached\"}" << sessions.size() << "\n"; + +- response << "# HELP mist_outputs Number of viewers active right now, server-wide, by output type.\n"; +- response << "# TYPE mist_outputs gauge\n"; +- for (std::map::iterator it = outputs.begin(); it != outputs.end(); ++it){ +- response << "mist_outputs{output=\"" << it->first << "\"}" << it->second << "\n"; +- } +- response << "\n"; +- +- response << "# HELP mist_sessions_count Counts of unique sessions by type since server " +- "start.\n"; +- response << "# TYPE mist_sessions_count counter\n"; +- response << "mist_sessions_count{sessType=\"viewers\"}" << servViewers << "\n"; +- response << "mist_sessions_count{sessType=\"incoming\"}" << servInputs << "\n"; +- response << "mist_sessions_count{sessType=\"outgoing\"}" << servOutputs << "\n\n"; +- +- response << "# HELP mist_bw_total Count of bytes handled since server start, by direction.\n"; +- response << "# TYPE mist_bw_total counter\n"; +- response << "stat_bw_total{direction=\"up\"}" << bw_up_total << "\n"; +- response << "stat_bw_total{direction=\"down\"}" << bw_down_total << "\n\n"; +- response << "mist_bw_total{direction=\"up\"}" << servUpBytes << "\n"; +- response << "mist_bw_total{direction=\"down\"}" << servDownBytes << "\n\n"; +- response << "mist_bw_other{direction=\"up\"}" << servUpOtherBytes << "\n"; +- response << "mist_bw_other{direction=\"down\"}" << servDownOtherBytes << "\n\n"; +- response << "mist_bw_limit " << bwLimit << "\n\n"; +- +- response << "# HELP mist_packets_total Total number of packets sent/received/lost over lossy protocols, server-wide.\n"; +- response << "# TYPE mist_packets_total counter\n"; +- response << "mist_packets_total{pkttype=\"sent\"}" << servPackSent << "\n"; +- response << "mist_packets_total{pkttype=\"lost\"}" << servPackLoss << "\n"; +- response << "mist_packets_total{pkttype=\"retrans\"}" << servPackRetrans << "\n"; +- +- response << "\n# HELP mist_viewers Number of sessions by type and stream active right now.\n"; +- response << "# TYPE mist_viewers gauge\n"; +- response << "# HELP mist_viewcount Count of unique viewer sessions since stream start, per " ++ response << "\n# HELP mist_viewcount Count of unique viewer sessions since stream start, per " + "stream.\n"; + response << "# TYPE mist_viewcount counter\n"; +- response << "# HELP mist_bw Count of bytes handled since stream start, by direction.\n"; +- response << "# TYPE mist_bw counter\n"; + response << "# HELP mist_viewseconds Number of seconds any media was received by a viewer.\n"; + response << "# TYPE mist_viewseconds counter\n"; ++ response << "# HELP mist_bw Count of bytes handled since stream start, by direction.\n"; ++ response << "# TYPE mist_bw counter\n"; + response << "# HELP mist_packets Total number of packets sent/received/lost over lossy protocols.\n"; + response << "# TYPE mist_packets counter\n"; +- response << "mist_viewseconds_total " << servSeconds << "\n"; + for (std::map::iterator it = streamStats.begin(); +- it != streamStats.end(); ++it){ ++ it != streamStats.end(); ++it){ + response << "mist_sessions{stream=\"" << it->first << "\",sessType=\"viewers\"}" +- << it->second.currViews << "\n"; ++ << it->second.currViews << "\n"; + response << "mist_sessions{stream=\"" << it->first << "\",sessType=\"incoming\"}" +- << it->second.currIns << "\n"; ++ << it->second.currIns << "\n"; + response << "mist_sessions{stream=\"" << it->first << "\",sessType=\"outgoing\"}" +- << it->second.currOuts << "\n"; ++ << it->second.currOuts << "\n"; + response << "mist_viewcount{stream=\"" << it->first << "\"}" << it->second.viewers << "\n"; + response << "mist_viewseconds{stream=\"" << it->first << "\"} " << it->second.viewSeconds << "\n"; + response << "mist_bw{stream=\"" << it->first << "\",direction=\"up\"}" << it->second.upBytes << "\n"; +@@ -2015,6 +1664,19 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int + response << "mist_packets{stream=\"" << it->first << "\",pkttype=\"lost\"}" << it->second.packLoss << "\n"; + response << "mist_packets{stream=\"" << it->first << "\",pkttype=\"retrans\"}" << it->second.packRetrans << "\n"; + } ++ ++ if (Controller::triggerStats.size()){ ++ response << "\n# HELP mist_trigger_count Total executions for the given trigger\n"; ++ response << "# HELP mist_trigger_time Total execution time in millis for the given trigger\n"; ++ response << "# HELP mist_trigger_fails Total failed executions for the given trigger\n"; ++ for (std::map::iterator it = Controller::triggerStats.begin(); ++ it != Controller::triggerStats.end(); it++){ ++ response << "mist_trigger_count{trigger=\"" << it->first << "\"}" << it->second.totalCount << "\n"; ++ response << "mist_trigger_time{trigger=\"" << it->first << "\"}" << it->second.ms << "\n"; ++ response << "mist_trigger_fails{trigger=\"" << it->first << "\"}" << it->second.failCount << "\n"; ++ } ++ response << "\n"; ++ } + } + H.Chunkify(response.str(), conn); + } +@@ -2026,58 +1688,33 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int + resp["shm_total"] = shm_total; + resp["shm_used"] = (shm_total - shm_free); + resp["logs"] = Controller::logCounter; +- if (Controller::triggerStats.size()){ +- for (std::map::iterator it = Controller::triggerStats.begin(); +- it != Controller::triggerStats.end(); it++){ +- JSON::Value &tVal = resp["triggers"][it->first]; +- tVal["count"] = it->second.totalCount; +- tVal["ms"] = it->second.ms; +- tVal["fails"] = it->second.failCount; +- } +- } ++ resp["curr"].append(totViewers); ++ resp["curr"].append(totInputs); ++ resp["curr"].append(totOutputs); ++ resp["tot"].append(servViewers); ++ resp["tot"].append(servInputs); ++ resp["tot"].append(servOutputs); ++ resp["st"].append(bw_up_total); ++ resp["st"].append(bw_down_total); ++ resp["bw"].append(servUpBytes); ++ resp["bw"].append(servDownBytes); ++ resp["pkts"].append(servPackSent); ++ resp["pkts"].append(servPackLoss); ++ resp["pkts"].append(servPackRetrans); ++ resp["bwlimit"] = bwLimit; + {// Scope for shortest possible blocking of statsMutex + tthread::lock_guard guard(statsMutex); +- // collect the data first +- std::map outputs; +- uint64_t totViewers = 0, totInputs = 0, totOutputs = 0; +- uint64_t tOut = Util::bootSecs() - STATS_DELAY; +- uint64_t tIn = Util::bootSecs() - STATS_INPUT_DELAY; +- // check all sessions +- if (sessions.size()){ +- for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ +- switch (it->second.getSessType()){ +- case SESS_UNSET: break; +- case SESS_VIEWER: +- if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){ +- outputs[it->first.connector]++; +- totViewers++; +- } +- break; +- case SESS_INPUT: +- if (it->second.hasDataFor(tIn) && it->second.isViewerOn(tIn)){totInputs++;} +- break; +- case SESS_OUTPUT: +- if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){totOutputs++;} +- break; +- } ++ resp["curr"].append((uint64_t)sessions.size()); ++ ++ if (Controller::triggerStats.size()){ ++ for (std::map::iterator it = Controller::triggerStats.begin(); ++ it != Controller::triggerStats.end(); it++){ ++ JSON::Value &tVal = resp["triggers"][it->first]; ++ tVal["count"] = it->second.totalCount; ++ tVal["ms"] = it->second.ms; ++ tVal["fails"] = it->second.failCount; + } + } +- +- resp["curr"].append(totViewers); +- resp["curr"].append(totInputs); +- resp["curr"].append(totOutputs); +- resp["curr"].append((uint64_t)sessions.size()); +- resp["tot"].append(servViewers); +- resp["tot"].append(servInputs); +- resp["tot"].append(servOutputs); +- resp["st"].append(bw_up_total); +- resp["st"].append(bw_down_total); +- resp["bw"].append(servUpBytes); +- resp["bw"].append(servDownBytes); +- resp["pkts"].append(servPackSent); +- resp["pkts"].append(servPackLoss); +- resp["pkts"].append(servPackRetrans); +- resp["bwlimit"] = bwLimit; + if (Storage["config"].isMember("location") && Storage["config"]["location"].isMember("lat") && Storage["config"]["location"].isMember("lon")){ + resp["loc"]["lat"] = Storage["config"]["location"]["lat"].asDouble(); + resp["loc"]["lon"] = Storage["config"]["location"]["lon"].asDouble(); +diff --git a/src/controller/controller_statistics.h b/src/controller/controller_statistics.h +index 1cc4a82d..f798f811 100644 +--- a/src/controller/controller_statistics.h ++++ b/src/controller/controller_statistics.h +@@ -18,8 +18,7 @@ + namespace Controller{ + + extern bool killOnExit; +- extern unsigned int maxConnsPerIP; +- ++ + /// This function is ran whenever a stream becomes active. + void streamStarted(std::string stream); + /// This function is ran whenever a stream becomes inactive. +@@ -35,34 +34,14 @@ namespace Controller{ + uint64_t pktCount; + uint64_t pktLost; + uint64_t pktRetransmit; ++ std::string connectors; + }; + + enum sessType{SESS_UNSET = 0, SESS_INPUT, SESS_OUTPUT, SESS_VIEWER}; + +- /// This is a comparison and storage class that keeps sessions apart from each other. +- /// Whenever two of these objects are not equal, it will create a new session. +- class sessIndex{ +- public: +- sessIndex(); +- sessIndex(const Comms::Statistics &statComm, size_t id); +- std::string ID; +- std::string host; +- unsigned int crc; +- std::string streamName; +- std::string connector; +- +- bool operator==(const sessIndex &o) const; +- bool operator!=(const sessIndex &o) const; +- bool operator>(const sessIndex &o) const; +- bool operator<=(const sessIndex &o) const; +- bool operator<(const sessIndex &o) const; +- bool operator>=(const sessIndex &o) const; +- std::string toStr(); +- }; +- + class statStorage{ + public: +- void update(Comms::Statistics &statComm, size_t index); ++ void update(Comms::Sessions &statComm, size_t index); + bool hasDataFor(unsigned long long); + statLog &getDataFor(unsigned long long); + std::map log; +@@ -75,36 +54,33 @@ namespace Controller{ + uint64_t firstActive; + uint64_t firstSec; + uint64_t lastSec; +- uint64_t wipedUp; +- uint64_t wipedDown; +- uint64_t wipedPktCount; +- uint64_t wipedPktLost; +- uint64_t wipedPktRetransmit; +- std::deque oldConns; + sessType sessionType; + bool tracked; + uint8_t noBWCount; ///< Set to 2 when not to count for external bandwidth ++ std::string streamName; ++ std::string host; ++ std::string curConnector; ++ std::string sessId; ++ + public: + statSession(); +- uint32_t invalidate(); +- uint32_t kill(); +- char sync; +- std::map curConns; ++ ~statSession(); ++ statStorage curData; + std::set tags; + sessType getSessType(); +- void wipeOld(uint64_t); +- void finish(uint64_t index); +- void switchOverTo(statSession &newSess, uint64_t index); +- void update(uint64_t index, Comms::Statistics &data); +- void dropSession(const sessIndex &index); ++ void update(uint64_t index, Comms::Sessions &data); + uint64_t getStart(); + uint64_t getEnd(); + bool isViewerOn(uint64_t time); +- bool isConnected(); + bool isTracked(); + bool hasDataFor(uint64_t time); +- bool hasData(); ++ std::string getStreamName(); ++ std::string getHost(); ++ std::string getSessId(); ++ std::string getCurrentProtocols(); ++ uint64_t newestDataPoint(); + uint64_t getConnTime(uint64_t time); ++ uint64_t getConnTime(); + uint64_t getLastSecond(uint64_t time); + uint64_t getDown(uint64_t time); + uint64_t getUp(); +@@ -122,8 +98,6 @@ namespace Controller{ + uint64_t getBpsUp(uint64_t start, uint64_t end); + }; + +- extern std::map sessions; +- extern std::map connToSession; + extern tthread::mutex statsMutex; + extern uint64_t statDropoff; + +@@ -155,6 +129,7 @@ namespace Controller{ + void sessions_shutdown(const std::string &streamname, const std::string &protocol = ""); + bool hasViewers(std::string streamName); + void writeSessionCache(); /*LTS*/ ++ void killConnections(std::string sessId); + + #define PROMETHEUS_TEXT 0 + #define PROMETHEUS_JSON 1 +diff --git a/src/controller/controller_storage.cpp b/src/controller/controller_storage.cpp +index 58e7b521..bdd52893 100644 +--- a/src/controller/controller_storage.cpp ++++ b/src/controller/controller_storage.cpp +@@ -91,19 +91,6 @@ namespace Controller{ + rlxAccs->setString("tags", tags, newEndPos); + rlxAccs->setEndPos(newEndPos + 1); + } +- if (Triggers::shouldTrigger("USER_END", strm)){ +- std::stringstream plgen; +- plgen << sessId << "\n" +- << strm << "\n" +- << conn << "\n" +- << host << "\n" +- << duration << "\n" +- << up << "\n" +- << down << "\n" +- << tags; +- std::string payload = plgen.str(); +- Triggers::doTrigger("USER_END", payload, strm); +- } + } + + void normalizeTrustedProxies(JSON::Value &tp){ +@@ -450,7 +437,8 @@ namespace Controller{ + systemBoot = globAccX.getInt("systemBoot"); + } + if(!globAccX.getFieldAccX("defaultStream") +- || !globAccX.getFieldAccX("systemBoot")){ ++ || !globAccX.getFieldAccX("systemBoot") ++ || !globAccX.getFieldAccX("sessionMode")){ + globAccX.setReload(); + globCfg.master = true; + globCfg.close(); +@@ -461,12 +449,16 @@ namespace Controller{ + if (!globAccX.isReady()){ + globAccX.addField("defaultStream", RAX_128STRING); + globAccX.addField("systemBoot", RAX_64UINT); ++ globAccX.addField("sessionMode", RAX_64UINT); ++ if (!Storage["config"]["sessionMode"]){ ++ Storage["config"]["sessionMode"] = SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID; ++ } + globAccX.setRCount(1); + globAccX.setEndPos(1); + globAccX.setReady(); + } + globAccX.setString("defaultStream", Storage["config"]["defaultStream"].asStringRef()); +- globAccX.setInt("systemBoot", systemBoot); ++ globAccX.setInt("sessionMode", Storage["config"]["sessionMode"].asInt()); + globCfg.master = false; // leave the page after closing + } + } +diff --git a/src/input/input.cpp b/src/input/input.cpp +index a86fe3cd..0524f55b 100644 +--- a/src/input/input.cpp ++++ b/src/input/input.cpp +@@ -794,7 +794,7 @@ namespace Mist{ + void Input::streamMainLoop(){ + uint64_t statTimer = 0; + uint64_t startTime = Util::bootSecs(); +- Comms::Statistics statComm; ++ Comms::Connections statComm; + getNext(); + if (thisPacket && !userSelect.count(thisIdx)){ + userSelect[thisIdx].reload(streamName, thisIdx, COMM_STATUS_ACTIVE | COMM_STATUS_SOURCE | COMM_STATUS_DONOTTRACK); +@@ -820,7 +820,7 @@ namespace Mist{ + + if (Util::bootSecs() - statTimer > 1){ + // Connect to stats for INPUT detection +- if (!statComm){statComm.reload();} ++ if (!statComm){statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID);} + if (statComm){ + if (!statComm){ + config->is_active = false; +@@ -829,7 +829,6 @@ namespace Mist{ + } + uint64_t now = Util::bootSecs(); + statComm.setNow(now); +- statComm.setCRC(getpid()); + statComm.setStream(streamName); + statComm.setConnector("INPUT:" + capa["name"].asStringRef()); + statComm.setTime(now - startTime); +@@ -842,7 +841,7 @@ namespace Mist{ + } + } + +- void Input::connStats(Comms::Statistics &statComm){ ++ void Input::connStats(Comms::Connections &statComm){ + statComm.setUp(0); + statComm.setDown(streamByteCount()); + statComm.setHost(getConnectedBinHost()); +@@ -853,7 +852,7 @@ namespace Mist{ + uint64_t statTimer = 0; + uint64_t startTime = Util::bootSecs(); + size_t idx; +- Comms::Statistics statComm; ++ Comms::Connections statComm; + + + DTSC::Meta liveMeta(config->getString("streamname"), false); +@@ -985,7 +984,7 @@ namespace Mist{ + + if (Util::bootSecs() - statTimer > 1){ + // Connect to stats for INPUT detection +- if (!statComm){statComm.reload();} ++ if (!statComm){statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID);} + if (statComm){ + if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ + config->is_active = false; +@@ -994,7 +993,6 @@ namespace Mist{ + } + uint64_t now = Util::bootSecs(); + statComm.setNow(now); +- statComm.setCRC(getpid()); + statComm.setStream(streamName); + statComm.setConnector("INPUT:" + capa["name"].asStringRef()); + statComm.setTime(now - startTime); +diff --git a/src/input/input.h b/src/input/input.h +index 8d7e8891..ce7686c1 100644 +--- a/src/input/input.h ++++ b/src/input/input.h +@@ -69,7 +69,7 @@ namespace Mist{ + virtual void userOnActive(size_t id); + virtual void userOnDisconnect(size_t id); + virtual void userLeadOut(); +- virtual void connStats(Comms::Statistics & statComm); ++ virtual void connStats(Comms::Connections & statComm); + virtual void parseHeader(); + bool bufferFrame(size_t track, uint32_t keyNum); + +diff --git a/src/input/input_rtsp.cpp b/src/input/input_rtsp.cpp +index fef613bb..ebb812e1 100644 +--- a/src/input/input_rtsp.cpp ++++ b/src/input/input_rtsp.cpp +@@ -196,7 +196,7 @@ namespace Mist{ + } + + void InputRTSP::streamMainLoop(){ +- Comms::Statistics statComm; ++ Comms::Connections statComm; + uint64_t startTime = Util::epoch(); + uint64_t lastPing = Util::bootSecs(); + uint64_t lastSecs = 0; +@@ -210,7 +210,7 @@ namespace Mist{ + if (lastSecs != currSecs){ + lastSecs = currSecs; + // Connect to stats for INPUT detection +- statComm.reload(); ++ statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID); + if (statComm){ + if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ + config->is_active = false; +@@ -219,7 +219,6 @@ namespace Mist{ + } + uint64_t now = Util::bootSecs(); + statComm.setNow(now); +- statComm.setCRC(getpid()); + statComm.setStream(streamName); + statComm.setConnector("INPUT:" + capa["name"].asStringRef()); + statComm.setUp(tcpCon.dataUp()); +diff --git a/src/input/input_sdp.cpp b/src/input/input_sdp.cpp +index 62c02aef..3169a836 100644 +--- a/src/input/input_sdp.cpp ++++ b/src/input/input_sdp.cpp +@@ -193,7 +193,7 @@ namespace Mist{ + + // Updates stats and quits if parsePacket returns false + void InputSDP::streamMainLoop(){ +- Comms::Statistics statComm; ++ Comms::Connections statComm; + uint64_t startTime = Util::epoch(); + uint64_t lastSecs = 0; + // Get RTP packets from UDP socket and stop if this fails +@@ -202,7 +202,7 @@ namespace Mist{ + if (lastSecs != currSecs){ + lastSecs = currSecs; + // Connect to stats for INPUT detection +- statComm.reload(); ++ statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID); + if (statComm){ + if (statComm.getStatus() == COMM_STATUS_REQDISCONNECT){ + config->is_active = false; +@@ -211,7 +211,6 @@ namespace Mist{ + } + uint64_t now = Util::bootSecs(); + statComm.setNow(now); +- statComm.setCRC(getpid()); + statComm.setStream(streamName); + statComm.setConnector("INPUT:" + capa["name"].asStringRef()); + statComm.setDown(bytesRead); +diff --git a/src/input/input_ts.cpp b/src/input/input_ts.cpp +index cfe3f7cd..23311dc0 100644 +--- a/src/input/input_ts.cpp ++++ b/src/input/input_ts.cpp +@@ -527,7 +527,7 @@ namespace Mist{ + void inputTS::streamMainLoop(){ + meta.removeTrack(tmpIdx); + INFO_MSG("Removed temptrack %zu", tmpIdx); +- Comms::Statistics statComm; ++ Comms::Connections statComm; + uint64_t downCounter = 0; + uint64_t startTime = Util::bootSecs(); + uint64_t noDataSince = Util::bootSecs(); +@@ -621,7 +621,7 @@ namespace Mist{ + // Check for and spawn threads here. + if (Util::bootSecs() - threadCheckTimer > 1){ + // Connect to stats for INPUT detection +- statComm.reload(); ++ statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID); + if (statComm){ + if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ + config->is_active = false; +@@ -630,7 +630,6 @@ namespace Mist{ + } + uint64_t now = Util::bootSecs(); + statComm.setNow(now); +- statComm.setCRC(getpid()); + statComm.setStream(streamName); + statComm.setConnector("INPUT:" + capa["name"].asStringRef()); + statComm.setUp(0); +diff --git a/src/input/input_tssrt.cpp b/src/input/input_tssrt.cpp +index 4219ebbc..8fef6d7d 100644 +--- a/src/input/input_tssrt.cpp ++++ b/src/input/input_tssrt.cpp +@@ -282,7 +282,7 @@ namespace Mist{ + + void inputTSSRT::setSingular(bool newSingular){singularFlag = newSingular;} + +- void inputTSSRT::connStats(Comms::Statistics &statComm){ ++ void inputTSSRT::connStats(Comms::Connections &statComm){ + statComm.setUp(srtConn.dataUp()); + statComm.setDown(srtConn.dataDown()); + statComm.setHost(getConnectedBinHost()); +diff --git a/src/input/input_tssrt.h b/src/input/input_tssrt.h +index 4f337b48..143174cb 100644 +--- a/src/input/input_tssrt.h ++++ b/src/input/input_tssrt.h +@@ -40,7 +40,7 @@ namespace Mist{ + + Socket::SRTConnection srtConn; + bool singularFlag; +- virtual void connStats(Comms::Statistics &statComm); ++ virtual void connStats(Comms::Connections &statComm); + + Util::ResizeablePointer rawBuffer; + size_t rawIdx; +diff --git a/src/output/output.cpp b/src/output/output.cpp +index 1fa86eee..50d67549 100644 +--- a/src/output/output.cpp ++++ b/src/output/output.cpp +@@ -7,7 +7,7 @@ + #include + #include + +-#include "output.h" ++#include "output.h" + #include + #include + #include +@@ -92,7 +92,7 @@ namespace Mist{ + firstTime = 0; + firstPacketTime = 0xFFFFFFFFFFFFFFFFull; + lastPacketTime = 0; +- crc = getpid(); ++ sid = ""; + parseData = false; + wantRequest = true; + sought = false; +@@ -100,7 +100,7 @@ namespace Mist{ + isBlocking = false; + needsLookAhead = 0; + extraKeepAway = 0; +- lastStats = 0; ++ lastStats = 0xFFFFFFFFFFFFFFFFull; + maxSkipAhead = 7500; + uaDelay = 10; + realTime = 1000; +@@ -111,6 +111,7 @@ namespace Mist{ + lastPushUpdate = 0; + previousFile = ""; + currentFile = ""; ++ sessionMode = 0xFFFFFFFFFFFFFFFFull; + + lastRecv = Util::bootSecs(); + if (myConn){ +@@ -211,95 +212,9 @@ namespace Mist{ + onFail("Not allowed to play (CONN_PLAY)"); + } + } +- doSync(true); + /*LTS-END*/ + } + +- /// If called with force set to true and a USER_NEW trigger enabled, forces a sync immediately. +- /// Otherwise, does nothing unless the sync byte is set to 2, in which case it forces a sync as well. +- /// May be called recursively because it calls stats() which calls this function. +- /// If this happens, the extra calls to the function return instantly. +- void Output::doSync(bool force){ +- if (!statComm){return;} +- if (recursingSync){return;} +- recursingSync = true; +- if (statComm.getSync() == 2 || force){ +- if (getStatsName() == capa["name"].asStringRef() && Triggers::shouldTrigger("USER_NEW", streamName)){ +- // sync byte 0 = no sync yet, wait for sync from controller... +- char initialSync = 0; +- // attempt to load sync status from session cache in shm +- { +- IPC::semaphore cacheLock(SEM_SESSCACHE, O_RDWR, ACCESSPERMS, 16); +- if (cacheLock){cacheLock.wait();} +- IPC::sharedPage shmSessions(SHM_SESSIONS, SHM_SESSIONS_SIZE, false, false); +- if (shmSessions.mapped){ +- char shmEmpty[SHM_SESSIONS_ITEM]; +- memset(shmEmpty, 0, SHM_SESSIONS_ITEM); +- std::string host; +- Socket::hostBytesToStr(statComm.getHost().data(), 16, host); +- uint32_t shmOffset = 0; +- const std::string &cName = capa["name"].asStringRef(); +- while (shmOffset + SHM_SESSIONS_ITEM < SHM_SESSIONS_SIZE){ +- // compare crc +- if (*(uint32_t*)(shmSessions.mapped + shmOffset) == crc){ +- // compare stream name +- if (strncmp(shmSessions.mapped + shmOffset + 4, streamName.c_str(), 100) == 0){ +- // compare connector +- if (strncmp(shmSessions.mapped + shmOffset + 104, cName.c_str(), 20) == 0){ +- // compare host +- if (strncmp(shmSessions.mapped + shmOffset + 124, host.c_str(), 40) == 0){ +- initialSync = shmSessions.mapped[shmOffset + 164]; +- HIGH_MSG("Instant-sync from session cache to %u", (unsigned int)initialSync); +- break; +- } +- } +- } +- } +- // stop if we reached the end +- if (memcmp(shmSessions.mapped + shmOffset, shmEmpty, SHM_SESSIONS_ITEM) == 0){ +- break; +- } +- shmOffset += SHM_SESSIONS_ITEM; +- } +- } +- if (cacheLock){cacheLock.post();} +- } +- unsigned int i = 0; +- statComm.setSync(initialSync); +- // wait max 10 seconds for sync +- while ((!statComm.getSync() || statComm.getSync() == 2) && i++ < 100){ +- Util::wait(100); +- stats(true); +- } +- HIGH_MSG("USER_NEW sync achieved: %u", statComm.getSync()); +- // 1 = check requested (connection is new) +- if (statComm.getSync() == 1){ +- std::string payload = streamName + "\n" + getConnectedHost() + "\n" + +- JSON::Value(crc).asString() + "\n" + capa["name"].asStringRef() + +- "\n" + reqUrl + "\n" + statComm.getSessId(); +- if (!Triggers::doTrigger("USER_NEW", payload, streamName)){ +- onFail("Not allowed to play (USER_NEW)"); +- statComm.setSync(100); // 100 = denied +- }else{ +- statComm.setSync(10); // 10 = accepted +- } +- } +- // 100 = denied +- if (statComm.getSync() == 100){onFail("Not allowed to play (USER_NEW cache)");} +- if (statComm.getSync() == 0){ +- onFail("Not allowed to play (USER_NEW init timeout)", true); +- } +- if (statComm.getSync() == 2){ +- onFail("Not allowed to play (USER_NEW re-init timeout)", true); +- } +- // anything else = accepted +- }else{ +- statComm.setSync(10); // auto-accept if no trigger +- } +- } +- recursingSync = false; +- } +- + std::string Output::getConnectedHost(){return myConn.getHost();} + + std::string Output::getConnectedBinHost(){ +@@ -433,10 +348,10 @@ namespace Mist{ + + //Connect to stats reporting, if not connected already + if (!statComm){ +- statComm.reload(); ++ statComm.reload(streamName, getConnectedHost(), sid, capa["name"].asStringRef(), reqUrl, sessionMode); + stats(true); + } +- ++ + //push inputs do not need to wait for stream to be ready for playback + if (isPushing()){return;} + +@@ -986,7 +901,7 @@ namespace Mist{ + INFO_MSG("Will split recording every %lld seconds", atoll(targetParams["split"].c_str())); + targetParams["nxt-split"] = JSON::Value((int64_t)(seekPos + endRec)).asString(); + } +- // Duration to record in seconds. Overrides recstop. ++ // Duration to record in seconds. Oversides recstop. + if (targetParams.count("duration")){ + long long endRec = atoll(targetParams["duration"].c_str()) * 1000; + targetParams["recstop"] = JSON::Value((int64_t)(seekPos + endRec)).asString(); +@@ -1301,6 +1216,7 @@ namespace Mist{ + /// request URL (if any) + /// ~~~~~~~~~~~~~~~ + int Output::run(){ ++ sessionMode = Util::getGlobalConfig("sessionMode").asInt(); + /*LTS-START*/ + // Connect to file target, if needed + if (isFileTarget()){ +@@ -1507,6 +1423,8 @@ namespace Mist{ + /*LTS-END*/ + + disconnect(); ++ stats(true); ++ userSelect.clear(); + myConn.close(); + return 0; + } +@@ -1822,7 +1740,7 @@ namespace Mist{ + // also cancel if it has been less than a second since the last update + // unless force is set to true + uint64_t now = Util::bootSecs(); +- if (now == lastStats && !force){return;} ++ if (now <= lastStats && !force){return;} + + if (isRecording()){ + if(lastPushUpdate == 0){ +@@ -1861,13 +1779,17 @@ namespace Mist{ + } + } + +- if (!statComm){statComm.reload();} +- if (!statComm){return;} ++ if (!statComm){statComm.reload(streamName, getConnectedHost(), sid, capa["name"].asStringRef(), reqUrl, sessionMode);} ++ if (!statComm){return;} ++ if (statComm.getExit()){ ++ onFail("Shutting down since this session is not allowed to view this stream"); ++ return; ++ } + + lastStats = now; + +- VERYHIGH_MSG("Writing stats: %s, %s, %u, %" PRIu64 ", %" PRIu64, getConnectedHost().c_str(), streamName.c_str(), +- crc & 0xFFFFFFFFu, myConn.dataUp(), myConn.dataDown()); ++ VERYHIGH_MSG("Writing stats: %s, %s, %s, %" PRIu64 ", %" PRIu64, getConnectedHost().c_str(), streamName.c_str(), ++ sid.c_str(), myConn.dataUp(), myConn.dataDown()); + /*LTS-START*/ + if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ + onFail("Shutting down on controller request"); +@@ -1875,9 +1797,6 @@ namespace Mist{ + } + /*LTS-END*/ + statComm.setNow(now); +- statComm.setHost(getConnectedBinHost()); +- statComm.setCRC(crc); +- statComm.setStream(streamName); + statComm.setConnector(getStatsName()); + connStats(now, statComm); + statComm.setLastSecond(thisPacket ? thisPacket.getTime() : 0); +@@ -1887,7 +1806,7 @@ namespace Mist{ + // Tag the session with the user agent + if (newUA && ((now - myConn.connTime()) >= uaDelay || !myConn) && UA.size()){ + std::string APIcall = +- "{\"tag_sessid\":{\"" + statComm.getSessId() + "\":" + JSON::string_escape("UA:" + UA) + "}}"; ++ "{\"tag_sessid\":{\"" + statComm.sessionId + "\":" + JSON::string_escape("UA:" + UA) + "}}"; + Socket::UDPConnection uSock; + uSock.SetDestination(UDP_API_HOST, UDP_API_PORT); + uSock.SendNow(APIcall); +@@ -1895,8 +1814,6 @@ namespace Mist{ + } + /*LTS-END*/ + +- doSync(); +- + if (isPushing()){ + for (std::map::iterator it = userSelect.begin(); it != userSelect.end(); it++){ + if (it->second.getStatus() & COMM_STATUS_REQDISCONNECT){ +@@ -1912,7 +1829,7 @@ namespace Mist{ + } + } + +- void Output::connStats(uint64_t now, Comms::Statistics &statComm){ ++ void Output::connStats(uint64_t now, Comms::Connections &statComm){ + statComm.setUp(myConn.dataUp()); + statComm.setDown(myConn.dataDown()); + statComm.setTime(now - myConn.connTime()); +diff --git a/src/output/output.h b/src/output/output.h +index 84d17482..173b3840 100644 +--- a/src/output/output.h ++++ b/src/output/output.h +@@ -86,7 +86,6 @@ namespace Mist{ + std::string hostLookup(std::string ip); + bool onList(std::string ip, std::string list); + std::string getCountry(std::string ip); +- void doSync(bool force = false); + /*LTS-END*/ + std::map currentPage; + void loadPageForKey(size_t trackId, size_t keyNum); +@@ -105,6 +104,7 @@ namespace Mist{ + bool firstData; + uint64_t lastPushUpdate; + bool newUA; ++ + protected: // these are to be messed with by child classes + virtual bool inlineRestartCapable() const{ + return false; +@@ -122,15 +122,16 @@ namespace Mist{ + virtual std::string getStatsName(); + virtual bool hasSessionIDs(){return false;} + +- virtual void connStats(uint64_t now, Comms::Statistics &statComm); ++ virtual void connStats(uint64_t now, Comms::Connections &statComm); + + std::set getSupportedTracks(const std::string &type = "") const; + + inline virtual bool keepGoing(){return config->is_active && myConn;} + +- Comms::Statistics statComm; ++ Comms::Connections statComm; + bool isBlocking; ///< If true, indicates that myConn is blocking. +- uint32_t crc; ///< Checksum, if any, for usage in the stats. ++ std::string sid; ///< Random identifier used to split connections into sessions ++ uint64_t sessionMode; + uint64_t nextKeyTime(); + + // stream delaying variables +diff --git a/src/output/output_cmaf.cpp b/src/output/output_cmaf.cpp +index 4582a8b6..c2fbc5bc 100644 +--- a/src/output/output_cmaf.cpp ++++ b/src/output/output_cmaf.cpp +@@ -101,7 +101,7 @@ namespace Mist{ + } + } + +- void OutCMAF::connStats(uint64_t now, Comms::Statistics &statComm){ ++ void OutCMAF::connStats(uint64_t now, Comms::Connections &statComm){ + // For non-push usage, call usual function. + if (!isRecording()){ + Output::connStats(now, statComm); +diff --git a/src/output/output_cmaf.h b/src/output/output_cmaf.h +index efc36511..390549cc 100644 +--- a/src/output/output_cmaf.h ++++ b/src/output/output_cmaf.h +@@ -39,7 +39,7 @@ namespace Mist{ + bool isReadyForPlay(); + + protected: +- virtual void connStats(uint64_t now, Comms::Statistics &statComm); ++ virtual void connStats(uint64_t now, Comms::Connections &statComm); + void onTrackEnd(size_t idx); + bool hasSessionIDs(){return !config->getBool("mergesessions");} + +@@ -72,6 +72,7 @@ namespace Mist{ + void startPushOut(); + void pushNext(); + ++ uint32_t crc; + HTTP::URL pushUrl; + std::map pushTracks; + void setupTrackObject(size_t idx); +diff --git a/src/output/output_http.cpp b/src/output/output_http.cpp +index 252f069a..f55b974d 100644 +--- a/src/output/output_http.cpp ++++ b/src/output/output_http.cpp +@@ -55,7 +55,6 @@ namespace Mist{ + } + + void HTTPOutput::onFail(const std::string &msg, bool critical){ +- INFO_MSG("Failing '%s': %s", H.url.c_str(), msg.c_str()); + if (!webSock && !isRecording() && !responded){ + H.Clean(); // make sure no parts of old requests are left in any buffers + H.SetHeader("Server", APPIDENT); +@@ -238,18 +237,6 @@ namespace Mist{ + } + /*LTS-END*/ + if (H.hasHeader("User-Agent")){UA = H.GetHeader("User-Agent");} +- if (hasSessionIDs()){ +- if (H.GetVar("sessId").size()){ +- std::string ua = H.GetVar("sessId"); +- crc = checksum::crc32(0, ua.data(), ua.size()); +- }else{ +- std::string ua = JSON::Value(getpid()).asString(); +- crc = checksum::crc32(0, ua.data(), ua.size()); +- } +- }else{ +- std::string mixed_ua = UA + H.GetHeader("X-Playback-Session-Id"); +- crc = checksum::crc32(0, mixed_ua.data(), mixed_ua.size()); +- } + + if (H.GetVar("audio") != ""){targetParams["audio"] = H.GetVar("audio");} + if (H.GetVar("video") != ""){targetParams["video"] = H.GetVar("video");} +@@ -281,6 +268,21 @@ namespace Mist{ + realTime = 0; + } + } ++ // Get session ID cookie or generate a random one if it wasn't set ++ if (!sid.size()){ ++ std::map storage; ++ const std::string koekjes = H.GetHeader("Cookie"); ++ HTTP::parseVars(koekjes, storage); ++ if (storage.count("sid")){ ++ // Get sid cookie, which is used to divide connections into sessions ++ sid = storage.at("sid"); ++ }else{ ++ // Else generate one ++ const std::string newSid = UA + JSON::Value(getpid()).asString(); ++ sid = JSON::Value(checksum::crc32(0, newSid.data(), newSid.size())).asString(); ++ H.SetHeader("sid", sid.c_str()); ++ } ++ } + // Handle upgrade to websocket if the output supports it + std::string upgradeHeader = H.GetHeader("Upgrade"); + Util::stringToLower(upgradeHeader); +diff --git a/src/output/output_tssrt.cpp b/src/output/output_tssrt.cpp +index db07dc91..dc04b247 100644 +--- a/src/output/output_tssrt.cpp ++++ b/src/output/output_tssrt.cpp +@@ -344,7 +344,7 @@ namespace Mist{ + } + } + +- void OutTSSRT::connStats(uint64_t now, Comms::Statistics &statComm){ ++ void OutTSSRT::connStats(uint64_t now, Comms::Connections &statComm){ + if (!srtConn){return;} + statComm.setUp(srtConn.dataUp()); + statComm.setDown(srtConn.dataDown()); +diff --git a/src/output/output_tssrt.h b/src/output/output_tssrt.h +index 1423af8d..71c9b72f 100644 +--- a/src/output/output_tssrt.h ++++ b/src/output/output_tssrt.h +@@ -15,7 +15,7 @@ namespace Mist{ + bool isReadyForPlay(){return true;} + virtual void requestHandler(); + protected: +- virtual void connStats(uint64_t now, Comms::Statistics &statComm); ++ virtual void connStats(uint64_t now, Comms::Connections &statComm); + virtual std::string getConnectedHost(){return srtConn.remotehost;} + virtual std::string getConnectedBinHost(){return srtConn.getBinHost();} + +diff --git a/src/output/output_webrtc.cpp b/src/output/output_webrtc.cpp +index b4289b64..058cd737 100644 +--- a/src/output/output_webrtc.cpp ++++ b/src/output/output_webrtc.cpp +@@ -1015,7 +1015,7 @@ namespace Mist{ + } + } + +- void OutWebRTC::connStats(uint64_t now, Comms::Statistics &statComm){ ++ void OutWebRTC::connStats(uint64_t now, Comms::Connections &statComm){ + statComm.setUp(myConn.dataUp()); + statComm.setDown(myConn.dataDown()); + statComm.setPacketCount(totalPkts); +diff --git a/src/output/output_webrtc.h b/src/output/output_webrtc.h +index 9c3db580..b2b528c5 100644 +--- a/src/output/output_webrtc.h ++++ b/src/output/output_webrtc.h +@@ -144,7 +144,7 @@ namespace Mist{ + void onDTSCConverterHasInitData(const size_t trackID, const std::string &initData); + void onRTPPacketizerHasRTPPacket(const char *data, size_t nbytes); + void onRTPPacketizerHasRTCPPacket(const char *data, uint32_t nbytes); +- virtual void connStats(uint64_t now, Comms::Statistics &statComm); ++ virtual void connStats(uint64_t now, Comms::Connections &statComm); + + private: + uint64_t lastRecv; +diff --git a/src/process/process_exec.cpp b/src/process/process_exec.cpp +index 7b775848..34ac53fd 100644 +--- a/src/process/process_exec.cpp ++++ b/src/process/process_exec.cpp +@@ -72,7 +72,7 @@ namespace Mist{ + } + bool needsLock(){return false;} + bool isSingular(){return false;} +- void connStats(Comms::Statistics &statComm){ ++ void connStats(Comms::Connections &statComm){ + for (std::map::iterator it = userSelect.begin(); it != userSelect.end(); it++){ + if (it->second){it->second.setStatus(COMM_STATUS_DONOTTRACK | it->second.getStatus());} + } +@@ -117,7 +117,7 @@ namespace Mist{ + realTime = 0; + OutEBML::sendHeader(); + }; +- void connStats(uint64_t now, Comms::Statistics &statComm){ ++ void connStats(uint64_t now, Comms::Connections &statComm){ + for (std::map::iterator it = userSelect.begin(); it != userSelect.end(); it++){ + if (it->second){it->second.setStatus(COMM_STATUS_DONOTTRACK | it->second.getStatus());} + } +diff --git a/src/session.cpp b/src/session.cpp +new file mode 100644 +index 00000000..3865e0ec +--- /dev/null ++++ b/src/session.cpp +@@ -0,0 +1,367 @@ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++// Stats of connections which have closed are added to these global counters ++uint64_t globalNow = 0; ++uint64_t globalTime = 0; ++uint64_t globalDown = 0; ++uint64_t globalUp = 0; ++uint64_t globalPktcount = 0; ++uint64_t globalPktloss = 0; ++uint64_t globalPktretrans = 0; ++// Counts the duration a connector has been active ++std::map connectorCount; ++std::map connectorLastActive; ++// Set to True when a session gets invalidated, so that we know to run a new USER_NEW trigger ++bool forceTrigger = false; ++void handleSignal(int signum){ ++ if (signum == SIGUSR1){ ++ forceTrigger = true; ++ } ++} ++ ++void userOnActive(uint64_t &connections){ ++ ++connections; ++} ++ ++std::string getEnvWithDefault(const std::string variableName, const std::string defaultValue){ ++ const char* value = getenv(variableName.c_str()); ++ if (value){ ++ unsetenv(variableName.c_str()); ++ return value; ++ }else{ ++ return defaultValue; ++ } ++} ++ ++/// \brief Adds stats of closed connections to global counters ++void userOnDisconnect(Comms::Connections & connections, size_t idx){ ++ std::string thisConnector = connections.getConnector(idx); ++ if (thisConnector != ""){ ++ connectorCount[thisConnector] += connections.getTime(idx); ++ } ++ globalTime += connections.getTime(idx); ++ globalDown += connections.getDown(idx); ++ globalUp += connections.getUp(idx); ++ globalPktcount += connections.getPacketCount(idx); ++ globalPktloss += connections.getPacketLostCount(idx); ++ globalPktretrans += connections.getPacketRetransmitCount(idx); ++} ++ ++int main(int argc, char **argv){ ++ Comms::Connections connections; ++ Comms::Sessions sessions; ++ uint64_t lastSeen = Util::bootSecs(); ++ uint64_t currentConnections = 0; ++ Util::redirectLogsIfNeeded(); ++ signal(SIGUSR1, handleSignal); ++ // Init config and parse arguments ++ Util::Config config = Util::Config("MistSession"); ++ JSON::Value option; ++ ++ option.null(); ++ option["arg_num"] = 1; ++ option["arg"] = "string"; ++ option["help"] = "Session identifier of the entire session"; ++ option["default"] = ""; ++ config.addOption("sessionid", option); ++ ++ option.null(); ++ option["long"] = "sessionmode"; ++ option["short"] = "m"; ++ option["arg"] = "integer"; ++ option["default"] = 0; ++ config.addOption("sessionmode", option); ++ ++ option.null(); ++ option["long"] = "streamname"; ++ option["short"] = "n"; ++ option["arg"] = "string"; ++ option["default"] = ""; ++ config.addOption("streamname", option); ++ ++ option.null(); ++ option["long"] = "ip"; ++ option["short"] = "i"; ++ option["arg"] = "string"; ++ option["default"] = ""; ++ config.addOption("ip", option); ++ ++ option.null(); ++ option["long"] = "sid"; ++ option["short"] = "s"; ++ option["arg"] = "string"; ++ option["default"] = ""; ++ config.addOption("sid", option); ++ ++ option.null(); ++ option["long"] = "protocol"; ++ option["short"] = "p"; ++ option["arg"] = "string"; ++ option["default"] = ""; ++ config.addOption("protocol", option); ++ ++ option.null(); ++ option["long"] = "requrl"; ++ option["short"] = "r"; ++ option["arg"] = "string"; ++ option["default"] = ""; ++ config.addOption("requrl", option); ++ ++ config.activate(); ++ if (!(config.parseArgs(argc, argv))){ ++ FAIL_MSG("Cannot start a new session due to invalid arguments"); ++ return 1; ++ } ++ ++ const uint64_t bootTime = Util::getMicros(); ++ // Get session ID, session mode and other variables used as payload for the USER_NEW and USER_END triggers ++ const std::string thisStreamName = config.getString("streamname"); ++ const std::string thisHost = config.getString("ip"); ++ const std::string thisSid = config.getString("sid"); ++ const std::string thisProtocol = config.getString("protocol"); ++ const std::string thisReqUrl = config.getString("requrl"); ++ const std::string thisSessionId = config.getString("sessionid"); ++ const uint64_t sessionMode = config.getInteger("sessionmode"); ++ ++ if (thisSessionId == "" || thisProtocol == "" || thisStreamName == ""){ ++ FAIL_MSG("Given the following incomplete arguments: SessionId: '%s', protocol: '%s', stream name: '%s'. Aborting opening a new session", ++ thisSessionId.c_str(), thisProtocol.c_str(), thisStreamName.c_str()); ++ return 1; ++ } ++ ++ MEDIUM_MSG("Starting a new session for sessionId '%s'", thisSessionId.c_str()); ++ if (sessionMode < 1 || sessionMode > 15) { ++ FAIL_MSG("Invalid session mode of value %lu. Should be larger than 0 and smaller than 16", sessionMode); ++ return 1; ++ } ++ ++ // Try to lock to ensure we are the only process initialising this session ++ IPC::semaphore sessionLock; ++ char semName[NAME_BUFFER_SIZE]; ++ snprintf(semName, NAME_BUFFER_SIZE, SEM_SESSION, thisSessionId.c_str()); ++ sessionLock.open(semName, O_CREAT | O_RDWR, ACCESSPERMS, 1); ++ // If the lock fails, the previous Session process must've failed in spectacular fashion ++ // It's the Controller's task to clean everything up. When the lock fails, this cleanup hasn't happened yet ++ if (!sessionLock.tryWaitOneSecond()){ ++ FAIL_MSG("Session '%s' already locked", thisSessionId.c_str()); ++ return 1; ++ } ++ ++ // Check if a page already exists for this session ID. If so, quit ++ { ++ IPC::sharedPage dataPage; ++ char userPageName[NAME_BUFFER_SIZE]; ++ snprintf(userPageName, NAME_BUFFER_SIZE, COMMS_SESSIONS, thisSessionId.c_str()); ++ dataPage.init(userPageName, 0, false, false); ++ if (dataPage){ ++ INFO_MSG("Session '%s' already has a running process", thisSessionId.c_str()); ++ sessionLock.post(); ++ return 0; ++ } ++ } ++ ++ // Claim a spot in shared memory for this session on the global statistics page ++ sessions.reload(); ++ if (!sessions){ ++ FAIL_MSG("Unable to register entry for session '%s' on the stats page", thisSessionId.c_str()); ++ sessionLock.post(); ++ return 1; ++ } ++ // Open the shared memory page containing statistics for each individual connection in this session ++ connections.reload(thisStreamName, thisHost, thisSid, thisProtocol, thisReqUrl, sessionMode, true, false); ++ // Initialise global session data ++ sessions.setHost(thisHost); ++ sessions.setSessId(thisSessionId); ++ sessions.setStream(thisStreamName); ++ sessionLock.post(); ++ ++ // Determine session type, since triggers only get run for viewer type sessions ++ uint64_t thisType = 0; ++ if (thisSessionId[0] == 'I'){ ++ INFO_MSG("Started new input session %s in %lu microseconds", thisSessionId.c_str(), Util::getMicros(bootTime)); ++ thisType = 1; ++ } ++ else if (thisSessionId[0] == 'O'){ ++ INFO_MSG("Started new output session %s in %lu microseconds", thisSessionId.c_str(), Util::getMicros(bootTime)); ++ thisType = 2; ++ } ++ else{ ++ INFO_MSG("Started new viewer session %s in %lu microseconds", thisSessionId.c_str(), Util::getMicros(bootTime)); ++ } ++ ++ // Do a USER_NEW trigger if it is defined for this stream ++ if (!thisType && Triggers::shouldTrigger("USER_NEW", thisStreamName)){ ++ std::string payload = thisStreamName + "\n" + thisHost + "\n" + ++ thisSid + "\n" + thisProtocol + ++ "\n" + thisReqUrl + "\n" + thisSessionId; ++ if (!Triggers::doTrigger("USER_NEW", payload, thisStreamName)){ ++ // Mark all connections of this session as finished, since this viewer is not allowed to view this stream ++ connections.setExit(); ++ connections.finishAll(); ++ } ++ } ++ ++ uint64_t lastSecond = 0; ++ uint64_t now = 0; ++ uint64_t time = 0; ++ uint64_t down = 0; ++ uint64_t up = 0; ++ uint64_t pktcount = 0; ++ uint64_t pktloss = 0; ++ uint64_t pktretrans = 0; ++ std::string connector = ""; ++ // Stay active until Mist exits or we no longer have an active connection ++ while (config.is_active && (currentConnections || Util::bootSecs() - lastSeen <= 10)){ ++ time = 0; ++ connector = ""; ++ down = 0; ++ up = 0; ++ pktcount = 0; ++ pktloss = 0; ++ pktretrans = 0; ++ currentConnections = 0; ++ ++ // Count active connections ++ COMM_LOOP(connections, userOnActive(currentConnections), userOnDisconnect(connections, id)); ++ // Loop through all connection entries to get a summary of statistics ++ for (uint64_t idx = 0; idx < connections.recordCount(); idx++){ ++ if (connections.getStatus(idx) == COMM_STATUS_INVALID || connections.getStatus(idx) & COMM_STATUS_DISCONNECT){continue;} ++ uint64_t thisLastSecond = connections.getLastSecond(idx); ++ std::string thisConnector = connections.getConnector(idx); ++ // Save info on the latest active connection separately ++ if (thisLastSecond > lastSecond){ ++ lastSecond = thisLastSecond; ++ now = connections.getNow(idx); ++ } ++ connectorLastActive[thisConnector] = thisLastSecond; ++ // Sum all other variables ++ time += connections.getTime(idx); ++ down += connections.getDown(idx); ++ up += connections.getUp(idx); ++ pktcount += connections.getPacketCount(idx); ++ pktloss += connections.getPacketLostCount(idx); ++ pktretrans += connections.getPacketRetransmitCount(idx); ++ } ++ ++ // Convert connector duration to string ++ std::stringstream connectorSummary; ++ bool addDelimiter = false; ++ connectorSummary << "{"; ++ for (std::map::iterator it = connectorLastActive.begin(); ++ it != connectorLastActive.end(); ++it){ ++ if (lastSecond - it->second < 10000){ ++ connectorSummary << (addDelimiter ? "," : "") << it->first; ++ addDelimiter = true; ++ } ++ } ++ connectorSummary << "}"; ++ ++ // Write summary to global statistics ++ sessions.setTime(time + globalTime); ++ sessions.setDown(down + globalDown); ++ sessions.setUp(up + globalUp); ++ sessions.setPacketCount(pktcount + globalPktcount); ++ sessions.setPacketLostCount(pktloss + globalPktloss); ++ sessions.setPacketRetransmitCount(pktretrans + globalPktretrans); ++ sessions.setLastSecond(lastSecond); ++ sessions.setConnector(connectorSummary.str()); ++ sessions.setNow(now); ++ ++ // Retrigger USER_NEW if a re-sync was requested ++ if (!thisType && forceTrigger){ ++ forceTrigger = false; ++ if (Triggers::shouldTrigger("USER_NEW", thisStreamName)){ ++ INFO_MSG("Triggering USER_NEW for stream %s", thisStreamName.c_str()); ++ std::string payload = thisStreamName + "\n" + thisHost + "\n" + ++ thisSid + "\n" + thisProtocol + ++ "\n" + thisReqUrl + "\n" + thisSessionId; ++ if (!Triggers::doTrigger("USER_NEW", payload, thisStreamName)){ ++ INFO_MSG("USER_NEW rejected stream %s", thisStreamName.c_str()); ++ connections.setExit(); ++ connections.finishAll(); ++ }else{ ++ INFO_MSG("USER_NEW accepted stream %s", thisStreamName.c_str()); ++ } ++ } ++ } ++ ++ // Invalidate connections if the session is marked as invalid ++ if(connections.getExit()){ ++ connections.finishAll(); ++ break; ++ } ++ // Remember latest activity so we know when this session ends ++ if (currentConnections){ ++ lastSeen = Util::bootSecs(); ++ } ++ Util::sleep(1000); ++ } ++ ++ // Trigger USER_END ++ if (!thisType && Triggers::shouldTrigger("USER_END", thisStreamName)){ ++ lastSecond = 0; ++ time = 0; ++ down = 0; ++ up = 0; ++ ++ // Get a final summary of this session ++ for (uint64_t idx = 0; idx < connections.recordCount(); idx++){ ++ if (connections.getStatus(idx) == COMM_STATUS_INVALID || connections.getStatus(idx) & COMM_STATUS_DISCONNECT){continue;} ++ uint64_t thisLastSecond = connections.getLastSecond(idx); ++ // Set last second to the latest entry ++ if (thisLastSecond > lastSecond){ ++ lastSecond = thisLastSecond; ++ } ++ // Count protocol durations across the entire session ++ std::string thisConnector = connections.getConnector(idx); ++ if (thisConnector != ""){ ++ connectorCount[thisConnector] += connections.getTime(idx); ++ } ++ // Sum all other variables ++ time += connections.getTime(idx); ++ down += connections.getDown(idx); ++ up += connections.getUp(idx); ++ } ++ ++ // Convert connector duration to string ++ std::stringstream connectorSummary; ++ bool addDelimiter = false; ++ connectorSummary << "{"; ++ for (std::map::iterator it = connectorCount.begin(); ++ it != connectorCount.end(); ++it){ ++ connectorSummary << (addDelimiter ? "," : "") << it->first << ":" << it->second; ++ addDelimiter = true; ++ } ++ connectorSummary << "}"; ++ ++ const uint64_t duration = lastSecond - (bootTime / 1000); ++ std::stringstream summary; ++ summary << thisSessionId << "\n" ++ << thisStreamName << "\n" ++ << connectorSummary.str() << "\n" ++ << thisHost << "\n" ++ << duration << "\n" ++ << up << "\n" ++ << down << "\n" ++ << sessions.getTags(); ++ Triggers::doTrigger("USER_END", summary.str(), thisStreamName); ++ } ++ ++ if (!thisType && connections.getExit()){ ++ WARN_MSG("Session %s has been invalidated since it is not allowed to view stream %s", thisSessionId.c_str(), thisStreamName.c_str()); ++ uint64_t sleepStart = Util::bootSecs(); ++ // Keep session invalidated for 10 minutes, or until the session stops ++ while (config.is_active && sleepStart - Util::bootSecs() < 600){ ++ Util::sleep(1000); ++ } ++ } ++ INFO_MSG("Shutting down session %s", thisSessionId.c_str()); ++ return 0; ++} +-- +2.25.1 + + +From 8ac486b815cac711c434422a1b0955486c28bb68 Mon Sep 17 00:00:00 2001 +From: Marco van Dijk +Date: Wed, 16 Mar 2022 13:46:14 +0100 +Subject: [PATCH 27/38] Completed new sessions system + +Co-authored-by: Thulinma +--- + lib/comms.cpp | 206 ++++++++---- + lib/comms.h | 36 +- + lib/defines.h | 2 +- + lib/hls_support.cpp | 12 +- + lib/http_parser.cpp | 6 +- + lib/http_parser.h | 2 +- + lib/socket.cpp | 10 + + lib/socket.h | 1 + + lib/websocket.cpp | 22 +- + lib/websocket.h | 2 +- + src/controller/controller.cpp | 18 + + src/controller/controller_api.cpp | 11 +- + src/controller/controller_statistics.cpp | 375 +++++++++++++-------- + src/controller/controller_statistics.h | 31 +- + src/controller/controller_storage.cpp | 28 +- + src/controller/controller_storage.h | 1 + + src/input/input.cpp | 21 +- + src/input/input_rtsp.cpp | 3 +- + src/input/input_sdp.cpp | 3 +- + src/input/input_ts.cpp | 3 +- + src/input/input_tsrist.cpp | 2 +- + src/input/input_tsrist.h | 2 +- + src/output/output.cpp | 40 ++- + src/output/output.h | 3 +- + src/output/output_cmaf.cpp | 27 +- + src/output/output_hls.cpp | 28 +- + src/output/output_http.cpp | 51 ++- + src/output/output_http_internal.cpp | 179 +++++----- + src/output/output_http_internal.h | 8 +- + src/output/output_sdp.cpp | 12 + + src/output/output_sdp.h | 2 + + src/output/output_ts.cpp | 12 + + src/output/output_ts.h | 2 + + src/output/output_tsrist.cpp | 16 +- + src/output/output_tsrist.h | 4 +- + src/session.cpp | 410 +++++++++++++---------- + 36 files changed, 981 insertions(+), 610 deletions(-) + +diff --git a/lib/comms.cpp b/lib/comms.cpp +index 85f1e6d7..14faa980 100644 +--- a/lib/comms.cpp ++++ b/lib/comms.cpp +@@ -3,6 +3,7 @@ + #include "comms.h" + #include "defines.h" + #include "encode.h" ++#include "stream.h" + #include "procs.h" + #include "timing.h" + #include +@@ -10,6 +11,34 @@ + #include "config.h" + + namespace Comms{ ++ uint8_t sessionViewerMode = SESS_BUNDLE_DEFAULT_VIEWER; ++ uint8_t sessionInputMode = SESS_BUNDLE_DEFAULT_OTHER; ++ uint8_t sessionOutputMode = SESS_BUNDLE_DEFAULT_OTHER; ++ uint8_t sessionUnspecifiedMode = 0; ++ uint8_t sessionStreamInfoMode = SESS_DEFAULT_STREAM_INFO_MODE; ++ uint8_t tknMode = SESS_TKN_DEFAULT_MODE; ++ ++ /// \brief Refreshes the session configuration if the last update was more than 5 seconds ago ++ void sessionConfigCache(){ ++ static uint64_t lastUpdate = 0; ++ if (Util::bootSecs() > lastUpdate + 5){ ++ VERYHIGH_MSG("Updating session config"); ++ JSON::Value tmpVal = Util::getGlobalConfig("sessionViewerMode"); ++ if (!tmpVal.isNull()){ sessionViewerMode = tmpVal.asInt(); } ++ tmpVal = Util::getGlobalConfig("sessionInputMode"); ++ if (!tmpVal.isNull()){ sessionInputMode = tmpVal.asInt(); } ++ tmpVal = Util::getGlobalConfig("sessionOutputMode"); ++ if (!tmpVal.isNull()){ sessionOutputMode = tmpVal.asInt(); } ++ tmpVal = Util::getGlobalConfig("sessionUnspecifiedMode"); ++ if (!tmpVal.isNull()){ sessionUnspecifiedMode = tmpVal.asInt(); } ++ tmpVal = Util::getGlobalConfig("sessionStreamInfoMode"); ++ if (!tmpVal.isNull()){ sessionStreamInfoMode = tmpVal.asInt(); } ++ tmpVal = Util::getGlobalConfig("tknMode"); ++ if (!tmpVal.isNull()){ tknMode = tmpVal.asInt(); } ++ lastUpdate = Util::bootSecs(); ++ } ++ } ++ + Comms::Comms(){ + index = INVALID_RECORD_INDEX; + currentSize = 0; +@@ -17,7 +46,7 @@ namespace Comms{ + } + + Comms::~Comms(){ +- if (index != INVALID_RECORD_INDEX){ ++ if (index != INVALID_RECORD_INDEX && status){ + setStatus(COMM_STATUS_DISCONNECT | getStatus()); + } + if (master){ +@@ -123,6 +152,10 @@ namespace Comms{ + return; + } + dataAccX = Util::RelAccX(dataPage.mapped); ++ if (dataAccX.isExit()){ ++ dataPage.close(); ++ return; ++ } + fieldAccess(); + if (index == INVALID_RECORD_INDEX || reIssue){ + size_t reqCount = dataAccX.getRCount(); +@@ -170,19 +203,30 @@ namespace Comms{ + + void Sessions::addFields(){ + Connections::addFields(); ++ dataAccX.addField("tags", RAX_STRING, 512); + dataAccX.addField("sessid", RAX_STRING, 80); + } + + void Sessions::nullFields(){ + Connections::nullFields(); + setSessId(""); ++ setTags(""); + } + + void Sessions::fieldAccess(){ + Connections::fieldAccess(); ++ tags = dataAccX.getFieldAccX("tags"); + sessId = dataAccX.getFieldAccX("sessid"); + } + ++ std::string Sessions::getTags() const{return tags.string(index);} ++ std::string Sessions::getTags(size_t idx) const{return (master ? tags.string(idx) : 0);} ++ void Sessions::setTags(std::string _sid){tags.set(_sid, index);} ++ void Sessions::setTags(std::string _sid, size_t idx){ ++ if (!master){return;} ++ tags.set(_sid, idx); ++ } ++ + Users::Users() : Comms(){} + + Users::Users(const Users &rhs) : Comms(){ +@@ -251,31 +295,60 @@ namespace Comms{ + keyNum.set(_keyNum, idx); + } + ++ ++ ++ void Connections::reload(const std::string & sessId, bool _master, bool reIssue){ ++ // Open SEM_SESSION ++ if(!sem){ ++ char semName[NAME_BUFFER_SIZE]; ++ snprintf(semName, NAME_BUFFER_SIZE, SEM_SESSION, sessId.c_str()); ++ sem.open(semName, O_RDWR, ACCESSPERMS, 1); ++ if (!sem){return;} ++ } ++ char userPageName[NAME_BUFFER_SIZE]; ++ snprintf(userPageName, NAME_BUFFER_SIZE, COMMS_SESSIONS, sessId.c_str()); ++ Comms::reload(userPageName, COMMS_SESSIONS_INITSIZE, _master, reIssue); ++ } ++ + /// \brief Claims a spot on the connections page for the input/output which calls this function + /// Starts the MistSession binary for each session, which handles the statistics + /// and the USER_NEW and USER_END triggers + /// \param streamName: Name of the stream the input is providing or an output is making available to viewers + /// \param ip: IP address of the viewer which wants to access streamName. For inputs this value can be set to any value +- /// \param sid: Session ID given by the player or randomly generated ++ /// \param tkn: Session token given by the player or randomly generated + /// \param protocol: Protocol currently in use for this connection +- /// \param sessionMode: Determines how a viewer session is defined: +- // If set to 0, all connections with the same viewer IP and stream name are bundled. +- // If set to 1, all connections with the same viewer IP and player ID are bundled. +- // If set to 2, all connections with the same player ID and stream name are bundled. +- // If set to 3, all connections with the same viewer IP, player ID and stream name are bundled. + /// \param _master: If True, we are reading from this page. If False, we are writing (to our entry) on this page + /// \param reIssue: If True, claim a new entry on this page +- void Connections::reload(std::string streamName, std::string ip, std::string sid, std::string protocol, std::string reqUrl, uint64_t sessionMode, bool _master, bool reIssue){ +- if (sessionMode == 0xFFFFFFFFFFFFFFFFull){ +- FAIL_MSG("The session mode was not initialised properly. Assuming default behaviour of bundling by viewer IP, stream name and player id"); +- sessionMode = SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID; +- } ++ void Connections::reload(const std::string & streamName, const std::string & ip, const std::string & tkn, const std::string & protocol, const std::string & reqUrl, bool _master, bool reIssue){ ++ initialTkn = tkn; ++ uint8_t sessMode = sessionViewerMode; + // Generate a unique session ID for each viewer, input or output +- sessionId = generateSession(streamName, ip, sid, protocol, sessionMode); + if (protocol.size() >= 6 && protocol.substr(0, 6) == "INPUT:"){ +- sessionId = "I" + sessionId; ++ sessMode = sessionInputMode; ++ sessionId = "I" + generateSession(streamName, ip, tkn, protocol, sessMode); + }else if (protocol.size() >= 7 && protocol.substr(0, 7) == "OUTPUT:"){ +- sessionId = "O" + sessionId; ++ sessMode = sessionOutputMode; ++ sessionId = "O" + generateSession(streamName, ip, tkn, protocol, sessMode); ++ }else{ ++ // If the session only contains the HTTP connector, check sessionStreamInfoMode ++ if (protocol.size() == 4 && protocol == "HTTP"){ ++ if (sessionStreamInfoMode == SESS_HTTP_AS_VIEWER){ ++ sessionId = generateSession(streamName, ip, tkn, protocol, sessMode); ++ }else if (sessionStreamInfoMode == SESS_HTTP_AS_OUTPUT){ ++ sessMode = sessionOutputMode; ++ sessionId = "O" + generateSession(streamName, ip, tkn, protocol, sessMode); ++ }else if (sessionStreamInfoMode == SESS_HTTP_DISABLED){ ++ return; ++ }else if (sessionStreamInfoMode == SESS_HTTP_AS_UNSPECIFIED){ ++ // Set sessMode to include all variables when determining the session ID ++ sessMode = sessionUnspecifiedMode; ++ sessionId = "U" + generateSession(streamName, ip, tkn, protocol, sessMode); ++ }else{ ++ sessionId = generateSession(streamName, ip, tkn, protocol, sessMode); ++ } ++ }else{ ++ sessionId = generateSession(streamName, ip, tkn, protocol, sessMode); ++ } + } + char userPageName[NAME_BUFFER_SIZE]; + snprintf(userPageName, NAME_BUFFER_SIZE, COMMS_SESSIONS, sessionId.c_str()); +@@ -283,36 +356,59 @@ namespace Comms{ + if (!_master){ + dataPage.init(userPageName, 0, false, false); + if (!dataPage){ ++ std::string host; ++ Socket::hostBytesToStr(ip.data(), 16, host); + pid_t thisPid; + std::deque args; + args.push_back(Util::getMyPath() + "MistSession"); + args.push_back(sessionId); +- args.push_back("--sessionmode"); +- args.push_back(JSON::Value(sessionMode).asString()); +- args.push_back("--streamname"); +- args.push_back(streamName); +- args.push_back("--ip"); +- args.push_back(ip); +- args.push_back("--sid"); +- args.push_back(sid); +- args.push_back("--protocol"); +- args.push_back(protocol); +- args.push_back("--requrl"); +- args.push_back(reqUrl); ++ ++ // First bit defines whether to include stream name ++ if (sessMode & 0x08){ ++ args.push_back("--streamname"); ++ args.push_back(streamName); ++ }else{ ++ setenv("SESSION_STREAM", streamName.c_str(), 1); ++ } ++ // Second bit defines whether to include viewer ip ++ if (sessMode & 0x04){ ++ args.push_back("--ip"); ++ args.push_back(host); ++ }else{ ++ setenv("SESSION_IP", host.c_str(), 1); ++ } ++ // Third bit defines whether to include tkn ++ if (sessMode & 0x02){ ++ args.push_back("--tkn"); ++ args.push_back(tkn); ++ }else{ ++ setenv("SESSION_TKN", tkn.c_str(), 1); ++ } ++ // Fourth bit defines whether to include protocol ++ if (sessMode & 0x01){ ++ args.push_back("--protocol"); ++ args.push_back(protocol); ++ }else{ ++ setenv("SESSION_PROTOCOL", protocol.c_str(), 1); ++ } ++ setenv("SESSION_REQURL", reqUrl.c_str(), 1); + int err = fileno(stderr); + thisPid = Util::Procs::StartPiped(args, 0, 0, &err); + Util::Procs::forget(thisPid); +- HIGH_MSG("Spawned new session executeable (pid %u) for sessionId '%s', corresponding to host %s and stream %s", thisPid, sessionId.c_str(), ip.c_str(), streamName.c_str()); ++ unsetenv("SESSION_STREAM"); ++ unsetenv("SESSION_IP"); ++ unsetenv("SESSION_TKN"); ++ unsetenv("SESSION_PROTOCOL"); ++ unsetenv("SESSION_REQURL"); + } + } +- // Open SEM_SESSION +- if(!sem){ +- char semName[NAME_BUFFER_SIZE]; +- snprintf(semName, NAME_BUFFER_SIZE, SEM_SESSION, sessionId.c_str()); +- sem.open(semName, O_RDWR, ACCESSPERMS, 1); ++ reload(sessionId, _master, reIssue); ++ if (index != INVALID_RECORD_INDEX){ ++ setConnector(protocol); ++ setHost(ip); ++ setStream(streamName); ++ VERYHIGH_MSG("Reloading connection. Claimed record %lu", index); + } +- Comms::reload(userPageName, COMMS_SESSIONS_INITSIZE, _master, reIssue); +- VERYHIGH_MSG("Reloading connection. Claimed record %lu", index); + } + + /// \brief Marks the data page as closed, so that we longer write any new data to is +@@ -341,7 +437,6 @@ namespace Comms{ + dataAccX.addField("host", RAX_RAW, 16); + dataAccX.addField("stream", RAX_STRING, 100); + dataAccX.addField("connector", RAX_STRING, 20); +- dataAccX.addField("tags", RAX_STRING, 512); + dataAccX.addField("pktcount", RAX_64UINT); + dataAccX.addField("pktloss", RAX_64UINT); + dataAccX.addField("pktretrans", RAX_64UINT); +@@ -349,7 +444,6 @@ namespace Comms{ + + void Connections::nullFields(){ + Comms::nullFields(); +- setTags(""); + setConnector(""); + setStream(""); + setHost(""); +@@ -373,7 +467,6 @@ namespace Comms{ + host = dataAccX.getFieldAccX("host"); + stream = dataAccX.getFieldAccX("stream"); + connector = dataAccX.getFieldAccX("connector"); +- tags = dataAccX.getFieldAccX("tags"); + pktcount = dataAccX.getFieldAccX("pktcount"); + pktloss = dataAccX.getFieldAccX("pktloss"); + pktretrans = dataAccX.getFieldAccX("pktretrans"); +@@ -461,14 +554,6 @@ namespace Comms{ + return false; + } + +- std::string Connections::getTags() const{return tags.string(index);} +- std::string Connections::getTags(size_t idx) const{return (master ? tags.string(idx) : 0);} +- void Connections::setTags(std::string _sid){tags.set(_sid, index);} +- void Connections::setTags(std::string _sid, size_t idx){ +- if (!master){return;} +- tags.set(_sid, idx); +- } +- + uint64_t Connections::getPacketCount() const{return pktcount.uint(index);} + uint64_t Connections::getPacketCount(size_t idx) const{ + return (master ? pktcount.uint(idx) : 0); +@@ -501,31 +586,32 @@ namespace Comms{ + + /// \brief Generates a session ID which is unique per viewer + /// \return generated session ID as string +- std::string Connections::generateSession(std::string streamName, std::string ip, std::string sid, std::string connector, uint64_t sessionMode){ ++ std::string Connections::generateSession(const std::string & streamName, const std::string & ip, const std::string & tkn, const std::string & connector, uint64_t sessionMode){ + std::string concat; ++ std::string debugMsg = "Generating session id based on"; + // First bit defines whether to include stream name +- if (sessionMode > 7){ ++ if (sessionMode & 0x08){ + concat += streamName; +- sessionMode -= 8; ++ debugMsg += " stream name '" + streamName + "'"; + } + // Second bit defines whether to include viewer ip +- if (sessionMode > 3){ ++ if (sessionMode & 0x04){ + concat += ip; +- sessionMode -= 4; ++ std::string ipHex; ++ Socket::hostBytesToStr(ip.c_str(), ip.size(), ipHex); ++ debugMsg += " IP '" + ipHex + "'"; + } +- // Third bit defines whether to include player ip +- if (sessionMode > 1){ +- concat += sid; +- sessionMode -= 2; ++ // Third bit defines whether to include client-side session token ++ if (sessionMode & 0x02){ ++ concat += tkn; ++ debugMsg += " session token '" + tkn + "'"; + } + // Fourth bit defines whether to include protocol +- if (sessionMode == 1){ ++ if (sessionMode & 0x01){ + concat += connector; +- sessionMode = 0; +- } +- if (sessionMode > 0){ +- WARN_MSG("Could not resolve session mode of value %lu", sessionMode); ++ debugMsg += " protocol '" + connector + "'"; + } ++ VERYHIGH_MSG("%s", debugMsg.c_str()); + return Secure::sha256(concat.c_str(), concat.length()); + } + }// namespace Comms +diff --git a/lib/comms.h b/lib/comms.h +index 9a5c0ea9..ec36dcb0 100644 +--- a/lib/comms.h ++++ b/lib/comms.h +@@ -9,13 +9,21 @@ + #define COMM_STATUS_REQDISCONNECT 0x10 + #define COMM_STATUS_ACTIVE 0x1 + #define COMM_STATUS_INVALID 0x0 ++#define SESS_BUNDLE_DEFAULT_VIEWER 14 ++#define SESS_BUNDLE_DEFAULT_OTHER 15 ++#define SESS_DEFAULT_STREAM_INFO_MODE 1 ++#define SESS_HTTP_AS_VIEWER 1 ++#define SESS_HTTP_AS_OUTPUT 2 ++#define SESS_HTTP_DISABLED 3 ++#define SESS_HTTP_AS_UNSPECIFIED 4 ++#define SESS_TKN_DEFAULT_MODE 15 + + + #define COMM_LOOP(comm, onActive, onDisconnect) \ + {\ + for (size_t id = 0; id < comm.recordCount(); id++){\ + if (comm.getStatus(id) == COMM_STATUS_INVALID){continue;}\ +- if (!Util::Procs::isRunning(comm.getPid(id))){\ ++ if (!(comm.getStatus(id) & COMM_STATUS_DISCONNECT) && comm.getPid(id) && !Util::Procs::isRunning(comm.getPid(id))){\ + comm.setStatus(COMM_STATUS_DISCONNECT | comm.getStatus(id), id);\ + }\ + onActive;\ +@@ -27,6 +35,14 @@ + } + + namespace Comms{ ++ extern uint8_t sessionViewerMode; ++ extern uint8_t sessionInputMode; ++ extern uint8_t sessionOutputMode; ++ extern uint8_t sessionUnspecifiedMode; ++ extern uint8_t sessionStreamInfoMode; ++ extern uint8_t tknMode; ++ void sessionConfigCache(); ++ + class Comms{ + public: + Comms(); +@@ -66,11 +82,13 @@ namespace Comms{ + + class Connections : public Comms{ + public: +- void reload(std::string streamName, std::string ip, std::string sid, std::string protocol, std::string reqUrl, uint64_t sessionMode, bool _master = false, bool reIssue = false); ++ void reload(const std::string & streamName, const std::string & ip, const std::string & tkn, const std::string & protocol, const std::string & reqUrl, bool _master = false, bool reIssue = false); ++ void reload(const std::string & sessId, bool _master = false, bool reIssue = false); + void unload(); + operator bool() const{return dataPage.mapped && (master || index != INVALID_RECORD_INDEX);} +- std::string generateSession(std::string streamName, std::string ip, std::string sid, std::string connector, uint64_t sessionMode); ++ std::string generateSession(const std::string & streamName, const std::string & ip, const std::string & tkn, const std::string & connector, uint64_t sessionMode); + std::string sessionId; ++ std::string initialTkn; + + void setExit(); + bool getExit(); +@@ -79,6 +97,8 @@ namespace Comms{ + virtual void nullFields(); + virtual void fieldAccess(); + ++ const std::string & getTkn() const{return initialTkn;} ++ + uint64_t getNow() const; + uint64_t getNow(size_t idx) const; + void setNow(uint64_t _now); +@@ -120,11 +140,6 @@ namespace Comms{ + void setConnector(std::string _connector, size_t idx); + bool hasConnector(size_t idx, std::string protocol); + +- std::string getTags() const; +- std::string getTags(size_t idx) const; +- void setTags(std::string _sid); +- void setTags(std::string _sid, size_t idx); +- + uint64_t getPacketCount() const; + uint64_t getPacketCount(size_t idx) const; + void setPacketCount(uint64_t _count); +@@ -197,5 +212,10 @@ namespace Comms{ + virtual void addFields(); + virtual void nullFields(); + virtual void fieldAccess(); ++ ++ std::string getTags() const; ++ std::string getTags(size_t idx) const; ++ void setTags(std::string _sid); ++ void setTags(std::string _sid, size_t idx); + }; + }// namespace Comms +diff --git a/lib/defines.h b/lib/defines.h +index 203b75db..d08d1d04 100644 +--- a/lib/defines.h ++++ b/lib/defines.h +@@ -231,7 +231,7 @@ static inline void show_stackframe(){} + #define SEM_TRACKLIST "/MstTRKS%s" //%s stream name + #define SEM_SESSION "MstSess%s" + #define SEM_SESSCACHE "/MstSessCacheLock" +-#define SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID 14 ++#define SESS_TIMEOUT 600 // Session timeout in seconds + #define SHM_CAPA "MstCapa" + #define SHM_PROTO "MstProt" + #define SHM_PROXY "MstProx" +diff --git a/lib/hls_support.cpp b/lib/hls_support.cpp +index c311a7ac..130e691d 100644 +--- a/lib/hls_support.cpp ++++ b/lib/hls_support.cpp +@@ -274,7 +274,7 @@ namespace HLS{ + if (trackData.mediaFormat == ".ts"){return;} + + result << "#EXT-X-MAP:URI=\"" << trackData.urlPrefix << "init" << trackData.mediaFormat; +- if (trackData.sessionId.size()){result << "?sessId=" << trackData.sessionId;} ++ if (trackData.sessionId.size()){result << "?tkn=" << trackData.sessionId;} + result << "\"\r\n"; + } + +@@ -327,7 +327,7 @@ namespace HLS{ + result << "?msn=" << fragData.currentFrag; + result << "&mTrack=" << trackData.timingTrackId; + result << "&dur=" << fragData.duration; +- if (trackData.sessionId.size()){result << "&sessId=" << trackData.sessionId;} ++ if (trackData.sessionId.size()){result << "&tkn=" << trackData.sessionId;} + result << "\r\n"; + } + +@@ -341,7 +341,7 @@ namespace HLS{ + result << "?msn=" << fragData.currentFrag; + result << "&mTrack=" << trackData.timingTrackId; + result << "&dur=" << duration; +- if (trackData.sessionId.size()){result << "&sessId=" << trackData.sessionId;} ++ if (trackData.sessionId.size()){result << "&tkn=" << trackData.sessionId;} + result << "\""; + + // NOTE: INDEPENDENT tags, specified ONLY for VIDEO tracks, indicate the first partial fragment +@@ -448,7 +448,7 @@ namespace HLS{ + result << "?msn=" << fragData.currentFrag - 1; + result << "&mTrack=" << trackData.timingTrackId; + result << "&dur=" << partDurationMaxMs; +- if (trackData.sessionId.size()){result << "&sessId=" << trackData.sessionId;} ++ if (trackData.sessionId.size()){result << "&tkn=" << trackData.sessionId;} + result << "\"\r\n"; + } + +@@ -509,7 +509,7 @@ namespace HLS{ + result << ",NAME=\"" << name << "\",URI=\"" << trackId << "/index.m3u8"; + result << "?mTrack=" << masterData.mainTrack; + result << "&iMsn=" << iFrag; +- if (masterData.hasSessId){result << "&sessId=" << masterData.sessId;} ++ if (masterData.sessId.size()){result << "&tkn=" << masterData.sessId;} + if (masterData.noLLHLS){result << "&llhls=0";} + result << "\"\r\n"; + } +@@ -529,7 +529,7 @@ namespace HLS{ + result << "/index.m3u8"; + result << "?mTrack=" << masterData.mainTrack; + result << "&iMsn=" << iFrag; +- if (masterData.hasSessId){result << "&sessId=" << masterData.sessId;} ++ if (masterData.sessId.size()){result << "&tkn=" << masterData.sessId;} + if (masterData.noLLHLS){result << "&llhls=0";} + result << "\r\n"; + } +diff --git a/lib/http_parser.cpp b/lib/http_parser.cpp +index c08e3d56..113fdb51 100644 +--- a/lib/http_parser.cpp ++++ b/lib/http_parser.cpp +@@ -742,13 +742,13 @@ bool HTTP::Parser::parse(std::string &HTTPbuffer, Util::DataCallback &cb){ + + /// HTTP variable parser to std::map structure. + /// Reads variables from data, decodes and stores them to storage. +-void HTTP::parseVars(const std::string &data, std::map &storage){ ++void HTTP::parseVars(const std::string &data, std::map &storage, const std::string & separator){ + std::string varname; + std::string varval; + // position where a part starts (e.g. after &) + size_t pos = 0; + while (pos < data.length()){ +- size_t nextpos = data.find('&', pos); ++ size_t nextpos = data.find(separator, pos); + if (nextpos == std::string::npos){nextpos = data.length();} + size_t eq_pos = data.find('=', pos); + if (eq_pos < nextpos){ +@@ -769,7 +769,7 @@ void HTTP::parseVars(const std::string &data, std::map + break; + } + // erase & +- pos = nextpos + 1; ++ pos = nextpos + separator.size(); + } + } + +diff --git a/lib/http_parser.h b/lib/http_parser.h +index 9c843f20..a5528df3 100644 +--- a/lib/http_parser.h ++++ b/lib/http_parser.h +@@ -14,7 +14,7 @@ namespace HTTP{ + + /// HTTP variable parser to std::map structure. + /// Reads variables from data, decodes and stores them to storage. +- void parseVars(const std::string &data, std::map &storage); ++ void parseVars(const std::string &data, std::map &storage, const std::string & separator = "&"); + + /// Simple class for reading and writing HTTP 1.0 and 1.1. + class Parser : public Util::DataCallback{ +diff --git a/lib/socket.cpp b/lib/socket.cpp +index 6d63af00..47b2b6bd 100644 +--- a/lib/socket.cpp ++++ b/lib/socket.cpp +@@ -166,6 +166,8 @@ bool Socket::isBinAddress(const std::string &binAddr, std::string addr){ + /// Converts the given address with optional subnet to binary IPv6 form. + /// Returns 16 bytes of address, followed by 1 byte of subnet bits, zero or more times. + std::string Socket::getBinForms(std::string addr){ ++ // Check for empty address ++ if (!addr.size()){return std::string(17, (char)0);} + // Check if we need to do prefix matching + uint8_t prefixLen = 128; + if (addr.find('/') != std::string::npos){ +@@ -1796,6 +1798,14 @@ void Socket::UDPConnection::GetDestination(std::string &destIp, uint32_t &port){ + FAIL_MSG("Could not get destination for UDP socket"); + }// Socket::UDPConnection GetDestination + ++/// Gets the properties of the receiving end of this UDP socket. ++/// This will be the receiving end for all SendNow calls. ++std::string Socket::UDPConnection::getBinDestination(){ ++ std::string binList = getIPv6BinAddr(*(sockaddr_in6*)destAddr); ++ if (binList.size() < 16){ return std::string("\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000", 16); } ++ return binList.substr(0, 16); ++}// Socket::UDPConnection GetDestination ++ + /// Returns the port number of the receiving end of this socket. + /// Returns 0 on error. + uint32_t Socket::UDPConnection::getDestPort() const{ +diff --git a/lib/socket.h b/lib/socket.h +index a85fbf39..b369ac84 100644 +--- a/lib/socket.h ++++ b/lib/socket.h +@@ -215,6 +215,7 @@ namespace Socket{ + void setBlocking(bool blocking); + void SetDestination(std::string hostname, uint32_t port); + void GetDestination(std::string &hostname, uint32_t &port); ++ std::string getBinDestination(); + const void * getDestAddr(){return destAddr;} + size_t getDestAddrLen(){return destAddr_size;} + std::string getBoundAddress(); +diff --git a/lib/websocket.cpp b/lib/websocket.cpp +index 05703b55..c82b4cba 100644 +--- a/lib/websocket.cpp ++++ b/lib/websocket.cpp +@@ -73,31 +73,31 @@ namespace HTTP{ + } + + /// Takes an incoming HTTP::Parser request for a Websocket, and turns it into one. +- Websocket::Websocket(Socket::Connection &c, HTTP::Parser &h) : C(c){ ++ Websocket::Websocket(Socket::Connection &c, const HTTP::Parser &req, HTTP::Parser &resp) : C(c){ + frameType = 0; + maskOut = false; +- std::string connHeader = h.GetHeader("Connection"); ++ std::string connHeader = req.GetHeader("Connection"); + Util::stringToLower(connHeader); + if (connHeader.find("upgrade") == std::string::npos){ + FAIL_MSG("Could not negotiate websocket, connection header incorrect (%s).", connHeader.c_str()); + C.close(); + return; + } +- std::string upgradeHeader = h.GetHeader("Upgrade"); ++ std::string upgradeHeader = req.GetHeader("Upgrade"); + Util::stringToLower(upgradeHeader); + if (upgradeHeader != "websocket"){ + FAIL_MSG("Could not negotiate websocket, upgrade header incorrect (%s).", upgradeHeader.c_str()); + C.close(); + return; + } +- if (h.GetHeader("Sec-WebSocket-Version") != "13"){ ++ if (req.GetHeader("Sec-WebSocket-Version") != "13"){ + FAIL_MSG("Could not negotiate websocket, version incorrect (%s).", +- h.GetHeader("Sec-WebSocket-Version").c_str()); ++ req.GetHeader("Sec-WebSocket-Version").c_str()); + C.close(); + return; + } + #ifdef SSL +- std::string client_key = h.GetHeader("Sec-WebSocket-Key"); ++ std::string client_key = req.GetHeader("Sec-WebSocket-Key"); + if (!client_key.size()){ + FAIL_MSG("Could not negotiate websocket, missing key!"); + C.close(); +@@ -105,15 +105,13 @@ namespace HTTP{ + } + #endif + +- h.Clean(); +- h.setCORSHeaders(); +- h.SetHeader("Upgrade", "websocket"); +- h.SetHeader("Connection", "Upgrade"); ++ resp.SetHeader("Upgrade", "websocket"); ++ resp.SetHeader("Connection", "Upgrade"); + #ifdef SSL +- h.SetHeader("Sec-WebSocket-Accept", calculateKeyAccept(client_key)); ++ resp.SetHeader("Sec-WebSocket-Accept", calculateKeyAccept(client_key)); + #endif + // H.SetHeader("Sec-WebSocket-Protocol", "json"); +- h.SendResponse("101", "Websocket away!", C); ++ resp.SendResponse("101", "Websocket away!", C); + } + + /// Loops calling readFrame until the connection is closed, sleeping in between reads if needed. +diff --git a/lib/websocket.h b/lib/websocket.h +index 819f18c9..07861a19 100644 +--- a/lib/websocket.h ++++ b/lib/websocket.h +@@ -7,7 +7,7 @@ + namespace HTTP{ + class Websocket{ + public: +- Websocket(Socket::Connection &c, HTTP::Parser &h); ++ Websocket(Socket::Connection &c, const HTTP::Parser &req, HTTP::Parser &resp); + Websocket(Socket::Connection &c, const HTTP::URL & url, std::map * headers = 0); + Websocket(Socket::Connection &c, bool client); + operator bool() const; +diff --git a/src/controller/controller.cpp b/src/controller/controller.cpp +index 9e17ca6b..972c7ed7 100644 +--- a/src/controller/controller.cpp ++++ b/src/controller/controller.cpp +@@ -309,6 +309,24 @@ int main_loop(int argc, char **argv){ + Controller::Storage["config"]["prometheus"] = Controller::conf.getString("prometheus"); + Controller::Storage["config"]["accesslog"] = Controller::conf.getString("accesslog"); + Controller::normalizeTrustedProxies(Controller::Storage["config"]["trustedproxy"]); ++ if (!Controller::Storage["config"]["sessionViewerMode"]){ ++ Controller::Storage["config"]["sessionViewerMode"] = SESS_BUNDLE_DEFAULT_VIEWER; ++ } ++ if (!Controller::Storage["config"]["sessionInputMode"]){ ++ Controller::Storage["config"]["sessionInputMode"] = SESS_BUNDLE_DEFAULT_OTHER; ++ } ++ if (!Controller::Storage["config"]["sessionOutputMode"]){ ++ Controller::Storage["config"]["sessionOutputMode"] = SESS_BUNDLE_DEFAULT_OTHER; ++ } ++ if (!Controller::Storage["config"]["sessionUnspecifiedMode"]){ ++ Controller::Storage["config"]["sessionUnspecifiedMode"] = 0; ++ } ++ if (!Controller::Storage["config"]["sessionStreamInfoMode"]){ ++ Controller::Storage["config"]["sessionStreamInfoMode"] = SESS_DEFAULT_STREAM_INFO_MODE; ++ } ++ if (!Controller::Storage["config"].isMember("tknMode")){ ++ Controller::Storage["config"]["tknMode"] = SESS_TKN_DEFAULT_MODE; ++ } + Controller::prometheus = Controller::Storage["config"]["prometheus"].asStringRef(); + Controller::accesslog = Controller::Storage["config"]["accesslog"].asStringRef(); + Controller::writeConfig(); +diff --git a/src/controller/controller_api.cpp b/src/controller/controller_api.cpp +index 02491065..4e935aba 100644 +--- a/src/controller/controller_api.cpp ++++ b/src/controller/controller_api.cpp +@@ -188,7 +188,9 @@ void Controller::handleWebSocket(HTTP::Parser &H, Socket::Connection &C){ + std::string logs = H.GetVar("logs"); + std::string accs = H.GetVar("accs"); + bool doStreams = H.GetVar("streams").size(); +- HTTP::Websocket W(C, H); ++ HTTP::Parser req = H; ++ H.Clean(); ++ HTTP::Websocket W(C, req, H); + if (!W){return;} + + IPC::sharedPage shmLogs(SHM_STATE_LOGS, 1024 * 1024); +@@ -594,7 +596,12 @@ void Controller::handleAPICommands(JSON::Value &Request, JSON::Value &Response){ + out["prometheus"] = in["prometheus"]; + Controller::prometheus = out["prometheus"].asStringRef(); + } +- if (in.isMember("sessionMode")){out["sessionMode"] = in["sessionMode"];} ++ if (in.isMember("sessionViewerMode")){out["sessionViewerMode"] = in["sessionViewerMode"];} ++ if (in.isMember("sessionInputMode")){out["sessionInputMode"] = in["sessionInputMode"];} ++ if (in.isMember("sessionOutputMode")){out["sessionOutputMode"] = in["sessionOutputMode"];} ++ if (in.isMember("sessionUnspecifiedMode")){out["sessionUnspecifiedMode"] = in["sessionUnspecifiedMode"];} ++ if (in.isMember("sessionStreamInfoMode")){out["sessionStreamInfoMode"] = in["sessionStreamInfoMode"];} ++ if (in.isMember("tknMode")){out["tknMode"] = in["tknMode"];} + if (in.isMember("defaultStream")){out["defaultStream"] = in["defaultStream"];} + if (in.isMember("location") && in["location"].isObject()){ + out["location"]["lat"] = in["location"]["lat"].asDouble(); +diff --git a/src/controller/controller_statistics.cpp b/src/controller/controller_statistics.cpp +index 6e51f4ee..ffb89897 100644 +--- a/src/controller/controller_statistics.cpp ++++ b/src/controller/controller_statistics.cpp +@@ -58,6 +58,11 @@ static uint64_t cpu_use = 0; + char noBWCountMatches[1717]; + uint64_t bwLimit = 128 * 1024 * 1024; // gigabit default limit + ++const char nullAddress[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; ++static Controller::statLog emptyLogEntry = {0, 0, 0, 0, 0, 0 ,0 ,0, "", nullAddress, ""}; ++bool notEmpty(const Controller::statLog & dta){ ++ return dta.time || dta.firstActive || dta.lastSecond || dta.down || dta.up || dta.streamName.size() || dta.connectors.size(); ++} + + // For server-wide totals. Local to this file only. + struct streamTotals{ +@@ -66,9 +71,11 @@ struct streamTotals{ + uint64_t inputs; + uint64_t outputs; + uint64_t viewers; ++ uint64_t unspecified; + uint64_t currIns; + uint64_t currOuts; + uint64_t currViews; ++ uint64_t currUnspecified; + uint8_t status; + uint64_t viewSeconds; + uint64_t packSent; +@@ -84,6 +91,7 @@ static uint64_t servDownBytes = 0; + static uint64_t servUpOtherBytes = 0; + static uint64_t servDownOtherBytes = 0; + static uint64_t servInputs = 0; ++static uint64_t servUnspecified = 0; + static uint64_t servOutputs = 0; + static uint64_t servViewers = 0; + static uint64_t servSeconds = 0; +@@ -95,17 +103,19 @@ static uint64_t viewSecondsTotal = 0; + // Mapping of streamName -> summary of stream-wide statistics + static std::map streamStats; + +-// If sessId does not exist yet in streamStats, create and init an entry for it +-static void createEmptyStatsIfNeeded(const std::string & sessId){ +- if (streamStats.count(sessId)){return;} +- streamTotals & sT = streamStats[sessId]; ++// If streamName does not exist yet in streamStats, create and init an entry for it ++static void createEmptyStatsIfNeeded(const std::string & streamName){ ++ if (streamStats.count(streamName)){return;} ++ streamTotals & sT = streamStats[streamName]; + sT.upBytes = 0; + sT.downBytes = 0; + sT.inputs = 0; + sT.outputs = 0; + sT.viewers = 0; ++ sT.unspecified = 0; + sT.currIns = 0; + sT.currOuts = 0; ++ sT.currUnspecified = 0; + sT.currViews = 0; + sT.status = 0; + sT.viewSeconds = 0; +@@ -335,15 +345,23 @@ void Controller::SharedMemStats(void *config){ + it->second.currViews = 0; + it->second.currIns = 0; + it->second.currOuts = 0; ++ it->second.currUnspecified = 0; + } + } + // wipe old statistics and set session type counters + if (sessions.size()){ + std::list mustWipe; +- uint64_t cutOffPoint = Util::bootSecs() - STAT_CUTOFF; ++ // Ensure cutOffPoint is either time of boot or 10 minutes ago, whichever is closer. ++ // Prevents wrapping around to high values close to system boot time. ++ uint64_t cutOffPoint = Util::bootSecs(); ++ if (cutOffPoint > STAT_CUTOFF){ ++ cutOffPoint -= STAT_CUTOFF; ++ }else{ ++ cutOffPoint = 0; ++ } + for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ + // This part handles ending sessions, keeping them in cache for now +- if (it->second.getEnd() < cutOffPoint && it->second.newestDataPoint() < cutOffPoint){ ++ if (it->second.getEnd() < cutOffPoint){ + viewSecondsTotal += it->second.getConnTime(); + mustWipe.push_back(it->first); + // Don't count this session as a viewer +@@ -353,19 +371,24 @@ void Controller::SharedMemStats(void *config){ + switch (it->second.getSessType()){ + case SESS_UNSET: break; + case SESS_VIEWER: +- if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){ +- streamStats[it->first].currViews++; ++ if (it->second.hasDataFor(tOut)){ ++ streamStats[it->second.getStreamName()].currViews++; + } + servSeconds += it->second.getConnTime(); + break; + case SESS_INPUT: +- if (it->second.hasDataFor(tIn) && it->second.isViewerOn(tIn)){ +- streamStats[it->first].currIns++; ++ if (it->second.hasDataFor(tIn)){ ++ streamStats[it->second.getStreamName()].currIns++; + } + break; + case SESS_OUTPUT: +- if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){ +- streamStats[it->first].currOuts++; ++ if (it->second.hasDataFor(tOut)){ ++ streamStats[it->second.getStreamName()].currOuts++; ++ } ++ break; ++ case SESS_UNSPECIFIED: ++ if (it->second.hasDataFor(tOut)){ ++ streamStats[it->second.getStreamName()].currUnspecified++; + } + break; + } +@@ -406,6 +429,7 @@ void Controller::SharedMemStats(void *config){ + strmStats->setInt("viewers", it->second.currViews, strmPos); + strmStats->setInt("inputs", it->second.currIns, strmPos); + strmStats->setInt("outputs", it->second.currOuts, strmPos); ++ strmStats->setInt("unspecified", it->second.currUnspecified, strmPos); + ++strmPos; + } + } +@@ -489,47 +513,46 @@ void Controller::killConnections(std::string sessId){ + + /// Updates the given active connection with new stats data. + void Controller::statSession::update(uint64_t index, Comms::Sessions &statComm){ +- if (host == ""){ +- Socket::hostBytesToStr(statComm.getHost(index).data(), 16, host); +- } +- if (streamName == ""){ +- streamName = statComm.getStream(index); +- } +- if (curConnector == ""){ +- curConnector = statComm.getConnector(index); +- } + if (sessId == ""){ + sessId = statComm.getSessId(index); + } +- // Export tags to session +- if (tags.size()){ +- std::stringstream tagStream; +- for (std::set::iterator it = tags.begin(); it != tags.end(); ++it){ +- tagStream << "[" << *it << "]"; ++ ++ if (sessionType == SESS_UNSET){ ++ if (sessId[0] == 'I'){ ++ sessionType = SESS_INPUT; ++ }else if (sessId[0] == 'O'){ ++ sessionType = SESS_OUTPUT; ++ }else if (sessId[0] == 'U'){ ++ sessionType = SESS_UNSPECIFIED; ++ }else{ ++ sessionType = SESS_VIEWER; + } +- statComm.setTags(tagStream.str(), index); + } + ++ uint64_t prevNow = curData.log.size() ? curData.log.rbegin()->first : 0; ++ // only parse last received data, if newer ++ if (prevNow > statComm.getNow(index)){return;}; + long long prevDown = getDown(); + long long prevUp = getUp(); + uint64_t prevPktSent = getPktCount(); + uint64_t prevPktLost = getPktLost(); + uint64_t prevPktRetrans = getPktRetransmit(); ++ uint64_t prevFirstActive = getFirstActive(); ++ + curData.update(statComm, index); +- // store timestamp of first received data, if older +- if (firstSec > statComm.getNow(index)){firstSec = statComm.getNow(index);} +- uint64_t secIncr = 0; +- // store timestamp of last received data, if newer +- if (statComm.getNow(index) > lastSec){ +- lastSec = statComm.getNow(index); +- if (!tracked){ +- tracked = true; +- firstActive = firstSec; +- }else{ +- secIncr = (statComm.getNow(index) - lastSec); ++ const std::string& streamName = getStreamName(); ++ // Export tags to session ++ if (tags.size()){ ++ std::stringstream tagStream; ++ for (std::set::iterator it = tags.begin(); it != tags.end(); ++it){ ++ tagStream << "[" << *it << "]"; + } +- lastSec = statComm.getNow(index); ++ statComm.setTags(tagStream.str(), index); ++ } else { ++ statComm.setTags("", index); + } ++ ++ uint64_t secIncr = prevFirstActive ? (statComm.getNow(index) - prevNow) : 0; + long long currDown = getDown(); + long long currUp = getUp(); + uint64_t currPktSent = getPktCount(); +@@ -537,7 +560,7 @@ void Controller::statSession::update(uint64_t index, Comms::Sessions &statComm){ + uint64_t currPktRetrans = getPktRetransmit(); + if (currUp - prevUp < 0 || currDown - prevDown < 0){ + INFO_MSG("Negative data usage! %lldu/%lldd (u%lld->%lld) in %s over %s, #%" PRIu64, currUp - prevUp, +- currDown - prevDown, prevUp, currUp, streamName.c_str(), curConnector.c_str(), index); ++ currDown - prevDown, prevUp, currUp, streamName.c_str(), curData.log.rbegin()->second.connectors.c_str(), index); + }else{ + if (!noBWCount){ + size_t bwMatchOffset = 0; +@@ -567,40 +590,38 @@ void Controller::statSession::update(uint64_t index, Comms::Sessions &statComm){ + servPackRetrans += currPktRetrans - prevPktRetrans; + } + } +- if (sessionType == SESS_UNSET){ +- if (curConnector.size() >= 5 && curConnector.substr(0, 5) == "INPUT"){ +- ++servInputs; +- createEmptyStatsIfNeeded(streamName); +- streamStats[streamName].inputs++; +- streamStats[streamName].currIns++; +- sessionType = SESS_INPUT; +- }else if (curConnector.size() >= 6 && curConnector.substr(0, 6) == "OUTPUT"){ +- ++servOutputs; +- createEmptyStatsIfNeeded(streamName); +- streamStats[streamName].outputs++; +- streamStats[streamName].currOuts++; +- sessionType = SESS_OUTPUT; +- }else{ +- ++servViewers; +- createEmptyStatsIfNeeded(streamName); +- streamStats[streamName].viewers++; +- streamStats[streamName].currViews++; +- sessionType = SESS_VIEWER; ++ if (!prevFirstActive && streamName.size()){ ++ createEmptyStatsIfNeeded(streamName); ++ switch(sessionType){ ++ case SESS_INPUT: ++ ++servInputs; ++ streamStats[streamName].inputs++; ++ break; ++ case SESS_OUTPUT: ++ ++servOutputs; ++ streamStats[streamName].outputs++; ++ break; ++ case SESS_VIEWER: ++ ++servViewers; ++ streamStats[streamName].viewers++; ++ break; ++ case SESS_UNSPECIFIED: ++ ++servUnspecified; ++ streamStats[streamName].unspecified++; ++ break; ++ case SESS_UNSET: ++ break; + } + } + // Only count connections that are countable + if (noBWCount != 2){ +- if (!streamName.size() || streamName[0] == 0){ +- if (streamStats.count(streamName)){streamStats.erase(streamName);} +- }else{ +- createEmptyStatsIfNeeded(streamName); +- streamStats[streamName].upBytes += currUp - prevUp; +- streamStats[streamName].downBytes += currDown - prevDown; +- streamStats[streamName].packSent += currPktSent - prevPktSent; +- streamStats[streamName].packLoss += currPktLost - prevPktLost; +- streamStats[streamName].packRetrans += currPktRetrans - prevPktRetrans; +- if (sessionType == SESS_VIEWER){streamStats[streamName].viewSeconds += secIncr;} +- } ++ createEmptyStatsIfNeeded(streamName); ++ streamStats[streamName].upBytes += currUp - prevUp; ++ streamStats[streamName].downBytes += currDown - prevDown; ++ streamStats[streamName].packSent += currPktSent - prevPktSent; ++ streamStats[streamName].packLoss += currPktLost - prevPktLost; ++ streamStats[streamName].packRetrans += currPktRetrans - prevPktRetrans; ++ if (sessionType == SESS_VIEWER){streamStats[streamName].viewSeconds += secIncr;} + } + } + +@@ -608,21 +629,10 @@ Controller::sessType Controller::statSession::getSessType(){ + return sessionType; + } + +-Controller::statSession::~statSession(){ +- if (!tracked){return;} +- switch (sessionType){ +- case SESS_INPUT: +- if (streamStats.count(streamName) && streamStats[streamName].currIns){streamStats[streamName].currIns--;} +- break; +- case SESS_OUTPUT: +- if (streamStats.count(streamName) && streamStats[streamName].currOuts){streamStats[streamName].currOuts--;} +- break; +- case SESS_VIEWER: +- if (streamStats.count(streamName) && streamStats[streamName].currViews){streamStats[streamName].currViews--;} +- break; +- default: break; +- } +- uint64_t duration = lastSec - firstActive; ++/// Ends the currently active session by inserting a null datapoint one second after the last datapoint ++void Controller::statSession::finish(){ ++ if (!getFirstActive()){return;} ++ uint64_t duration = getEnd() - getFirstActive(); + if (duration < 1){duration = 1;} + std::stringstream tagStream; + if (tags.size()){ +@@ -630,6 +640,9 @@ Controller::statSession::~statSession(){ + tagStream << "[" << *it << "]"; + } + } ++ const std::string& streamName = getStreamName(); ++ const std::string& curConnector = getConnectors(); ++ const std::string& host = getStrHost(); + Controller::logAccess(sessId, streamName, curConnector, host, duration, getUp(), + getDown(), tagStream.str()); + if (Controller::accesslog.size()){ +@@ -668,74 +681,99 @@ Controller::statSession::~statSession(){ + } + } + } +- tracked = false; +- firstActive = 0; +- firstSec = 0xFFFFFFFFFFFFFFFFull; +- lastSec = 0; +- sessionType = SESS_UNSET; ++ tags.clear(); ++ // Insert null datapoint ++ curData.log[curData.log.rbegin()->first + 1] = emptyLogEntry; + } + + /// Constructs an empty session + Controller::statSession::statSession(){ +- firstActive = 0; +- tracked = false; +- firstSec = 0xFFFFFFFFFFFFFFFFull; +- lastSec = 0; + sessionType = SESS_UNSET; + noBWCount = 0; +- streamName = ""; +- host = ""; +- curConnector = ""; + sessId = ""; + } + + /// Returns the first measured timestamp in this session. + uint64_t Controller::statSession::getStart(){ +- return firstSec; ++ if (!curData.log.size()){return 0;} ++ return curData.log.begin()->first; + } + + /// Returns the last measured timestamp in this session. + uint64_t Controller::statSession::getEnd(){ +- return lastSec; ++ if (!curData.log.size()){return 0;} ++ return curData.log.rbegin()->first; + } + + /// Returns true if there is data for this session at timestamp t. + bool Controller::statSession::hasDataFor(uint64_t t){ +- if (lastSec < t){return false;} +- if (firstSec > t){return false;} + if (curData.hasDataFor(t)){return true;} + return false; + } + +-/// Returns true if this session should count as a viewer on the given timestamp. +-bool Controller::statSession::isViewerOn(uint64_t t){ +- return getUp(t) + getDown(t); ++const std::string& Controller::statSession::getSessId(){ ++ return sessId; ++} ++ ++uint64_t Controller::statSession::getFirstActive(){ ++ if (curData.log.size()){ ++ return curData.log.rbegin()->second.firstActive; ++ } ++ return 0; ++} ++ ++const std::string& Controller::statSession::getStreamName(uint64_t t){ ++ if (curData.hasDataFor(t)){ ++ return curData.getDataFor(t).streamName; ++ } ++ return emptyLogEntry.streamName; + } + +-std::string Controller::statSession::getStreamName(){ +- return streamName; ++const std::string& Controller::statSession::getStreamName(){ ++ if (curData.log.size()){ ++ return curData.log.rbegin()->second.streamName; ++ } ++ return emptyLogEntry.streamName; + } + +-std::string Controller::statSession::getHost(){ ++std::string Controller::statSession::getStrHost(uint64_t t){ ++ std::string host; ++ Socket::hostBytesToStr(getHost(t).data(), 16, host); + return host; + } + +-std::string Controller::statSession::getSessId(){ +- return sessId; ++std::string Controller::statSession::getStrHost(){ ++ std::string host; ++ Socket::hostBytesToStr(getHost().data(), 16, host); ++ return host; ++} ++ ++const std::string& Controller::statSession::getHost(uint64_t t){ ++ if (curData.hasDataFor(t)){ ++ return curData.getDataFor(t).host; ++ } ++ return emptyLogEntry.host; + } + +-std::string Controller::statSession::getCurrentProtocols(){ +- return curConnector; ++const std::string& Controller::statSession::getHost(){ ++ if (curData.log.size()){ ++ return curData.log.rbegin()->second.host; ++ } ++ return emptyLogEntry.host; + } + +-/// Returns true if this session should be considered connected +-uint64_t Controller::statSession::newestDataPoint(){ +- return lastSec; ++const std::string& Controller::statSession::getConnectors(uint64_t t){ ++ if (curData.hasDataFor(t)){ ++ return curData.getDataFor(t).connectors; ++ } ++ return emptyLogEntry.connectors; + } + +-/// Returns true if this session has started (tracked == true) but not yet ended (log entry written) +-bool Controller::statSession::isTracked(){ +- return tracked; ++const std::string& Controller::statSession::getConnectors(){ ++ if (curData.log.size()){ ++ return curData.log.rbegin()->second.connectors; ++ } ++ return emptyLogEntry.connectors; + } + + /// Returns the cumulative connected time for this session at timestamp t. +@@ -842,7 +880,7 @@ uint64_t Controller::statSession::getPktRetransmit(){ + /// Returns the cumulative downloaded bytes per second for this session at timestamp t. + uint64_t Controller::statSession::getBpsDown(uint64_t t){ + uint64_t aTime = t - 5; +- if (aTime < firstSec){aTime = firstSec;} ++ if (aTime < curData.log.begin()->first){aTime = curData.log.begin()->first;} + if (t <= aTime){return 0;} + uint64_t valA = getDown(aTime); + uint64_t valB = getDown(t); +@@ -852,7 +890,7 @@ uint64_t Controller::statSession::getBpsDown(uint64_t t){ + /// Returns the cumulative uploaded bytes per second for this session at timestamp t. + uint64_t Controller::statSession::getBpsUp(uint64_t t){ + uint64_t aTime = t - 5; +- if (aTime < firstSec){aTime = firstSec;} ++ if (aTime < curData.log.begin()->first){aTime = curData.log.begin()->first;} + if (t <= aTime){return 0;} + uint64_t valA = getUp(aTime); + uint64_t valB = getUp(t); +@@ -867,17 +905,8 @@ bool Controller::statStorage::hasDataFor(unsigned long long t){ + + /// Returns a reference to the most current data available at timestamp t. + Controller::statLog &Controller::statStorage::getDataFor(unsigned long long t){ +- static statLog empty; + if (!log.size()){ +- empty.time = 0; +- empty.lastSecond = 0; +- empty.down = 0; +- empty.up = 0; +- empty.pktCount = 0; +- empty.pktLost = 0; +- empty.pktRetransmit = 0; +- empty.connectors = ""; +- return empty; ++ return emptyLogEntry; + } + std::map::iterator it = log.upper_bound(t); + if (it != log.begin()){it--;} +@@ -889,6 +918,11 @@ Controller::statLog &Controller::statStorage::getDataFor(unsigned long long t){ + void Controller::statStorage::update(Comms::Sessions &statComm, size_t index){ + statLog tmp; + tmp.time = statComm.getTime(index); ++ if (!log.size() || !log.rbegin()->second.firstActive){ ++ tmp.firstActive = statComm.getNow(index); ++ } else{ ++ tmp.firstActive = log.rbegin()->second.firstActive; ++ } + tmp.lastSecond = statComm.getLastSecond(index); + tmp.down = statComm.getDown(index); + tmp.up = statComm.getUp(index); +@@ -896,9 +930,19 @@ void Controller::statStorage::update(Comms::Sessions &statComm, size_t index){ + tmp.pktLost = statComm.getPacketLostCount(index); + tmp.pktRetransmit = statComm.getPacketRetransmitCount(index); + tmp.connectors = statComm.getConnector(index); ++ tmp.streamName = statComm.getStream(index); ++ tmp.host = statComm.getHost(index); + log[statComm.getNow(index)] = tmp; + // wipe data older than STAT_CUTOFF seconds +- while (log.size() && log.begin()->first < Util::bootSecs() - STAT_CUTOFF){log.erase(log.begin());} ++ // Ensure cutOffPoint is either time of boot or 10 minutes ago, whichever is closer. ++ // Prevents wrapping around to high values close to system boot time. ++ uint64_t cutOffPoint = Util::bootSecs(); ++ if (cutOffPoint > STAT_CUTOFF){ ++ cutOffPoint -= STAT_CUTOFF; ++ }else{ ++ cutOffPoint = 0; ++ } ++ while (log.size() && log.begin()->first < cutOffPoint){log.erase(log.begin());} + } + + void Controller::statLeadIn(){ +@@ -915,6 +959,7 @@ void Controller::statOnActive(size_t id){ + void Controller::statOnDisconnect(size_t id){ + // Check to see if cleanup is required (when a Session binary fails) + const std::string thisSessionId = statComm.getSessId(id); ++ sessions[thisSessionId].finish(); + // Try to lock to see if the session crashed during boot + IPC::semaphore sessionLock; + char semName[NAME_BUFFER_SIZE]; +@@ -932,7 +977,7 @@ void Controller::statOnDisconnect(size_t id){ + if(dataPage){ + // Session likely crashed while it was running + dataPage.init(userPageName, 1, true); +- FAIL_MSG("Session '%s' got canceled unexpectedly. Hoovering up the left overs...", thisSessionId.c_str()); ++ FAIL_MSG("Session '%s' got cancelled unexpectedly. Cleaning up the leftovers...", thisSessionId.c_str()); + } + // Finally remove the session lock which was created on bootup of the session + sessionLock.unlink(); +@@ -999,13 +1044,23 @@ void Controller::fillClients(JSON::Value &req, JSON::Value &rep){ + if (req.isMember("time")){reqTime = req["time"].asInt();} + // to make sure no nasty timing business takes place, we store the case "now" as a bool. + bool now = (reqTime == 0); +- //if greater than current bootsecs, assume unix time and subtract epoch from it +- if (reqTime > (int64_t)epoch - STAT_CUTOFF){reqTime -= (epoch-bSecs);} ++ //if in the last 600 seconds of unix time (or higher), assume unix time and subtract epoch from it ++ if (reqTime > (int64_t)epoch - STAT_CUTOFF){reqTime -= Controller::systemBoot/1000;} + // add the current time, if negative or zero. + if (reqTime < 0){reqTime += bSecs;} +- if (reqTime == 0){reqTime = bSecs - STAT_CUTOFF;} ++ if (reqTime == 0){ ++ // Ensure cutOffPoint is either time of boot or 10 minutes ago, whichever is closer. ++ // Prevents wrapping around to high values close to system boot time. ++ uint64_t cutOffPoint = bSecs; ++ if (cutOffPoint > STAT_CUTOFF){ ++ cutOffPoint -= STAT_CUTOFF; ++ }else{ ++ cutOffPoint = 0; ++ } ++ reqTime = cutOffPoint; ++ } + // at this point, we have the absolute timestamp in bootsecs. +- rep["time"] = reqTime + (epoch-bSecs); // fill the absolute timestamp ++ rep["time"] = reqTime + (Controller::systemBoot/1000); // fill the absolute timestamp + + unsigned int fields = 0; + // next, figure out the fields wanted +@@ -1062,13 +1117,14 @@ void Controller::fillClients(JSON::Value &req, JSON::Value &rep){ + if (now && reqTime - it->second.getEnd() < 5){time = it->second.getEnd();} + // data present and wanted? insert it! + if ((it->second.getEnd() >= time && it->second.getStart() <= time) && +- (!streams.size() || streams.count(it->second.getStreamName())) && +- (!protos.size() || protos.count(it->second.getCurrentProtocols()))){ +- if (it->second.hasDataFor(time)){ ++ (!streams.size() || streams.count(it->second.getStreamName(time))) && ++ (!protos.size() || protos.count(it->second.getConnectors(time)))){ ++ const statLog & dta = it->second.curData.getDataFor(time); ++ if (notEmpty(dta)){ + JSON::Value d; +- if (fields & STAT_CLI_HOST){d.append(it->second.getHost());} +- if (fields & STAT_CLI_STREAM){d.append(it->second.getStreamName());} +- if (fields & STAT_CLI_PROTO){d.append(it->second.getCurrentProtocols());} ++ if (fields & STAT_CLI_HOST){d.append(it->second.getStrHost(time));} ++ if (fields & STAT_CLI_STREAM){d.append(it->second.getStreamName(time));} ++ if (fields & STAT_CLI_PROTO){d.append(it->second.getConnectors(time));} + if (fields & STAT_CLI_CONNTIME){d.append(it->second.getConnTime(time));} + if (fields & STAT_CLI_POSITION){d.append(it->second.getLastSecond(time));} + if (fields & STAT_CLI_DOWN){d.append(it->second.getDown(time));} +@@ -1252,6 +1308,8 @@ void Controller::fillActive(JSON::Value &req, JSON::Value &rep){ + F = it->second.currIns; + }else if (j->asStringRef() == "outputs"){ + F = it->second.currOuts; ++ }else if (j->asStringRef() == "unspecified"){ ++ F = it->second.currUnspecified; + }else if (j->asStringRef() == "views"){ + F = it->second.viewers; + }else if (j->asStringRef() == "viewseconds"){ +@@ -1323,6 +1381,7 @@ public: + clients = 0; + inputs = 0; + outputs = 0; ++ unspecified = 0; + downbps = 0; + upbps = 0; + pktCount = 0; +@@ -1334,6 +1393,7 @@ public: + case Controller::SESS_VIEWER: clients++; break; + case Controller::SESS_INPUT: inputs++; break; + case Controller::SESS_OUTPUT: outputs++; break; ++ case Controller::SESS_UNSPECIFIED: unspecified++; break; + default: break; + } + downbps += down; +@@ -1345,6 +1405,7 @@ public: + uint64_t clients; + uint64_t inputs; + uint64_t outputs; ++ uint64_t unspecified; + uint64_t downbps; + uint64_t upbps; + uint64_t pktCount; +@@ -1363,11 +1424,21 @@ void Controller::fillTotals(JSON::Value &req, JSON::Value &rep){ + if (req.isMember("start")){reqStart = req["start"].asInt();} + if (req.isMember("end")){reqEnd = req["end"].asInt();} + //if the reqStart or reqEnd is greater than current bootsecs, assume unix time and subtract epoch from it +- if (reqStart > (int64_t)epoch - STAT_CUTOFF){reqStart -= (epoch-bSecs);} +- if (reqEnd > (int64_t)epoch - STAT_CUTOFF){reqEnd -= (epoch-bSecs);} ++ if (reqStart > (int64_t)epoch - STAT_CUTOFF){reqStart -= Controller::systemBoot/1000;} ++ if (reqEnd > (int64_t)epoch - STAT_CUTOFF){reqEnd -= Controller::systemBoot/1000;} + // add the current time, if negative or zero. + if (reqStart < 0){reqStart += bSecs;} +- if (reqStart == 0){reqStart = bSecs - STAT_CUTOFF;} ++ if (reqStart == 0){ ++ // Ensure cutOffPoint is either time of boot or 10 minutes ago, whichever is closer. ++ // Prevents wrapping around to high values close to system boot time. ++ uint64_t cutOffPoint = bSecs; ++ if (cutOffPoint > STAT_CUTOFF){ ++ cutOffPoint -= STAT_CUTOFF; ++ }else{ ++ cutOffPoint = 0; ++ } ++ reqStart = cutOffPoint; ++ } + if (reqEnd <= 0){reqEnd += bSecs;} + // at this point, reqStart and reqEnd are the absolute timestamp in bootsecs. + if (reqEnd < reqStart){reqEnd = reqStart;} +@@ -1417,7 +1488,7 @@ void Controller::fillTotals(JSON::Value &req, JSON::Value &rep){ + if ((it->second.getEnd() >= (unsigned long long)reqStart || + it->second.getStart() <= (unsigned long long)reqEnd) && + (!streams.size() || streams.count(it->second.getStreamName())) && +- (!protos.size() || protos.count(it->second.getCurrentProtocols()))){ ++ (!protos.size() || protos.count(it->second.getConnectors()))){ + for (unsigned long long i = reqStart; i <= reqEnd; ++i){ + if (it->second.hasDataFor(i)){ + totalsCount[i].add(it->second.getBpsDown(i), it->second.getBpsUp(i), it->second.getSessType(), it->second.getPktCount(), it->second.getPktLost(), it->second.getPktRetransmit()); +@@ -1436,8 +1507,8 @@ void Controller::fillTotals(JSON::Value &req, JSON::Value &rep){ + return; + } + // yay! We have data! +- rep["start"] = totalsCount.begin()->first + (epoch-bSecs); +- rep["end"] = totalsCount.rbegin()->first + (epoch-bSecs); ++ rep["start"] = totalsCount.begin()->first + (Controller::systemBoot/1000); ++ rep["end"] = totalsCount.rbegin()->first + (Controller::systemBoot/1000); + rep["data"].null(); + rep["interval"].null(); + uint64_t prevT = 0; +@@ -1506,12 +1577,15 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int + uint32_t totViewers = 0; + uint32_t totInputs = 0; + uint32_t totOutputs = 0; ++ uint32_t totUnspecified = 0; + for (uint64_t idx = 0; idx < statComm.recordCount(); idx++){ + if (statComm.getStatus(idx) == COMM_STATUS_INVALID || statComm.getStatus(idx) & COMM_STATUS_DISCONNECT){continue;} + const std::string thisSessId = statComm.getSessId(idx); + // Count active viewers, inputs, outputs and protocols + if (thisSessId[0] == 'I'){ + totInputs++; ++ }else if (thisSessId[0] == 'U'){ ++ totUnspecified++; + }else if (thisSessId[0] == 'O'){ + totOutputs++; + outputs[statComm.getConnector(idx)]++; +@@ -1602,6 +1676,7 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int + response << "# TYPE mist_sessions_count counter\n"; + response << "mist_sessions_count{sessType=\"viewers\"}" << servViewers << "\n"; + response << "mist_sessions_count{sessType=\"incoming\"}" << servInputs << "\n"; ++ response << "mist_sessions_count{sessType=\"unspecified\"}" << servUnspecified << "\n"; + response << "mist_sessions_count{sessType=\"outgoing\"}" << servOutputs << "\n\n"; + + response << "# HELP mist_bw_total Count of bytes handled since server start, by direction.\n"; +@@ -1637,6 +1712,7 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int + response << "mist_sessions_total{sessType=\"viewers\"}" << totViewers << "\n"; + response << "mist_sessions_total{sessType=\"incoming\"}" << totInputs << "\n"; + response << "mist_sessions_total{sessType=\"outgoing\"}" << totOutputs << "\n"; ++ response << "mist_sessions_total{sessType=\"unspecified\"}" << totUnspecified << "\n"; + response << "mist_sessions_total{sessType=\"cached\"}" << sessions.size() << "\n"; + + response << "\n# HELP mist_viewcount Count of unique viewer sessions since stream start, per " +@@ -1656,6 +1732,8 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int + << it->second.currIns << "\n"; + response << "mist_sessions{stream=\"" << it->first << "\",sessType=\"outgoing\"}" + << it->second.currOuts << "\n"; ++ response << "mist_sessions{stream=\"" << it->first << "\",sessType=\"unspecified\"}" ++ << it->second.currUnspecified << "\n"; + response << "mist_viewcount{stream=\"" << it->first << "\"}" << it->second.viewers << "\n"; + response << "mist_viewseconds{stream=\"" << it->first << "\"} " << it->second.viewSeconds << "\n"; + response << "mist_bw{stream=\"" << it->first << "\",direction=\"up\"}" << it->second.upBytes << "\n"; +@@ -1691,9 +1769,11 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int + resp["curr"].append(totViewers); + resp["curr"].append(totInputs); + resp["curr"].append(totOutputs); ++ resp["curr"].append(totUnspecified); + resp["tot"].append(servViewers); + resp["tot"].append(servInputs); + resp["tot"].append(servOutputs); ++ resp["tot"].append(servUnspecified); + resp["st"].append(bw_up_total); + resp["st"].append(bw_down_total); + resp["bw"].append(servUpBytes); +@@ -1735,6 +1815,7 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int + resp["streams"][it->first]["curr"].append(it->second.currViews); + resp["streams"][it->first]["curr"].append(it->second.currIns); + resp["streams"][it->first]["curr"].append(it->second.currOuts); ++ resp["streams"][it->first]["curr"].append(it->second.currUnspecified); + resp["streams"][it->first]["pkts"].append(it->second.packSent); + resp["streams"][it->first]["pkts"].append(it->second.packLoss); + resp["streams"][it->first]["pkts"].append(it->second.packRetrans); +diff --git a/src/controller/controller_statistics.h b/src/controller/controller_statistics.h +index f798f811..69605942 100644 +--- a/src/controller/controller_statistics.h ++++ b/src/controller/controller_statistics.h +@@ -28,16 +28,19 @@ namespace Controller{ + + struct statLog{ + uint64_t time; ++ uint64_t firstActive; + uint64_t lastSecond; + uint64_t down; + uint64_t up; + uint64_t pktCount; + uint64_t pktLost; + uint64_t pktRetransmit; ++ std::string streamName; ++ std::string host; + std::string connectors; + }; + +- enum sessType{SESS_UNSET = 0, SESS_INPUT, SESS_OUTPUT, SESS_VIEWER}; ++ enum sessType{SESS_UNSET = 0, SESS_INPUT, SESS_OUTPUT, SESS_VIEWER, SESS_UNSPECIFIED}; + + class statStorage{ + public: +@@ -51,34 +54,30 @@ namespace Controller{ + /// Allows for moving of connections to another session. + class statSession{ + private: +- uint64_t firstActive; +- uint64_t firstSec; +- uint64_t lastSec; + sessType sessionType; +- bool tracked; + uint8_t noBWCount; ///< Set to 2 when not to count for external bandwidth +- std::string streamName; +- std::string host; +- std::string curConnector; + std::string sessId; + + public: + statSession(); +- ~statSession(); ++ void finish(); + statStorage curData; + std::set tags; + sessType getSessType(); + void update(uint64_t index, Comms::Sessions &data); + uint64_t getStart(); + uint64_t getEnd(); +- bool isViewerOn(uint64_t time); +- bool isTracked(); + bool hasDataFor(uint64_t time); +- std::string getStreamName(); +- std::string getHost(); +- std::string getSessId(); +- std::string getCurrentProtocols(); +- uint64_t newestDataPoint(); ++ const std::string& getSessId(); ++ const std::string& getStreamName(uint64_t t); ++ const std::string& getStreamName(); ++ std::string getStrHost(uint64_t t); ++ std::string getStrHost(); ++ const std::string& getHost(uint64_t t); ++ const std::string& getHost(); ++ const std::string& getConnectors(uint64_t t); ++ const std::string& getConnectors(); ++ uint64_t getFirstActive(); + uint64_t getConnTime(uint64_t time); + uint64_t getConnTime(); + uint64_t getLastSecond(uint64_t time); +diff --git a/src/controller/controller_storage.cpp b/src/controller/controller_storage.cpp +index bdd52893..532ee4fe 100644 +--- a/src/controller/controller_storage.cpp ++++ b/src/controller/controller_storage.cpp +@@ -196,6 +196,7 @@ namespace Controller{ + rlxStrm->addField("viewers", RAX_64UINT); + rlxStrm->addField("inputs", RAX_64UINT); + rlxStrm->addField("outputs", RAX_64UINT); ++ rlxStrm->addField("unspecified", RAX_64UINT); + rlxStrm->setReady(); + } + rlxStrm->setRCount((1024 * 1024 - rlxStrm->getOffset()) / rlxStrm->getRSize()); +@@ -433,12 +434,17 @@ namespace Controller{ + + // if fields missing, recreate the page + if (globAccX.isReady()){ +- if(globAccX.getFieldAccX("systemBoot")){ ++ if(globAccX.getFieldAccX("systemBoot") && globAccX.getInt("systemBoot")){ + systemBoot = globAccX.getInt("systemBoot"); + } + if(!globAccX.getFieldAccX("defaultStream") + || !globAccX.getFieldAccX("systemBoot") +- || !globAccX.getFieldAccX("sessionMode")){ ++ || !globAccX.getFieldAccX("sessionViewerMode") ++ || !globAccX.getFieldAccX("sessionInputMode") ++ || !globAccX.getFieldAccX("sessionOutputMode") ++ || !globAccX.getFieldAccX("sessionUnspecifiedMode") ++ || !globAccX.getFieldAccX("sessionStreamInfoMode") ++ || !globAccX.getFieldAccX("tknMode")){ + globAccX.setReload(); + globCfg.master = true; + globCfg.close(); +@@ -449,16 +455,24 @@ namespace Controller{ + if (!globAccX.isReady()){ + globAccX.addField("defaultStream", RAX_128STRING); + globAccX.addField("systemBoot", RAX_64UINT); +- globAccX.addField("sessionMode", RAX_64UINT); +- if (!Storage["config"]["sessionMode"]){ +- Storage["config"]["sessionMode"] = SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID; +- } ++ globAccX.addField("sessionViewerMode", RAX_64UINT); ++ globAccX.addField("sessionInputMode", RAX_64UINT); ++ globAccX.addField("sessionOutputMode", RAX_64UINT); ++ globAccX.addField("sessionUnspecifiedMode", RAX_64UINT); ++ globAccX.addField("sessionStreamInfoMode", RAX_64UINT); ++ globAccX.addField("tknMode", RAX_64UINT); + globAccX.setRCount(1); + globAccX.setEndPos(1); + globAccX.setReady(); + } + globAccX.setString("defaultStream", Storage["config"]["defaultStream"].asStringRef()); +- globAccX.setInt("sessionMode", Storage["config"]["sessionMode"].asInt()); ++ globAccX.setInt("sessionViewerMode", Storage["config"]["sessionViewerMode"].asInt()); ++ globAccX.setInt("sessionInputMode", Storage["config"]["sessionInputMode"].asInt()); ++ globAccX.setInt("sessionOutputMode", Storage["config"]["sessionOutputMode"].asInt()); ++ globAccX.setInt("sessionUnspecifiedMode", Storage["config"]["sessionUnspecifiedMode"].asInt()); ++ globAccX.setInt("sessionStreamInfoMode", Storage["config"]["sessionStreamInfoMode"].asInt()); ++ globAccX.setInt("tknMode", Storage["config"]["tknMode"].asInt()); ++ globAccX.setInt("systemBoot", systemBoot); + globCfg.master = false; // leave the page after closing + } + } +diff --git a/src/controller/controller_storage.h b/src/controller/controller_storage.h +index 6339cfd0..974d989b 100644 +--- a/src/controller/controller_storage.h ++++ b/src/controller/controller_storage.h +@@ -16,6 +16,7 @@ namespace Controller{ + extern bool isTerminal; ///< True if connected to a terminal and not a log file. + extern bool isColorized; ///< True if we colorize the output + extern uint64_t logCounter; ///< Count of logged messages since boot ++ extern uint64_t systemBoot; ///< Unix time in milliseconds of system boot + + Util::RelAccX *logAccessor(); + Util::RelAccX *accesslogAccessor(); +diff --git a/src/input/input.cpp b/src/input/input.cpp +index 0524f55b..782dc4ef 100644 +--- a/src/input/input.cpp ++++ b/src/input/input.cpp +@@ -502,6 +502,7 @@ namespace Mist{ + } + + int Input::run(){ ++ Comms::sessionConfigCache(); + if (streamStatus){streamStatus.mapped[0] = STRMSTAT_BOOT;} + checkHeaderTimes(config->getString("input")); + if (needHeader()){ +@@ -623,6 +624,8 @@ namespace Mist{ + /// ~~~~~~~~~~~~~~~ + void Input::serve(){ + users.reload(streamName, true); ++ Comms::Connections statComm; ++ uint64_t startTime = Util::bootSecs(); + + if (!M){ + // Initialize meta page +@@ -636,6 +639,7 @@ namespace Mist{ + meta.setSource(config->getString("input")); + + bool internalOnly = (config->getString("input").find("INTERNAL_ONLY") != std::string::npos); ++ bool isBuffer = (capa["name"].asStringRef() == "Buffer"); + + /*LTS-START*/ + if (Triggers::shouldTrigger("STREAM_READY", config->getString("streamname"))){ +@@ -666,6 +670,18 @@ namespace Mist{ + }else{ + if (connectedUsers && M.getValidTracks().size()){activityCounter = Util::bootSecs();} + } ++ // Connect to stats for INPUT detection ++ if (!internalOnly && !isBuffer){ ++ if (!statComm){statComm.reload(streamName, getConnectedBinHost(), JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "");} ++ if (statComm){ ++ uint64_t now = Util::bootSecs(); ++ statComm.setNow(now); ++ statComm.setStream(streamName); ++ statComm.setTime(now - startTime); ++ statComm.setLastSecond(0); ++ connStats(statComm); ++ } ++ } + // if not shutting down, wait 1 second before looping + if (config->is_active){Util::wait(INPUT_USER_INTERVAL);} + } +@@ -820,7 +836,7 @@ namespace Mist{ + + if (Util::bootSecs() - statTimer > 1){ + // Connect to stats for INPUT detection +- if (!statComm){statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID);} ++ if (!statComm){statComm.reload(streamName, getConnectedBinHost(), JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "");} + if (statComm){ + if (!statComm){ + config->is_active = false; +@@ -830,7 +846,6 @@ namespace Mist{ + uint64_t now = Util::bootSecs(); + statComm.setNow(now); + statComm.setStream(streamName); +- statComm.setConnector("INPUT:" + capa["name"].asStringRef()); + statComm.setTime(now - startTime); + statComm.setLastSecond(0); + connStats(statComm); +@@ -984,7 +999,7 @@ namespace Mist{ + + if (Util::bootSecs() - statTimer > 1){ + // Connect to stats for INPUT detection +- if (!statComm){statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID);} ++ if (!statComm){statComm.reload(streamName, getConnectedBinHost(), JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "");} + if (statComm){ + if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ + config->is_active = false; +diff --git a/src/input/input_rtsp.cpp b/src/input/input_rtsp.cpp +index ebb812e1..25be6e23 100644 +--- a/src/input/input_rtsp.cpp ++++ b/src/input/input_rtsp.cpp +@@ -210,7 +210,7 @@ namespace Mist{ + if (lastSecs != currSecs){ + lastSecs = currSecs; + // Connect to stats for INPUT detection +- statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID); ++ statComm.reload(streamName, getConnectedBinHost(), JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), ""); + if (statComm){ + if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ + config->is_active = false; +@@ -225,7 +225,6 @@ namespace Mist{ + statComm.setDown(tcpCon.dataDown()); + statComm.setTime(now - startTime); + statComm.setLastSecond(0); +- statComm.setHost(getConnectedBinHost()); + } + } + } +diff --git a/src/input/input_sdp.cpp b/src/input/input_sdp.cpp +index 3169a836..0b8ddeb5 100644 +--- a/src/input/input_sdp.cpp ++++ b/src/input/input_sdp.cpp +@@ -202,7 +202,7 @@ namespace Mist{ + if (lastSecs != currSecs){ + lastSecs = currSecs; + // Connect to stats for INPUT detection +- statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID); ++ statComm.reload(streamName, getConnectedBinHost(), JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), ""); + if (statComm){ + if (statComm.getStatus() == COMM_STATUS_REQDISCONNECT){ + config->is_active = false; +@@ -217,7 +217,6 @@ namespace Mist{ + statComm.setUp(bytesUp); + statComm.setTime(now - startTime); + statComm.setLastSecond(0); +- statComm.setHost(getConnectedBinHost()); + } + } + // If the error flag is raised or we are lacking data, try to recover +diff --git a/src/input/input_ts.cpp b/src/input/input_ts.cpp +index 23311dc0..e405e3b6 100644 +--- a/src/input/input_ts.cpp ++++ b/src/input/input_ts.cpp +@@ -621,7 +621,7 @@ namespace Mist{ + // Check for and spawn threads here. + if (Util::bootSecs() - threadCheckTimer > 1){ + // Connect to stats for INPUT detection +- statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID); ++ statComm.reload(streamName, getConnectedBinHost(), JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), ""); + if (statComm){ + if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ + config->is_active = false; +@@ -636,7 +636,6 @@ namespace Mist{ + statComm.setDown(downCounter + tcpCon.dataDown()); + statComm.setTime(now - startTime); + statComm.setLastSecond(0); +- statComm.setHost(getConnectedBinHost()); + } + + std::set activeTracks = liveStream.getActiveTracks(); +diff --git a/src/input/input_tsrist.cpp b/src/input/input_tsrist.cpp +index 5b95aa9f..df658da1 100644 +--- a/src/input/input_tsrist.cpp ++++ b/src/input/input_tsrist.cpp +@@ -284,7 +284,7 @@ namespace Mist{ + } + + +- void inputTSRIST::connStats(Comms::Statistics &statComm){ ++ void inputTSRIST::connStats(Comms::Connections &statComm){ + statComm.setUp(0); + statComm.setDown(downBytes); + statComm.setHost(getConnectedBinHost()); +diff --git a/src/input/input_tsrist.h b/src/input/input_tsrist.h +index 731f9b04..a75f6601 100644 +--- a/src/input/input_tsrist.h ++++ b/src/input/input_tsrist.h +@@ -30,7 +30,7 @@ namespace Mist{ + int64_t timeStampOffset; + uint64_t lastTimeStamp; + +- virtual void connStats(Comms::Statistics &statComm); ++ virtual void connStats(Comms::Connections &statComm); + + struct rist_ctx *receiver_ctx; + +diff --git a/src/output/output.cpp b/src/output/output.cpp +index 50d67549..d525b8ea 100644 +--- a/src/output/output.cpp ++++ b/src/output/output.cpp +@@ -92,7 +92,7 @@ namespace Mist{ + firstTime = 0; + firstPacketTime = 0xFFFFFFFFFFFFFFFFull; + lastPacketTime = 0; +- sid = ""; ++ tkn = ""; + parseData = false; + wantRequest = true; + sought = false; +@@ -111,7 +111,6 @@ namespace Mist{ + lastPushUpdate = 0; + previousFile = ""; + currentFile = ""; +- sessionMode = 0xFFFFFFFFFFFFFFFFull; + + lastRecv = Util::bootSecs(); + if (myConn){ +@@ -230,7 +229,7 @@ namespace Mist{ + bool Output::isReadyForPlay(){ + // If a protocol does not support any codecs, we assume you know what you're doing + if (!capa.isMember("codecs")){return true;} +- if (!isInitialized){initialize();} ++ if (!isInitialized){return false;} + meta.reloadReplacedPagesIfNeeded(); + if (getSupportedTracks().size()){ + size_t minTracks = 2; +@@ -277,6 +276,7 @@ namespace Mist{ + /// Assumes streamName class member has been set already. + /// Will start input if not currently active, calls onFail() if this does not succeed. + void Output::reconnect(){ ++ Comms::sessionConfigCache(); + thisPacket.null(); + if (config->hasOption("noinput") && config->getBool("noinput")){ + Util::sanitizeName(streamName); +@@ -347,11 +347,10 @@ namespace Mist{ + isInitialized = true; + + //Connect to stats reporting, if not connected already +- if (!statComm){ +- statComm.reload(streamName, getConnectedHost(), sid, capa["name"].asStringRef(), reqUrl, sessionMode); +- stats(true); +- } +- ++ stats(true); ++ //Abort if the stats code shut us down just now ++ if (!isInitialized){return;} ++ + //push inputs do not need to wait for stream to be ready for playback + if (isPushing()){return;} + +@@ -1216,7 +1215,7 @@ namespace Mist{ + /// request URL (if any) + /// ~~~~~~~~~~~~~~~ + int Output::run(){ +- sessionMode = Util::getGlobalConfig("sessionMode").asInt(); ++ Comms::sessionConfigCache(); + /*LTS-START*/ + // Connect to file target, if needed + if (isFileTarget()){ +@@ -1257,6 +1256,7 @@ namespace Mist{ + /*LTS-END*/ + DONTEVEN_MSG("MistOut client handler started"); + while (keepGoing() && (wantRequest || parseData)){ ++ Comms::sessionConfigCache(); + if (wantRequest){requestHandler();} + if (parseData){ + if (!isInitialized){ +@@ -1779,27 +1779,35 @@ namespace Mist{ + } + } + +- if (!statComm){statComm.reload(streamName, getConnectedHost(), sid, capa["name"].asStringRef(), reqUrl, sessionMode);} +- if (!statComm){return;} +- if (statComm.getExit()){ ++ // Disable stats for HTTP internal output ++ if (Comms::sessionStreamInfoMode == SESS_HTTP_DISABLED && capa["name"].asStringRef() == "HTTP"){return;} ++ ++ // Set the token to the pid for outputs which do not generate it in the requestHandler ++ if (!tkn.size()){ tkn = JSON::Value(getpid()).asString(); } ++ ++ if (!statComm){ ++ statComm.reload(streamName, getConnectedBinHost(), tkn, getStatsName(), reqUrl); ++ } ++ if (!statComm || statComm.getExit()){ + onFail("Shutting down since this session is not allowed to view this stream"); ++ statComm.unload(); + return; +- } ++ } + + lastStats = now; + + VERYHIGH_MSG("Writing stats: %s, %s, %s, %" PRIu64 ", %" PRIu64, getConnectedHost().c_str(), streamName.c_str(), +- sid.c_str(), myConn.dataUp(), myConn.dataDown()); ++ tkn.c_str(), myConn.dataUp(), myConn.dataDown()); + /*LTS-START*/ + if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ + onFail("Shutting down on controller request"); ++ statComm.unload(); + return; + } + /*LTS-END*/ + statComm.setNow(now); +- statComm.setConnector(getStatsName()); + connStats(now, statComm); +- statComm.setLastSecond(thisPacket ? thisPacket.getTime() : 0); ++ statComm.setLastSecond(thisPacket ? thisPacket.getTime()/1000 : 0); + statComm.setPid(getpid()); + + /*LTS-START*/ +diff --git a/src/output/output.h b/src/output/output.h +index 173b3840..441c06ab 100644 +--- a/src/output/output.h ++++ b/src/output/output.h +@@ -130,8 +130,7 @@ namespace Mist{ + + Comms::Connections statComm; + bool isBlocking; ///< If true, indicates that myConn is blocking. +- std::string sid; ///< Random identifier used to split connections into sessions +- uint64_t sessionMode; ++ std::string tkn; ///< Random identifier used to split connections into sessions + uint64_t nextKeyTime(); + + // stream delaying variables +diff --git a/src/output/output_cmaf.cpp b/src/output/output_cmaf.cpp +index c2fbc5bc..eb7c87da 100644 +--- a/src/output/output_cmaf.cpp ++++ b/src/output/output_cmaf.cpp +@@ -222,24 +222,17 @@ namespace Mist{ + void OutCMAF::sendHlsMasterManifest(){ + selectDefaultTracks(); + +- std::string sessId = ""; +- if (hasSessionIDs()){ +- std::string ua = UA + JSON::Value(getpid()).asString(); +- crc = checksum::crc32(0, ua.data(), ua.size()); +- sessId = JSON::Value(crc).asString(); +- } +- + // check for forced "no low latency" parameter + bool noLLHLS = H.GetVar("llhls").size() ? H.GetVar("llhls") == "0" : false; + + // Populate the struct that will help generate the master playlist + const HLS::MasterData masterData ={ +- hasSessionIDs(), ++ false,//hasSessionIDs, unused + noLLHLS, + hlsMediaFormat == ".ts", + getMainSelectedTrack(), + H.GetHeader("User-Agent"), +- sessId, ++ (Comms::tknMode & 0x04)?tkn:"", + systemBoot, + bootMsOffset, + }; +@@ -261,11 +254,8 @@ namespace Mist{ + + // Chunkpath & Session ID logic + std::string urlPrefix = ""; +- std::string sessId = ""; + if (config->getString("chunkpath").size()){ + urlPrefix = HTTP::URL(config->getString("chunkpath")).link("./" + H.url).link("./").getUrl(); +- }else{ +- sessId = H.GetVar("sessId"); + } + + // check for forced "no low latency" parameter +@@ -279,7 +269,7 @@ namespace Mist{ + noLLHLS, + hlsMediaFormat, + M.getEncryption(requestTid), +- sessId, ++ (Comms::tknMode & 0x04)?tkn:"", + timingTid, + requestTid, + M.biggestFragment(timingTid) / 1000, +@@ -346,6 +336,16 @@ namespace Mist{ + std::string url = H.url.substr(H.url.find('/', 6) + 1); + HTTP::URL req(reqUrl); + ++ ++ if (tkn.size()){ ++ if (Comms::tknMode & 0x08){ ++ const std::string koekjes = H.GetHeader("Cookie"); ++ std::stringstream cookieHeader; ++ cookieHeader << "tkn=" << tkn << "; Max-Age=" << SESS_TIMEOUT; ++ H.SetHeader("Set-Cookie", cookieHeader.str()); ++ } ++ } ++ + // Send a dash manifest for any URL with .mpd in the path + if (req.getExt() == "mpd"){ + sendDashManifest(); +@@ -438,6 +438,7 @@ namespace Mist{ + H.SendResponse("400", "Bad Request: Could not parse the url", myConn); + return; + } ++ + std::string headerData = + CMAF::keyHeader(M, idx, startTime, targetTime, fragmentIndex, false, false); + +diff --git a/src/output/output_hls.cpp b/src/output/output_hls.cpp +index ad810aa7..112e7ee2 100644 +--- a/src/output/output_hls.cpp ++++ b/src/output/output_hls.cpp +@@ -11,7 +11,7 @@ const std::string hlsMediaFormat = ".ts"; + + namespace Mist{ + bool OutHLS::isReadyForPlay(){ +- if (!isInitialized){initialize();} ++ if (!isInitialized){return false;} + meta.reloadReplacedPagesIfNeeded(); + if (!M.getValidTracks().size()){return false;} + uint32_t mainTrack = M.mainTrack(); +@@ -110,25 +110,17 @@ namespace Mist{ + ///\return The master playlist file for (LL)HLS. + void OutHLS::sendHlsMasterManifest(){ + selectDefaultTracks(); +- +- std::string sessId = ""; +- if (hasSessionIDs()){ +- std::string ua = UA + JSON::Value(getpid()).asString(); +- crc = checksum::crc32(0, ua.data(), ua.size()); +- sessId = JSON::Value(crc).asString(); +- } +- + // check for forced "no low latency" parameter + bool noLLHLS = H.GetVar("llhls").size() ? H.GetVar("llhls") == "0" : false; + + // Populate the struct that will help generate the master playlist + const HLS::MasterData masterData ={ +- hasSessionIDs(), ++ false,//hasSessionIDs, unused + noLLHLS, + hlsMediaFormat == ".ts", + getMainSelectedTrack(), + H.GetHeader("User-Agent"), +- sessId, ++ (Comms::tknMode & 0x04)?tkn:"", + systemBoot, + bootMsOffset, + }; +@@ -150,11 +142,8 @@ namespace Mist{ + + // Chunkpath & Session ID logic + std::string urlPrefix = ""; +- std::string sessId = ""; + if (config->getString("chunkpath").size()){ + urlPrefix = HTTP::URL(config->getString("chunkpath")).link("./" + H.url).link("./").getUrl(); +- }else{ +- sessId = H.GetVar("sessId"); + } + + // check for forced "no low latency" parameter +@@ -168,7 +157,7 @@ namespace Mist{ + noLLHLS, + hlsMediaFormat, + M.getEncryption(requestTid), +- sessId, ++ (Comms::tknMode & 0x04)?tkn:"", + timingTid, + requestTid, + M.biggestFragment(timingTid) / 1000, +@@ -226,6 +215,15 @@ namespace Mist{ + bootMsOffset = 0; + if (M.getLive()){bootMsOffset = M.getBootMsOffset();} + ++ if (tkn.size()){ ++ if (Comms::tknMode & 0x08){ ++ const std::string koekjes = H.GetHeader("Cookie"); ++ std::stringstream cookieHeader; ++ cookieHeader << "tkn=" << tkn << "; Max-Age=" << SESS_TIMEOUT; ++ H.SetHeader("Set-Cookie", cookieHeader.str()); ++ } ++ } ++ + if (H.url == "/crossdomain.xml"){ + H.SetHeader("Content-Type", "text/xml"); + H.SetHeader("Server", APPIDENT); +diff --git a/src/output/output_http.cpp b/src/output/output_http.cpp +index f55b974d..ed8fa33b 100644 +--- a/src/output/output_http.cpp ++++ b/src/output/output_http.cpp +@@ -217,7 +217,9 @@ namespace Mist{ + myConn.close(); + return; + } +- if (handler != capa["name"].asStringRef() || H.GetVar("stream") != streamName){ ++ ++ //Check if we need to change binary and/or reconnect ++ if (handler != capa["name"].asStringRef() || H.GetVar("stream") != streamName || (statComm && (statComm.getHost() != getConnectedBinHost() || statComm.getTkn() != tkn))){ + MEDIUM_MSG("Switching from %s (%s) to %s (%s)", capa["name"].asStringRef().c_str(), + streamName.c_str(), handler.c_str(), H.GetVar("stream").c_str()); + streamName = H.GetVar("stream"); +@@ -268,21 +270,32 @@ namespace Mist{ + realTime = 0; + } + } +- // Get session ID cookie or generate a random one if it wasn't set +- if (!sid.size()){ ++ // Read the session token ++ if (Comms::tknMode & 0x01){ ++ // Get session token from the request url ++ if (H.GetVar("tkn") != ""){ ++ tkn = H.GetVar("tkn"); ++ } else if (H.GetVar("sid") != ""){ ++ tkn = H.GetVar("sid"); ++ } else if (H.GetVar("sessId") != ""){ ++ tkn = H.GetVar("sessId"); ++ } ++ } ++ if ((Comms::tknMode & 0x02) && !tkn.size()){ ++ // Get session token from the request cookie + std::map storage; + const std::string koekjes = H.GetHeader("Cookie"); +- HTTP::parseVars(koekjes, storage); +- if (storage.count("sid")){ +- // Get sid cookie, which is used to divide connections into sessions +- sid = storage.at("sid"); +- }else{ +- // Else generate one +- const std::string newSid = UA + JSON::Value(getpid()).asString(); +- sid = JSON::Value(checksum::crc32(0, newSid.data(), newSid.size())).asString(); +- H.SetHeader("sid", sid.c_str()); ++ HTTP::parseVars(koekjes, storage, "; "); ++ if (storage.count("tkn")){ ++ tkn = storage.at("tkn"); + } + } ++ // Generate a session token if it is being sent as a cookie or url parameter and we couldn't read one ++ if (!tkn.size() && Comms::tknMode > 3){ ++ const std::string newTkn = UA + JSON::Value(getpid()).asString(); ++ tkn = JSON::Value(checksum::crc32(0, newTkn.data(), newTkn.size())).asString(); ++ HIGH_MSG("Generated tkn '%s'", tkn.c_str()); ++ } + // Handle upgrade to websocket if the output supports it + std::string upgradeHeader = H.GetHeader("Upgrade"); + Util::stringToLower(upgradeHeader); +@@ -290,7 +303,9 @@ namespace Mist{ + INFO_MSG("Switching to Websocket mode"); + setBlocking(false); + preWebsocketConnect(); +- webSock = new HTTP::Websocket(myConn, H); ++ HTTP::Parser req = H; ++ H.Clean(); ++ webSock = new HTTP::Websocket(myConn, req, H); + if (!(*webSock)){ + delete webSock; + webSock = 0; +@@ -333,6 +348,14 @@ namespace Mist{ + void HTTPOutput::respondHTTP(const HTTP::Parser & req, bool headersOnly){ + //We generally want the CORS headers to be set for all responses + H.setCORSHeaders(); ++ H.SetHeader("Server", APPIDENT); ++ if (tkn.size()){ ++ if (Comms::tknMode & 0x08){ ++ std::stringstream cookieHeader; ++ cookieHeader << "tkn=" << tkn << "; Max-Age=" << SESS_TIMEOUT; ++ H.SetHeader("Set-Cookie", cookieHeader.str()); ++ } ++ } + //Set attachment header to force download, if applicable + if (req.GetVar("dl").size()){ + //If we want to download, and the string contains a dot, use as-is. +@@ -395,6 +418,8 @@ namespace Mist{ + ///\brief Handles requests by starting a corresponding output process. + ///\param connector The type of connector to be invoked. + void HTTPOutput::reConnector(std::string &connector){ ++ // Clear tkn in order to deal with reverse proxies ++ tkn = ""; + // taken from CheckProtocols (controller_connectors.cpp) + char *argarr[32]; + for (int i = 0; i < 32; i++){argarr[i] = 0;} +diff --git a/src/output/output_http_internal.cpp b/src/output/output_http_internal.cpp +index 97ca455f..be955196 100644 +--- a/src/output/output_http_internal.cpp ++++ b/src/output/output_http_internal.cpp +@@ -76,11 +76,11 @@ namespace Mist{ + std::string method = H.method; + // send logo icon + if (H.url.length() > 4 && H.url.substr(H.url.length() - 4, 4) == ".ico"){ +- sendIcon(); ++ sendIcon(false); + return; + } + if (H.url.length() > 6 && H.url.substr(H.url.length() - 5, 5) == ".html"){ +- HTMLResponse(); ++ HTMLResponse(H, false); + return; + } + if (H.url.size() >= 3 && H.url.substr(H.url.size() - 3) == ".js"){ +@@ -337,9 +337,9 @@ namespace Mist{ + } + } + +- void OutHTTP::HTMLResponse(){ +- std::string method = H.method; +- HTTP::URL fullURL(H.GetHeader("Host")); ++ void OutHTTP::HTMLResponse(const HTTP::Parser & req, bool headersOnly){ ++ HTTPOutput::respondHTTP(req, headersOnly); ++ HTTP::URL fullURL(req.GetHeader("Host")); + if (!fullURL.protocol.size()){fullURL.protocol = getProtocolForPort(fullURL.getPort());} + if (config->getString("pubaddr") != ""){ + HTTP::URL altURL(config->getString("pubaddr")); +@@ -349,24 +349,22 @@ namespace Mist{ + fullURL.path = altURL.path; + } + if (mistPath.size()){fullURL = mistPath;} +- std::string uAgent = H.GetHeader("User-Agent"); ++ std::string uAgent = req.GetHeader("User-Agent"); + + std::string forceType = ""; + if (H.GetVar("forcetype").size()){ +- forceType = ",forceType:\"" + H.GetVar("forcetype") + "\""; ++ forceType = ",forceType:\"" + req.GetVar("forcetype") + "\""; + } + + std::string devSkin = ""; +- if (H.GetVar("dev").size()){devSkin = ",skin:\"dev\"";} +- H.SetVar("stream", ""); +- H.SetVar("dev", ""); ++ if (req.GetVar("dev").size()){devSkin = ",skin:\"dev\"";} + devSkin += ",urlappend:\"" + H.allVars() + "\""; + H.SetVar("stream", streamName); + + std::string seekTo = ""; +- if (H.GetVar("t").size()){ ++ if (req.GetVar("t").size()){ + uint64_t autoSeekTime = 0; +- std::string sTime = H.GetVar("t"); ++ std::string sTime = req.GetVar("t"); + unsigned long long h = 0, m = 0, s = 0; + autoSeekTime = JSON::Value(sTime).asInt(); + if (sscanf(sTime.c_str(), "%llum%llus", &m, &s) == 2){autoSeekTime = m * 60 + s;} +@@ -385,13 +383,10 @@ namespace Mist{ + streamName + "\").addEventListener(\"initialized\",f);"; + } + } +- +- H.Clean(); ++ + H.SetHeader("Content-Type", "text/html"); + H.SetHeader("X-UA-Compatible", "IE=edge"); +- H.SetHeader("Server", APPIDENT); +- H.setCORSHeaders(); +- if (method == "OPTIONS" || method == "HEAD"){ ++ if (headersOnly){ + H.SendResponse("200", "OK", myConn); + responded = true; + H.Clean(); +@@ -427,6 +422,7 @@ namespace Mist{ + } + H.SendResponse("200", "OK", myConn); + responded = true; ++ H.Clean(); + } + + JSON::Value OutHTTP::getStatusJSON(std::string &reqHost, const std::string &useragent){ +@@ -634,23 +630,31 @@ namespace Mist{ + + // loop over the added sources, add them to json_resp["sources"] + for (std::set::iterator it = sources.begin(); it != sources.end(); it++){ +- if ((*it)["simul_tracks"].asInt() > 0){json_resp["source"].append(*it);} ++ if ((*it)["simul_tracks"].asInt() > 0){ ++ if (Comms::tknMode & 0x04){ ++ JSON::Value tmp; ++ tmp = (*it); ++ tmp["url"] = tmp["url"].asStringRef() + "?tkn=" + tkn; ++ tmp["relurl"] = tmp["relurl"].asStringRef() + "?tkn=" + tkn; ++ json_resp["source"].append(tmp); ++ }else{ ++ json_resp["source"].append(*it); ++ } ++ } + } + return json_resp; + } + +- void OutHTTP::onHTTP(){ ++ void OutHTTP::respondHTTP(const HTTP::Parser & req, bool headersOnly){ + origStreamName = streamName; +- std::string method = H.method; + +- if (H.GetHeader("X-Mst-Path").size()){mistPath = H.GetHeader("X-Mst-Path");} ++ if (req.GetHeader("X-Mst-Path").size()){mistPath = req.GetHeader("X-Mst-Path");} + + // Handle certbot validations +- if (H.url.substr(0, 28) == "/.well-known/acme-challenge/"){ ++ if (req.url.substr(0, 28) == "/.well-known/acme-challenge/"){ + std::string cbToken = H.url.substr(28); + jsonForEach(config->getOption("certbot", true), it){ + if (it->asStringRef().substr(0, cbToken.size() + 1) == cbToken + ":"){ +- H.Clean(); + H.SetHeader("Content-Type", "text/plain"); + H.SetHeader("Server", APPIDENT); + H.setCORSHeaders(); +@@ -661,9 +665,7 @@ namespace Mist{ + return; + } + } +- H.Clean(); + H.SetHeader("Content-Type", "text/plain"); +- H.SetHeader("Server", APPIDENT); + H.setCORSHeaders(); + H.SetBody("No matching validation found for token '" + cbToken + "'"); + H.SendResponse("404", "Not found", myConn); +@@ -672,12 +674,11 @@ namespace Mist{ + return; + } + +- if (H.url == "/crossdomain.xml"){ +- H.Clean(); ++ if (req.url == "/crossdomain.xml"){ + H.SetHeader("Content-Type", "text/xml"); + H.SetHeader("Server", APPIDENT); + H.setCORSHeaders(); +- if (method == "OPTIONS" || method == "HEAD"){ ++ if (headersOnly){ + H.SendResponse("200", "OK", myConn); + responded = true; + H.Clean(); +@@ -693,12 +694,11 @@ namespace Mist{ + return; + }// crossdomain.xml + +- if (H.url == "/clientaccesspolicy.xml"){ +- H.Clean(); ++ if (req.url == "/clientaccesspolicy.xml"){ + H.SetHeader("Content-Type", "text/xml"); + H.SetHeader("Server", APPIDENT); + H.setCORSHeaders(); +- if (method == "OPTIONS" || method == "HEAD"){ ++ if (headersOnly){ + H.SendResponse("200", "OK", myConn); + responded = true; + H.Clean(); +@@ -716,8 +716,7 @@ namespace Mist{ + return; + }// clientaccesspolicy.xml + +- if (H.url == "/flashplayer.swf"){ +- H.Clean(); ++ if (req.url == "/flashplayer.swf"){ + H.SetHeader("Content-Type", "application/x-shockwave-flash"); + H.SetHeader("Server", APPIDENT); + H.SetBody((const char *)FlashMediaPlayback_101_swf, FlashMediaPlayback_101_swf_len); +@@ -725,8 +724,7 @@ namespace Mist{ + responded = true; + return; + } +- if (H.url == "/oldflashplayer.swf"){ +- H.Clean(); ++ if (req.url == "/oldflashplayer.swf"){ + H.SetHeader("Content-Type", "application/x-shockwave-flash"); + H.SetHeader("Server", APPIDENT); + H.SetBody((const char *)FlashMediaPlayback_swf, FlashMediaPlayback_swf_len); +@@ -735,20 +733,21 @@ namespace Mist{ + return; + } + // send logo icon +- if (H.url.length() > 4 && H.url.substr(H.url.length() - 4, 4) == ".ico"){ +- sendIcon(); ++ if (req.url.length() > 4 && req.url.substr(req.url.length() - 4, 4) == ".ico"){ ++ sendIcon(headersOnly); + return; + } + + // send generic HTML page +- if (H.url.length() > 6 && H.url.substr(H.url.length() - 5, 5) == ".html"){ +- HTMLResponse(); ++ if (req.url.length() > 6 && req.url.substr(req.url.length() - 5, 5) == ".html"){ ++ HTMLResponse(req, headersOnly); + return; + } + + // send smil MBR index +- if (H.url.length() > 6 && H.url.substr(H.url.length() - 5, 5) == ".smil"){ +- std::string reqHost = HTTP::URL(H.GetHeader("Host")).host; ++ if (req.url.length() > 6 && req.url.substr(req.url.length() - 5, 5) == ".smil"){ ++ HTTPOutput::respondHTTP(req, headersOnly); ++ std::string reqHost = HTTP::URL(req.GetHeader("Host")).host; + std::string port, url_rel; + std::string trackSources; // this string contains all track sources for MBR smil + { +@@ -782,11 +781,8 @@ namespace Mist{ + } + } + +- H.Clean(); + H.SetHeader("Content-Type", "application/smil"); +- H.SetHeader("Server", APPIDENT); +- H.setCORSHeaders(); +- if (method == "OPTIONS" || method == "HEAD"){ ++ if (headersOnly){ + H.SendResponse("200", "OK", myConn); + responded = true; + H.Clean(); +@@ -800,24 +796,22 @@ namespace Mist{ + return; + } + +- if ((H.url.length() > 9 && H.url.substr(0, 6) == "/info_" && H.url.substr(H.url.length() - 3, 3) == ".js") || +- (H.url.length() > 9 && H.url.substr(0, 6) == "/json_" && H.url.substr(H.url.length() - 3, 3) == ".js")){ +- if (websocketHandler()){return;} +- std::string reqHost = HTTP::URL(H.GetHeader("Host")).host; +- std::string useragent = H.GetVar("ua"); +- if (!useragent.size()){useragent = H.GetHeader("User-Agent");} ++ if ((req.url.length() > 9 && req.url.substr(0, 6) == "/info_" && req.url.substr(req.url.length() - 3, 3) == ".js") || ++ (req.url.length() > 9 && req.url.substr(0, 6) == "/json_" && req.url.substr(req.url.length() - 3, 3) == ".js")){ ++ HTTPOutput::respondHTTP(req, headersOnly); ++ if (websocketHandler(req, headersOnly)){return;} ++ std::string reqHost = HTTP::URL(req.GetHeader("Host")).host; ++ std::string useragent = req.GetVar("ua"); ++ if (!useragent.size()){useragent = req.GetHeader("User-Agent");} + std::string response; +- std::string rURL = H.url; +- if (method != "OPTIONS" && method != "HEAD"){initialize();} +- H.Clean(); +- H.SetHeader("Server", APPIDENT); +- H.setCORSHeaders(); ++ std::string rURL = req.url; ++ if (headersOnly){initialize();} + if (rURL.substr(0, 6) != "/json_"){ + H.SetHeader("Content-Type", "application/javascript"); + }else{ + H.SetHeader("Content-Type", "application/json"); + } +- if (method == "OPTIONS" || method == "HEAD"){ ++ if (headersOnly){ + H.SendResponse("200", "OK", myConn); + responded = true; + H.Clean(); +@@ -837,9 +831,9 @@ namespace Mist{ + return; + }// embed code generator + +- if ((H.url == "/player.js") || ((H.url.substr(0, 7) == "/embed_") && (H.url.length() > 10) && +- (H.url.substr(H.url.length() - 3, 3) == ".js"))){ +- HTTP::URL fullURL(H.GetHeader("Host")); ++ if ((req.url == "/player.js") || ((req.url.substr(0, 7) == "/embed_") && (req.url.length() > 10) && ++ (req.url.substr(H.url.length() - 3, 3) == ".js"))){ ++ HTTP::URL fullURL(req.GetHeader("Host")); + if (!fullURL.protocol.size()){fullURL.protocol = getProtocolForPort(fullURL.getPort());} + if (config->getString("pubaddr") != ""){ + HTTP::URL altURL(config->getString("pubaddr")); +@@ -850,12 +844,17 @@ namespace Mist{ + } + if (mistPath.size()){fullURL = mistPath;} + std::string response; +- std::string rURL = H.url; +- H.Clean(); ++ std::string rURL = req.url; ++ ++ if ((rURL.substr(0, 7) == "/embed_") && (rURL.length() > 10) && ++ (rURL.substr(rURL.length() - 3, 3) == ".js")){ ++ HTTPOutput::respondHTTP(req, headersOnly); ++ } ++ + H.SetHeader("Server", APPIDENT); + H.setCORSHeaders(); + H.SetHeader("Content-Type", "application/javascript; charset=utf-8"); +- if (method == "OPTIONS" || method == "HEAD"){ ++ if (headersOnly){ + H.SendResponse("200", "OK", myConn); + responded = true; + H.Clean(); +@@ -933,14 +932,13 @@ namespace Mist{ + return; + } + +- if (H.url.substr(0, 7) == "/skins/"){ ++ if (req.url.substr(0, 7) == "/skins/"){ + std::string response; +- std::string url = H.url; +- H.Clean(); ++ std::string url = req.url; + H.SetHeader("Server", APPIDENT); + H.setCORSHeaders(); + H.SetHeader("Content-Type", "text/css"); +- if (method == "OPTIONS" || method == "HEAD"){ ++ if (headersOnly){ + H.SendResponse("200", "OK", myConn); + responded = true; + H.Clean(); +@@ -970,13 +968,12 @@ namespace Mist{ + H.Clean(); + return; + } +- if (H.url == "/videojs.js"){ ++ if (req.url == "/videojs.js"){ + std::string response; +- H.Clean(); + H.SetHeader("Server", APPIDENT); + H.setCORSHeaders(); + H.SetHeader("Content-Type", "application/javascript"); +- if (method == "OPTIONS" || method == "HEAD"){ ++ if (headersOnly){ + H.SendResponse("200", "OK", myConn); + responded = true; + H.Clean(); +@@ -992,13 +989,12 @@ namespace Mist{ + H.Clean(); + return; + } +- if (H.url == "/dashjs.js"){ ++ if (req.url == "/dashjs.js"){ + std::string response; +- H.Clean(); + H.SetHeader("Server", APPIDENT); + H.setCORSHeaders(); + H.SetHeader("Content-Type", "application/javascript"); +- if (method == "OPTIONS" || method == "HEAD"){ ++ if (headersOnly){ + H.SendResponse("200", "OK", myConn); + responded = true; + H.Clean(); +@@ -1016,13 +1012,12 @@ namespace Mist{ + H.Clean(); + return; + } +- if (H.url == "/webrtc.js"){ ++ if (req.url == "/webrtc.js"){ + std::string response; +- H.Clean(); + H.SetHeader("Server", APPIDENT); + H.setCORSHeaders(); + H.SetHeader("Content-Type", "application/javascript"); +- if (method == "OPTIONS" || method == "HEAD"){ ++ if (headersOnly){ + H.SendResponse("200", "OK", myConn); + responded = true; + H.Clean(); +@@ -1038,13 +1033,12 @@ namespace Mist{ + H.Clean(); + return; + } +- if (H.url == "/flv.js"){ ++ if (req.url == "/flv.js"){ + std::string response; +- H.Clean(); + H.SetHeader("Server", "MistServer/" PACKAGE_VERSION); + H.setCORSHeaders(); + H.SetHeader("Content-Type", "application/javascript"); +- if (method == "OPTIONS" || method == "HEAD"){ ++ if (headersOnly){ + H.SendResponse("200", "OK", myConn); + H.Clean(); + return; +@@ -1058,13 +1052,12 @@ namespace Mist{ + H.Clean(); + return; + } +- if (H.url == "/hlsjs.js"){ ++ if (req.url == "/hlsjs.js"){ + std::string response; +- H.Clean(); + H.SetHeader("Server", "MistServer/" PACKAGE_VERSION); + H.setCORSHeaders(); + H.SetHeader("Content-Type", "application/javascript"); +- if (method == "OPTIONS" || method == "HEAD"){ ++ if (headersOnly){ + H.SendResponse("200", "OK", myConn); + H.Clean(); + return; +@@ -1084,7 +1077,7 @@ namespace Mist{ + H.SetHeader("Server", "MistServer/" PACKAGE_VERSION); + H.setCORSHeaders(); + H.SetHeader("Content-Type", "application/javascript"); +- if (method == "OPTIONS" || method == "HEAD"){ ++ if (headersOnly){ + H.SendResponse("200", "OK", myConn); + H.Clean(); + return; +@@ -1100,15 +1093,13 @@ namespace Mist{ + } + } + +- void OutHTTP::sendIcon(){ +- std::string method = H.method; +- H.Clean(); ++ void OutHTTP::sendIcon(bool headersOnly){ + #include "../icon.h" + H.SetHeader("Content-Type", "image/x-icon"); + H.SetHeader("Server", APPIDENT); + H.SetHeader("Content-Length", icon_len); + H.setCORSHeaders(); +- if (method == "OPTIONS" || method == "HEAD"){ ++ if (headersOnly){ + H.SendResponse("200", "OK", myConn); + responded = true; + H.Clean(); +@@ -1120,16 +1111,16 @@ namespace Mist{ + H.Clean(); + } + +- bool OutHTTP::websocketHandler(){ ++ bool OutHTTP::websocketHandler(const HTTP::Parser & req, bool headersOnly){ + stayConnected = true; +- std::string reqHost = HTTP::URL(H.GetHeader("Host")).host; +- if (H.GetHeader("X-Mst-Path").size()){mistPath = H.GetHeader("X-Mst-Path");} +- std::string useragent = H.GetVar("ua"); +- if (!useragent.size()){useragent = H.GetHeader("User-Agent");} +- std::string upgradeHeader = H.GetHeader("Upgrade"); ++ std::string reqHost = HTTP::URL(req.GetHeader("Host")).host; ++ if (req.GetHeader("X-Mst-Path").size()){mistPath = req.GetHeader("X-Mst-Path");} ++ std::string useragent = req.GetVar("ua"); ++ if (!useragent.size()){useragent = req.GetHeader("User-Agent");} ++ std::string upgradeHeader = req.GetHeader("Upgrade"); + Util::stringToLower(upgradeHeader); + if (upgradeHeader != "websocket"){return false;} +- HTTP::Websocket ws(myConn, H); ++ HTTP::Websocket ws(myConn, req, H); + if (!ws){return false;} + setBlocking(false); + // start the stream, if needed +diff --git a/src/output/output_http_internal.h b/src/output/output_http_internal.h +index 774eb186..8e610145 100644 +--- a/src/output/output_http_internal.h ++++ b/src/output/output_http_internal.h +@@ -10,10 +10,10 @@ namespace Mist{ + virtual void onFail(const std::string &msg, bool critical = false); + /// preHTTP is disabled in the internal HTTP output, since most don't need the stream alive to work + virtual void preHTTP(){}; +- void HTMLResponse(); +- void onHTTP(); +- void sendIcon(); +- bool websocketHandler(); ++ void HTMLResponse(const HTTP::Parser & req, bool headersOnly); ++ void respondHTTP(const HTTP::Parser & req, bool headersOnly); ++ void sendIcon(bool headersOnly); ++ bool websocketHandler(const HTTP::Parser & req, bool headersOnly); + JSON::Value getStatusJSON(std::string &reqHost, const std::string &useragent = ""); + bool stayConnected; + virtual bool onFinish(){return stayConnected;} +diff --git a/src/output/output_sdp.cpp b/src/output/output_sdp.cpp +index 6baf9067..50d491e8 100644 +--- a/src/output/output_sdp.cpp ++++ b/src/output/output_sdp.cpp +@@ -141,6 +141,18 @@ namespace Mist{ + } + } + ++ std::string OutSDP::getConnectedHost(){ ++ if (!sdpState.tracks.size()) { return Output::getConnectedHost(); } ++ std::string hostname; ++ uint32_t port; ++ sdpState.tracks[0].data.GetDestination(hostname, port); ++ return hostname; ++ } ++ std::string OutSDP::getConnectedBinHost(){ ++ if (!sdpState.tracks.size()) { return Output::getConnectedBinHost(); } ++ return sdpState.tracks[0].data.getBinDestination(); ++ } ++ + void OutSDP::sendNext(){ + char *dataPointer = 0; + size_t dataLen = 0; +diff --git a/src/output/output_sdp.h b/src/output/output_sdp.h +index 2d4b3368..6c466753 100644 +--- a/src/output/output_sdp.h ++++ b/src/output/output_sdp.h +@@ -16,6 +16,8 @@ namespace Mist{ + void sendNext(); + void sendHeader(); + bool onFinish(); ++ std::string getConnectedHost(); ++ std::string getConnectedBinHost(); + + private: + void initTracks(uint32_t & port, std::string targetIP); +diff --git a/src/output/output_ts.cpp b/src/output/output_ts.cpp +index 561bf9aa..bf46663b 100644 +--- a/src/output/output_ts.cpp ++++ b/src/output/output_ts.cpp +@@ -239,6 +239,18 @@ namespace Mist{ + } + } + ++ std::string OutTS::getConnectedHost(){ ++ if (!pushOut) { return Output::getConnectedHost(); } ++ std::string hostname; ++ uint32_t port; ++ pushSock.GetDestination(hostname, port); ++ return hostname; ++ } ++ std::string OutTS::getConnectedBinHost(){ ++ if (!pushOut) { return Output::getConnectedBinHost(); } ++ return pushSock.getBinDestination(); ++ } ++ + bool OutTS::listenMode(){return !(config->getString("target").size());} + + void OutTS::onRequest(){ +diff --git a/src/output/output_ts.h b/src/output/output_ts.h +index 32aa6958..0c38cc70 100644 +--- a/src/output/output_ts.h ++++ b/src/output/output_ts.h +@@ -12,6 +12,8 @@ namespace Mist{ + virtual void initialSeek(); + bool isReadyForPlay(); + void onRequest(); ++ std::string getConnectedHost(); ++ std::string getConnectedBinHost(); + + private: + size_t udpSize; +diff --git a/src/output/output_tsrist.cpp b/src/output/output_tsrist.cpp +index c4e03f80..10b29673 100644 +--- a/src/output/output_tsrist.cpp ++++ b/src/output/output_tsrist.cpp +@@ -178,6 +178,20 @@ namespace Mist{ + } + + OutTSRIST::~OutTSRIST(){} ++ ++ std::string OutTSRIST::getConnectedHost(){ ++ if (!pushOut) { return Output::getConnectedHost(); } ++ return target.host; ++ } ++ std::string OutTSRIST::getConnectedBinHost(){ ++ if (!pushOut) { return Output::getConnectedBinHost(); } ++ std::string binHost = Socket::getBinForms(target.host); ++ if (binHost.size() > 16){ binHost = binHost.substr(0, 16); } ++ if (binHost.size() < 16){ ++ binHost = std::string("\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\001", 16); ++ } ++ return binHost; ++ } + + void OutTSRIST::init(Util::Config *cfg){ + Output::init(cfg); +@@ -319,7 +333,7 @@ namespace Mist{ + } + } + +- void OutTSRIST::connStats(uint64_t now, Comms::Statistics &statComm){ ++ void OutTSRIST::connStats(uint64_t now, Comms::Connections &statComm){ + if (!myConn){return;} + statComm.setUp(upBytes); + statComm.setDown(0); +diff --git a/src/output/output_tsrist.h b/src/output/output_tsrist.h +index c9a5db39..80ee2376 100644 +--- a/src/output/output_tsrist.h ++++ b/src/output/output_tsrist.h +@@ -16,9 +16,11 @@ namespace Mist{ + bool isReadyForPlay(){return true;} + virtual void requestHandler(); + static void listener(Util::Config &conf, int (*callback)(Socket::Connection &S)); ++ std::string getConnectedHost(); ++ std::string getConnectedBinHost(); + + protected: +- virtual void connStats(uint64_t now, Comms::Statistics &statComm); ++ virtual void connStats(uint64_t now, Comms::Connections &statComm); + //virtual std::string getConnectedHost(){ + // return srtConn.remotehost; + //} +diff --git a/src/session.cpp b/src/session.cpp +index 3865e0ec..d2331c53 100644 +--- a/src/session.cpp ++++ b/src/session.cpp +@@ -7,17 +7,31 @@ + #include + #include + #include +-// Stats of connections which have closed are added to these global counters +-uint64_t globalNow = 0; ++ ++// Global counters ++uint64_t now = Util::bootSecs(); ++uint64_t currentConnections = 0; ++uint64_t lastSecond = 0; + uint64_t globalTime = 0; + uint64_t globalDown = 0; + uint64_t globalUp = 0; + uint64_t globalPktcount = 0; + uint64_t globalPktloss = 0; + uint64_t globalPktretrans = 0; ++// Stores last values of each connection ++std::map connTime; ++std::map connDown; ++std::map connUp; ++std::map connPktcount; ++std::map connPktloss; ++std::map connPktretrans; + // Counts the duration a connector has been active + std::map connectorCount; + std::map connectorLastActive; ++std::map hostCount; ++std::map hostLastActive; ++std::map streamCount; ++std::map streamLastActive; + // Set to True when a session gets invalidated, so that we know to run a new USER_NEW trigger + bool forceTrigger = false; + void handleSignal(int signum){ +@@ -26,96 +40,141 @@ void handleSignal(int signum){ + } + } + +-void userOnActive(uint64_t &connections){ +- ++connections; +-} ++const char nullAddress[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + +-std::string getEnvWithDefault(const std::string variableName, const std::string defaultValue){ +- const char* value = getenv(variableName.c_str()); +- if (value){ +- unsetenv(variableName.c_str()); +- return value; +- }else{ +- return defaultValue; +- } ++void userOnActive(Comms::Connections &connections, size_t idx){ ++ uint64_t lastUpdate = connections.getNow(idx); ++ if (lastUpdate < now - 10){return;} ++ ++currentConnections; ++ std::string thisConnector = connections.getConnector(idx); ++ std::string thisStreamName = connections.getStream(idx); ++ const std::string& thisHost = connections.getHost(idx); ++ ++ if (connections.getLastSecond(idx) > lastSecond){lastSecond = connections.getLastSecond(idx);} ++ // Save info on the latest active stream, protocol and host separately ++ if (thisConnector.size() && thisConnector != "HTTP"){ ++ connectorCount[thisConnector]++; ++ if (connectorLastActive[thisConnector] < lastUpdate){connectorLastActive[thisConnector] = lastUpdate;} ++ } ++ if (thisStreamName.size()){ ++ streamCount[thisStreamName]++; ++ if (streamLastActive[thisStreamName] < lastUpdate){streamLastActive[thisStreamName] = lastUpdate;} ++ } ++ if (memcmp(thisHost.data(), nullAddress, 16)){ ++ hostCount[thisHost]++; ++ if (!hostLastActive.count(thisHost) || hostLastActive[thisHost] < lastUpdate){hostLastActive[thisHost] = lastUpdate;} ++ } ++ // Sanity checks ++ if (connections.getDown(idx) < connDown[idx]){ ++ WARN_MSG("Connection downloaded bytes should be a counter, but has decreased in value"); ++ connDown[idx] = connections.getDown(idx); ++ } ++ if (connections.getUp(idx) < connUp[idx]){ ++ WARN_MSG("Connection uploaded bytes should be a counter, but has decreased in value"); ++ connUp[idx] = connections.getUp(idx); ++ } ++ if (connections.getPacketCount(idx) < connPktcount[idx]){ ++ WARN_MSG("Connection packet count should be a counter, but has decreased in value"); ++ connPktcount[idx] = connections.getPacketCount(idx); ++ } ++ if (connections.getPacketLostCount(idx) < connPktloss[idx]){ ++ WARN_MSG("Connection packet loss count should be a counter, but has decreased in value"); ++ connPktloss[idx] = connections.getPacketLostCount(idx); ++ } ++ if (connections.getPacketRetransmitCount(idx) < connPktretrans[idx]){ ++ WARN_MSG("Connection packets retransmitted should be a counter, but has decreased in value"); ++ connPktretrans[idx] = connections.getPacketRetransmitCount(idx); ++ } ++ // Add increase in stats to global stats ++ globalDown += connections.getDown(idx) - connDown[idx]; ++ globalUp += connections.getUp(idx) - connUp[idx]; ++ globalPktcount += connections.getPacketCount(idx) - connPktcount[idx]; ++ globalPktloss += connections.getPacketLostCount(idx) - connPktloss[idx]; ++ globalPktretrans += connections.getPacketRetransmitCount(idx) - connPktretrans[idx]; ++ // Set last values of this connection ++ connTime[idx]++; ++ connDown[idx] = connections.getDown(idx); ++ connUp[idx] = connections.getUp(idx); ++ connPktcount[idx] = connections.getPacketCount(idx); ++ connPktloss[idx] = connections.getPacketLostCount(idx); ++ connPktretrans[idx] = connections.getPacketRetransmitCount(idx); + } + +-/// \brief Adds stats of closed connections to global counters ++/// \brief Remove mappings of inactive connections + void userOnDisconnect(Comms::Connections & connections, size_t idx){ +- std::string thisConnector = connections.getConnector(idx); +- if (thisConnector != ""){ +- connectorCount[thisConnector] += connections.getTime(idx); +- } +- globalTime += connections.getTime(idx); +- globalDown += connections.getDown(idx); +- globalUp += connections.getUp(idx); +- globalPktcount += connections.getPacketCount(idx); +- globalPktloss += connections.getPacketLostCount(idx); +- globalPktretrans += connections.getPacketRetransmitCount(idx); ++ connTime.erase(idx); ++ connDown.erase(idx); ++ connUp.erase(idx); ++ connPktcount.erase(idx); ++ connPktloss.erase(idx); ++ connPktretrans.erase(idx); + } + + int main(int argc, char **argv){ + Comms::Connections connections; + Comms::Sessions sessions; + uint64_t lastSeen = Util::bootSecs(); +- uint64_t currentConnections = 0; + Util::redirectLogsIfNeeded(); + signal(SIGUSR1, handleSignal); + // Init config and parse arguments + Util::Config config = Util::Config("MistSession"); + JSON::Value option; ++ char * tmpStr = 0; + + option.null(); + option["arg_num"] = 1; + option["arg"] = "string"; + option["help"] = "Session identifier of the entire session"; +- option["default"] = ""; + config.addOption("sessionid", option); + +- option.null(); +- option["long"] = "sessionmode"; +- option["short"] = "m"; +- option["arg"] = "integer"; +- option["default"] = 0; +- config.addOption("sessionmode", option); +- + option.null(); + option["long"] = "streamname"; +- option["short"] = "n"; ++ option["short"] = "s"; + option["arg"] = "string"; +- option["default"] = ""; ++ option["help"] = "Stream name initial value. May also be passed as SESSION_STREAM"; ++ tmpStr = getenv("SESSION_STREAM"); ++ option["default"] = tmpStr?tmpStr:""; + config.addOption("streamname", option); + + option.null(); + option["long"] = "ip"; + option["short"] = "i"; + option["arg"] = "string"; +- option["default"] = ""; ++ option["help"] = "IP address initial value. May also be passed as SESSION_IP"; ++ tmpStr = getenv("SESSION_IP"); ++ option["default"] = tmpStr?tmpStr:""; + config.addOption("ip", option); + + option.null(); +- option["long"] = "sid"; +- option["short"] = "s"; ++ option["long"] = "tkn"; ++ option["short"] = "t"; + option["arg"] = "string"; +- option["default"] = ""; +- config.addOption("sid", option); ++ option["help"] = "Client-side session ID initial value. May also be passed as SESSION_TKN"; ++ tmpStr = getenv("SESSION_TKN"); ++ option["default"] = tmpStr?tmpStr:""; ++ config.addOption("tkn", option); + + option.null(); + option["long"] = "protocol"; + option["short"] = "p"; + option["arg"] = "string"; +- option["default"] = ""; ++ option["help"] = "Protocol initial value. May also be passed as SESSION_PROTOCOL"; ++ tmpStr = getenv("SESSION_PROTOCOL"); ++ option["default"] = tmpStr?tmpStr:""; + config.addOption("protocol", option); + + option.null(); + option["long"] = "requrl"; + option["short"] = "r"; + option["arg"] = "string"; +- option["default"] = ""; ++ option["help"] = "Request URL initial value. May also be passed as SESSION_REQURL"; ++ tmpStr = getenv("SESSION_REQURL"); ++ option["default"] = tmpStr?tmpStr:""; + config.addOption("requrl", option); + + config.activate(); + if (!(config.parseArgs(argc, argv))){ ++ config.printHelp(std::cout); + FAIL_MSG("Cannot start a new session due to invalid arguments"); + return 1; + } +@@ -123,24 +182,17 @@ int main(int argc, char **argv){ + const uint64_t bootTime = Util::getMicros(); + // Get session ID, session mode and other variables used as payload for the USER_NEW and USER_END triggers + const std::string thisStreamName = config.getString("streamname"); +- const std::string thisHost = config.getString("ip"); +- const std::string thisSid = config.getString("sid"); ++ const std::string thisToken = config.getString("tkn"); + const std::string thisProtocol = config.getString("protocol"); + const std::string thisReqUrl = config.getString("requrl"); + const std::string thisSessionId = config.getString("sessionid"); +- const uint64_t sessionMode = config.getInteger("sessionmode"); +- +- if (thisSessionId == "" || thisProtocol == "" || thisStreamName == ""){ +- FAIL_MSG("Given the following incomplete arguments: SessionId: '%s', protocol: '%s', stream name: '%s'. Aborting opening a new session", +- thisSessionId.c_str(), thisProtocol.c_str(), thisStreamName.c_str()); +- return 1; +- } ++ std::string thisHost = Socket::getBinForms(config.getString("ip")); ++ if (thisHost.size() > 16){thisHost = thisHost.substr(0, 16);} + +- MEDIUM_MSG("Starting a new session for sessionId '%s'", thisSessionId.c_str()); +- if (sessionMode < 1 || sessionMode > 15) { +- FAIL_MSG("Invalid session mode of value %lu. Should be larger than 0 and smaller than 16", sessionMode); +- return 1; +- } ++ std::string ipHex; ++ Socket::hostBytesToStr(thisHost.c_str(), thisHost.size(), ipHex); ++ VERYHIGH_MSG("Starting a new session. Passed variables are stream name '%s', session token '%s', protocol '%s', requested URL '%s', IP '%s' and session id '%s'", ++ thisStreamName.c_str(), thisToken.c_str(), thisProtocol.c_str(), thisReqUrl.c_str(), ipHex.c_str(), thisSessionId.c_str()); + + // Try to lock to ensure we are the only process initialising this session + IPC::semaphore sessionLock; +@@ -174,194 +226,200 @@ int main(int argc, char **argv){ + sessionLock.post(); + return 1; + } +- // Open the shared memory page containing statistics for each individual connection in this session +- connections.reload(thisStreamName, thisHost, thisSid, thisProtocol, thisReqUrl, sessionMode, true, false); ++ + // Initialise global session data + sessions.setHost(thisHost); + sessions.setSessId(thisSessionId); + sessions.setStream(thisStreamName); +- sessionLock.post(); ++ if (thisProtocol.size() && thisProtocol != "HTTP"){connectorLastActive[thisProtocol] = now;} ++ if (thisStreamName.size()){streamLastActive[thisStreamName] = now;} ++ if (memcmp(thisHost.data(), nullAddress, 16)){hostLastActive[thisHost] = now;} ++ ++ // Open the shared memory page containing statistics for each individual connection in this session ++ connections.reload(thisSessionId, true); + + // Determine session type, since triggers only get run for viewer type sessions + uint64_t thisType = 0; + if (thisSessionId[0] == 'I'){ +- INFO_MSG("Started new input session %s in %lu microseconds", thisSessionId.c_str(), Util::getMicros(bootTime)); + thisType = 1; +- } +- else if (thisSessionId[0] == 'O'){ +- INFO_MSG("Started new output session %s in %lu microseconds", thisSessionId.c_str(), Util::getMicros(bootTime)); ++ } else if (thisSessionId[0] == 'O'){ + thisType = 2; +- } +- else{ +- INFO_MSG("Started new viewer session %s in %lu microseconds", thisSessionId.c_str(), Util::getMicros(bootTime)); ++ } else if (thisSessionId[0] == 'U'){ ++ thisType = 3; + } + + // Do a USER_NEW trigger if it is defined for this stream + if (!thisType && Triggers::shouldTrigger("USER_NEW", thisStreamName)){ +- std::string payload = thisStreamName + "\n" + thisHost + "\n" + +- thisSid + "\n" + thisProtocol + ++ std::string payload = thisStreamName + "\n" + config.getString("ip") + "\n" + ++ thisToken + "\n" + thisProtocol + + "\n" + thisReqUrl + "\n" + thisSessionId; + if (!Triggers::doTrigger("USER_NEW", payload, thisStreamName)){ + // Mark all connections of this session as finished, since this viewer is not allowed to view this stream ++ Util::logExitReason("Session rejected by USER_NEW"); + connections.setExit(); + connections.finishAll(); + } + } + +- uint64_t lastSecond = 0; +- uint64_t now = 0; +- uint64_t time = 0; +- uint64_t down = 0; +- uint64_t up = 0; +- uint64_t pktcount = 0; +- uint64_t pktloss = 0; +- uint64_t pktretrans = 0; +- std::string connector = ""; ++ //start allowing viewers ++ sessionLock.post(); ++ ++ INFO_MSG("Started new session %s in %.3f ms", thisSessionId.c_str(), (double)Util::getMicros(bootTime)/1000.0); ++ + // Stay active until Mist exits or we no longer have an active connection +- while (config.is_active && (currentConnections || Util::bootSecs() - lastSeen <= 10)){ +- time = 0; +- connector = ""; +- down = 0; +- up = 0; +- pktcount = 0; +- pktloss = 0; +- pktretrans = 0; ++ while (config.is_active && (currentConnections || now - lastSeen <= STATS_DELAY) && !connections.getExit()){ + currentConnections = 0; ++ lastSecond = 0; ++ now = Util::bootSecs(); + +- // Count active connections +- COMM_LOOP(connections, userOnActive(currentConnections), userOnDisconnect(connections, id)); + // Loop through all connection entries to get a summary of statistics +- for (uint64_t idx = 0; idx < connections.recordCount(); idx++){ +- if (connections.getStatus(idx) == COMM_STATUS_INVALID || connections.getStatus(idx) & COMM_STATUS_DISCONNECT){continue;} +- uint64_t thisLastSecond = connections.getLastSecond(idx); +- std::string thisConnector = connections.getConnector(idx); +- // Save info on the latest active connection separately +- if (thisLastSecond > lastSecond){ +- lastSecond = thisLastSecond; +- now = connections.getNow(idx); +- } +- connectorLastActive[thisConnector] = thisLastSecond; +- // Sum all other variables +- time += connections.getTime(idx); +- down += connections.getDown(idx); +- up += connections.getUp(idx); +- pktcount += connections.getPacketCount(idx); +- pktloss += connections.getPacketLostCount(idx); +- pktretrans += connections.getPacketRetransmitCount(idx); ++ COMM_LOOP(connections, userOnActive(connections, id), userOnDisconnect(connections, id)); ++ if (currentConnections){ ++ globalTime++; ++ lastSeen = now; + } + +- // Convert connector duration to string +- std::stringstream connectorSummary; +- bool addDelimiter = false; +- connectorSummary << "{"; +- for (std::map::iterator it = connectorLastActive.begin(); +- it != connectorLastActive.end(); ++it){ +- if (lastSecond - it->second < 10000){ +- connectorSummary << (addDelimiter ? "," : "") << it->first; +- addDelimiter = true; +- } +- } +- connectorSummary << "}"; +- +- // Write summary to global statistics +- sessions.setTime(time + globalTime); +- sessions.setDown(down + globalDown); +- sessions.setUp(up + globalUp); +- sessions.setPacketCount(pktcount + globalPktcount); +- sessions.setPacketLostCount(pktloss + globalPktloss); +- sessions.setPacketRetransmitCount(pktretrans + globalPktretrans); ++ ++ sessions.setTime(globalTime); ++ sessions.setDown(globalDown); ++ sessions.setUp(globalUp); ++ sessions.setPacketCount(globalPktcount); ++ sessions.setPacketLostCount(globalPktloss); ++ sessions.setPacketRetransmitCount(globalPktretrans); + sessions.setLastSecond(lastSecond); +- sessions.setConnector(connectorSummary.str()); + sessions.setNow(now); + ++ if (currentConnections){ ++ { ++ // Convert active protocols to string ++ std::stringstream connectorSummary; ++ for (std::map::iterator it = connectorLastActive.begin(); ++ it != connectorLastActive.end(); ++it){ ++ if (now - it->second < STATS_DELAY){ ++ connectorSummary << (connectorSummary.str().size() ? "," : "") << it->first; ++ } ++ } ++ sessions.setConnector(connectorSummary.str()); ++ } ++ ++ { ++ // Set active host to last active or 0 if there were various hosts active recently ++ std::string thisHost; ++ for (std::map::iterator it = hostLastActive.begin(); ++ it != hostLastActive.end(); ++it){ ++ if (now - it->second < STATS_DELAY){ ++ if (!thisHost.size()){ ++ thisHost = it->first; ++ }else if (thisHost != it->first){ ++ thisHost = nullAddress; ++ break; ++ } ++ } ++ } ++ if (!thisHost.size()){ ++ thisHost = nullAddress; ++ } ++ sessions.setHost(thisHost); ++ } ++ ++ { ++ // Set active stream name to last active or "" if there were multiple streams active recently ++ std::string thisStream = ""; ++ for (std::map::iterator it = streamLastActive.begin(); ++ it != streamLastActive.end(); ++it){ ++ if (now - it->second < STATS_DELAY){ ++ if (!thisStream.size()){ ++ thisStream = it->first; ++ }else if (thisStream != it->first){ ++ thisStream = ""; ++ break; ++ } ++ } ++ } ++ sessions.setStream(thisStream); ++ } ++ } ++ + // Retrigger USER_NEW if a re-sync was requested + if (!thisType && forceTrigger){ + forceTrigger = false; ++ std::string host; ++ Socket::hostBytesToStr(thisHost.data(), 16, host); + if (Triggers::shouldTrigger("USER_NEW", thisStreamName)){ + INFO_MSG("Triggering USER_NEW for stream %s", thisStreamName.c_str()); +- std::string payload = thisStreamName + "\n" + thisHost + "\n" + +- thisSid + "\n" + thisProtocol + ++ std::string payload = thisStreamName + "\n" + host + "\n" + ++ thisToken + "\n" + thisProtocol + + "\n" + thisReqUrl + "\n" + thisSessionId; + if (!Triggers::doTrigger("USER_NEW", payload, thisStreamName)){ + INFO_MSG("USER_NEW rejected stream %s", thisStreamName.c_str()); ++ Util::logExitReason("Session rejected by USER_NEW"); + connections.setExit(); + connections.finishAll(); ++ break; + }else{ + INFO_MSG("USER_NEW accepted stream %s", thisStreamName.c_str()); + } + } + } + +- // Invalidate connections if the session is marked as invalid +- if(connections.getExit()){ +- connections.finishAll(); +- break; +- } + // Remember latest activity so we know when this session ends + if (currentConnections){ +- lastSeen = Util::bootSecs(); + } +- Util::sleep(1000); ++ Util::wait(1000); ++ } ++ if (Util::bootSecs() - lastSeen > STATS_DELAY){ ++ Util::logExitReason("Session inactive for %d seconds", STATS_DELAY); + } + + // Trigger USER_END + if (!thisType && Triggers::shouldTrigger("USER_END", thisStreamName)){ +- lastSecond = 0; +- time = 0; +- down = 0; +- up = 0; +- +- // Get a final summary of this session +- for (uint64_t idx = 0; idx < connections.recordCount(); idx++){ +- if (connections.getStatus(idx) == COMM_STATUS_INVALID || connections.getStatus(idx) & COMM_STATUS_DISCONNECT){continue;} +- uint64_t thisLastSecond = connections.getLastSecond(idx); +- // Set last second to the latest entry +- if (thisLastSecond > lastSecond){ +- lastSecond = thisLastSecond; +- } +- // Count protocol durations across the entire session +- std::string thisConnector = connections.getConnector(idx); +- if (thisConnector != ""){ +- connectorCount[thisConnector] += connections.getTime(idx); +- } +- // Sum all other variables +- time += connections.getTime(idx); +- down += connections.getDown(idx); +- up += connections.getUp(idx); +- } + +- // Convert connector duration to string ++ // Convert connector, host and stream into lists and counts + std::stringstream connectorSummary; +- bool addDelimiter = false; +- connectorSummary << "{"; +- for (std::map::iterator it = connectorCount.begin(); +- it != connectorCount.end(); ++it){ +- connectorSummary << (addDelimiter ? "," : "") << it->first << ":" << it->second; +- addDelimiter = true; ++ std::stringstream connectorTimes; ++ for (std::map::iterator it = connectorCount.begin(); it != connectorCount.end(); ++it){ ++ connectorSummary << (connectorSummary.str().size() ? "," : "") << it->first; ++ connectorTimes << (connectorTimes.str().size() ? "," : "") << it->second; ++ } ++ std::stringstream hostSummary; ++ std::stringstream hostTimes; ++ for (std::map::iterator it = hostCount.begin(); it != hostCount.end(); ++it){ ++ std::string host; ++ Socket::hostBytesToStr(it->first.data(), 16, host); ++ hostSummary << (hostSummary.str().size() ? "," : "") << host; ++ hostTimes << (hostTimes.str().size() ? "," : "") << it->second; ++ } ++ std::stringstream streamSummary; ++ std::stringstream streamTimes; ++ for (std::map::iterator it = streamCount.begin(); it != streamCount.end(); ++it){ ++ streamSummary << (streamSummary.str().size() ? "," : "") << it->first; ++ streamTimes << (streamTimes.str().size() ? "," : "") << it->second; + } +- connectorSummary << "}"; + +- const uint64_t duration = lastSecond - (bootTime / 1000); + std::stringstream summary; +- summary << thisSessionId << "\n" +- << thisStreamName << "\n" ++ summary << thisToken << "\n" ++ << streamSummary.str() << "\n" + << connectorSummary.str() << "\n" +- << thisHost << "\n" +- << duration << "\n" +- << up << "\n" +- << down << "\n" +- << sessions.getTags(); ++ << hostSummary.str() << "\n" ++ << globalTime << "\n" ++ << globalUp << "\n" ++ << globalDown << "\n" ++ << sessions.getTags() << "\n" ++ << hostTimes.str() << "\n" ++ << connectorTimes.str() << "\n" ++ << streamTimes.str() << "\n" ++ << thisSessionId; + Triggers::doTrigger("USER_END", summary.str(), thisStreamName); + } + + if (!thisType && connections.getExit()){ +- WARN_MSG("Session %s has been invalidated since it is not allowed to view stream %s", thisSessionId.c_str(), thisStreamName.c_str()); + uint64_t sleepStart = Util::bootSecs(); + // Keep session invalidated for 10 minutes, or until the session stops +- while (config.is_active && sleepStart - Util::bootSecs() < 600){ ++ while (config.is_active && Util::bootSecs() - sleepStart < SESS_TIMEOUT){ + Util::sleep(1000); ++ if (forceTrigger){break;} + } + } +- INFO_MSG("Shutting down session %s", thisSessionId.c_str()); ++ INFO_MSG("Shutting down session %s: %s", thisSessionId.c_str(), Util::exitReason); + return 0; + } +-- +2.25.1 + + +From 1a4a526a112d0bdeaa0f269a133fe55c39a0d129 Mon Sep 17 00:00:00 2001 +From: Cat +Date: Wed, 5 Oct 2022 03:10:04 +0200 +Subject: [PATCH 28/38] LSP: Moved some settings to new "General" tab, added + bitmask inputtype, removed LTSonly code + +--- + lsp/main.css | 30 +++ + lsp/minified.js | 396 ++++++++++++++++---------------- + lsp/mist.js | 582 +++++++++++++++++++++++++++++------------------- + 3 files changed, 590 insertions(+), 418 deletions(-) + +diff --git a/lsp/main.css b/lsp/main.css +index 6c72e6a7..9289d211 100644 +--- a/lsp/main.css ++++ b/lsp/main.css +@@ -279,6 +279,21 @@ main > button { + width: 1.5em; + flex-grow: 0; + } ++.input_container .field_container .field.bitmask { ++ display: flex; ++ flex-flow: row wrap; ++ justify-content: space-between; ++ padding: 0; ++ margin-bottom: 0.5em; ++} ++.input_container .field_container .field.bitmask > label { ++ display: flex; ++ flex-flow: row nowrap; ++ align-items: center; ++ margin-right: -0.25em; ++} ++.input_container .field_container .field.bitmask > label > input { margin: 0; } ++.input_container .field_container .field.bitmask > label > span { margin: 0 0.25em; } + textarea { + font-size: 0.8em; + } +@@ -349,6 +364,20 @@ select option[value=""] { + display: block; + z-index: 0; + } ++.input_container label.active { position: relative; } ++.input_container label.active:after { ++ content: ""; ++ display: block; ++ position: absolute; ++ z-index: -1; ++ top: -0.25em; ++ left: -0.25em; ++ width: 55.5em; ++ bottom: -0.25em; ++ background: linear-gradient(to right,#9ec2da,transparent 55em); ++ opacity: 0.3; ++ filter: blur(0.1em); ++} + .input_container label.active .ih_balloon, + .input_container > .help_container .ih_balloon { + display: block; +@@ -577,6 +606,7 @@ table.valigntop th { + } + .input_container .field_container .field.inputlist > * { + width: 100%; ++ box-sizing: border-box; + } + .input_container .field_container .field.checkcontainer { + font-size: 0.8em; +diff --git a/lsp/minified.js b/lsp/minified.js +index 52a6f6bc..9db52d63 100644 +--- a/lsp/minified.js ++++ b/lsp/minified.js +@@ -1,11 +1,11 @@ +-var MD5=function(a){function b(a,b){var c,d,g,i,e;g=a&2147483648;i=b&2147483648;c=a&1073741824;d=b&1073741824;e=(a&1073741823)+(b&1073741823);return c&d?e^2147483648^g^i:c|d?e&1073741824?e^3221225472^g^i:e^1073741824^g^i:e^g^i}function c(a,c,d,g,i,e,f){a=b(a,b(b(c&d|~c&g,i),f));return b(a<>>32-e,c)}function d(a,c,d,g,i,e,f){a=b(a,b(b(c&g|d&~g,i),f));return b(a<>>32-e,c)}function e(a,c,d,g,e,i,f){a=b(a,b(b(c^d^g,e),f));return b(a<>>32-i,c)}function l(a,c,d,g,i,e,f){a=b(a,b(b(d^(c|~g), +-i),f));return b(a<>>32-e,c)}function m(a){var b="",c="",d;for(d=0;3>=d;d++)c=a>>>8*d&255,c="0"+c.toString(16),b+=c.substr(c.length-2,2);return b}var f=[],t,o,k,w,h,g,i,j,f=a.replace(/\r\n/g,"\n"),a="";for(t=0;to?a+=String.fromCharCode(o):(127o?a+=String.fromCharCode(o>>6|192):(a+=String.fromCharCode(o>>12|224),a+=String.fromCharCode(o>>6&63|128)),a+=String.fromCharCode(o&63|128));f=a;a=f.length;t=a+8;o=16*((t-t%64)/64+1);k=Array(o-1);for(h=w=0;h>>29;f=k;h=1732584193;g=4023233417;i=2562383102;j=271733878;for(a=0;ac?1*d:a .menu"),main:$("main"),header:$("header"),connection:{status:$("#connection"),user_and_host:$("#user_and_host"),msg:$("#message")}};UI.buildMenu();UI.stored.getOpts();try{if("mistLogin"in sessionStorage){var a=JSON.parse(sessionStorage.mistLogin);mist.user.name=a.name;mist.user.password=a.password;mist.user.host=a.host}}catch(b){}location.hash&&(a=decodeURIComponent(location.hash).substring(1).split("@")[0].split("&"),mist.user.name=a[0],a[1]&&(mist.user.host= ++var MD5=function(a){function b(a,b){var c,d,g,f,e;g=a&2147483648;f=b&2147483648;c=a&1073741824;d=b&1073741824;e=(a&1073741823)+(b&1073741823);return c&d?e^2147483648^g^f:c|d?e&1073741824?e^3221225472^g^f:e^1073741824^g^f:e^g^f}function c(a,c,d,g,f,e,h){a=b(a,b(b(c&d|~c&g,f),h));return b(a<>>32-e,c)}function d(a,c,d,g,e,f,h){a=b(a,b(b(c&g|d&~g,e),h));return b(a<>>32-f,c)}function e(a,c,d,g,f,e,h){a=b(a,b(b(c^d^g,f),h));return b(a<>>32-e,c)}function l(a,c,d,g,e,f,h){a=b(a,b(b(d^(c|~g), ++e),h));return b(a<>>32-f,c)}function n(a){var b="",c="",d;for(d=0;3>=d;d++)c=a>>>8*d&255,c="0"+c.toString(16),b+=c.substr(c.length-2,2);return b}var h=[],t,m,k,w,i,g,f,j,h=a.replace(/\r\n/g,"\n"),a="";for(t=0;tm?a+=String.fromCharCode(m):(127m?a+=String.fromCharCode(m>>6|192):(a+=String.fromCharCode(m>>12|224),a+=String.fromCharCode(m>>6&63|128)),a+=String.fromCharCode(m&63|128));h=a;a=h.length;t=a+8;m=16*((t-t%64)/64+1);k=Array(m-1);for(i=w=0;i>>29;h=k;i=1732584193;g=4023233417;f=2562383102;j=271733878;for(a=0;ac?1*d:a .menu"),main:$("main"),header:$("header"),connection:{status:$("#connection"),user_and_host:$("#user_and_host"),msg:$("#message")}};UI.buildMenu();UI.stored.getOpts();try{if("mistLogin"in sessionStorage){var a=JSON.parse(sessionStorage.mistLogin);mist.user.name=a.name;mist.user.password=a.password;mist.user.host=a.host}}catch(b){}location.hash&&(a=decodeURIComponent(location.hash).substring(1).split("@")[0].split("&"),mist.user.name=a[0],a[1]&&(mist.user.host= + a[1]));mist.send(function(){$(window).trigger("hashchange")},{},{timeout:5,hide:!0});var c=0;$("body > div.filler").on("scroll",function(){var a=$(this).scrollLeft();a!=c&&UI.elements.header.css("margin-right",-1*a+"px");c=a})});$(window).on("hashchange",function(){var a=decodeURIComponent(location.hash).substring(1).split("@");a[1]||(a[1]="");a=a[1].split("&");""==a[0]&&(a[0]="Overview");UI.showTab(a[0],a[1])}); + var MistVideoObject={},otherhost={host:!1,https:!1},UI={debug:!1,elements:{},stored:{getOpts:function(){var a=localStorage.stored;a&&(a=JSON.parse(a));$.extend(!0,this.vars,a);return this.vars},saveOpt:function(a,b){this.vars[a]=b;localStorage.stored=JSON.stringify(this.vars);return this.vars},vars:{helpme:!0}},interval:{clear:function(){"undefined"!=typeof this.opts&&(clearInterval(this.opts.id),delete this.opts)},set:function(a,b){this.opts&&log("[interval]","Set called on interval, but an interval is already active."); + this.opts={delay:b,callback:a};this.opts.id=setInterval(a,b)}},returnTab:["Overview"],countrylist:{AF:"Afghanistan",AX:"Åland Islands",AL:"Albania",DZ:"Algeria",AS:"American Samoa",AD:"Andorra",AO:"Angola",AI:"Anguilla",AQ:"Antarctica",AG:"Antigua and Barbuda",AR:"Argentina",AM:"Armenia",AW:"Aruba",AU:"Australia",AT:"Austria",AZ:"Azerbaijan",BS:"Bahamas",BH:"Bahrain",BD:"Bangladesh",BB:"Barbados",BY:"Belarus",BE:"Belgium",BZ:"Belize",BJ:"Benin",BM:"Bermuda",BT:"Bhutan",BO:"Bolivia, Plurinational State of", +@@ -20,61 +20,61 @@ TV:"Tuvalu",UG:"Uganda",UA:"Ukraine",AE:"United Arab Emirates",GB:"United Kingdo + clearTimeout(this.hiding);delete this.hiding;var c=$(document).height()-$tooltip.outerHeight(),d=$(document).width()-$tooltip.outerWidth();$tooltip.css("left",Math.min(a.pageX+10,d-10));$tooltip.css("top",Math.min(a.pageY+25,c-10));$tooltip.show().addClass("show")},hide:function(){$tooltip=this.element;$tooltip.removeClass("show");this.hiding=setTimeout(function(){$tooltip.hide()},500)},element:$("
").attr("id","tooltip")},humanMime:function(a){var b=!1;switch(a){case "html5/application/vnd.apple.mpegurl":b= + "HLS (TS)";break;case "html5/application/vnd.apple.mpegurl;version=7":b="HLS (CMAF)";break;case "html5/video/webm":b="WebM";break;case "html5/video/mp4":b="MP4";break;case "dash/video/mp4":b="DASH";break;case "flash/11":b="HDS";break;case "flash/10":b="RTMP";break;case "flash/7":b="Progressive";break;case "html5/audio/mp3":b="MP3";break;case "html5/audio/wav":b="WAV";break;case "html5/video/mp2t":case "html5/video/mpeg":b="TS";break;case "html5/application/vnd.ms-sstr+xml":case "html5/application/vnd.ms-ss":b= + "Smooth Streaming";break;case "html5/text/vtt":b="VTT Subtitles";break;case "html5/text/plain":b="SRT Subtitles";break;case "html5/text/javascript":b="JSON Subtitles";break;case "rtsp":b="RTSP";break;case "webrtc":b="WebRTC"}return b},popup:{element:null,show:function(a){this.element=$("
").attr("id","popup").append($("