From 0430a644dfcd0fcc0ea6494bdb8d431073b8361d Mon Sep 17 00:00:00 2001 From: Matthew James Date: Fri, 22 Jul 2022 00:44:28 +0800 Subject: [PATCH 01/38] Enable Parameters on TSSRT listener --- lib/socket_srt.cpp | 7 +++---- lib/socket_srt.h | 2 +- src/input/input_tssrt.cpp | 4 +++- src/output/output_tssrt.cpp | 3 ++- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/socket_srt.cpp b/lib/socket_srt.cpp index a01117db..3e03bce4 100644 --- a/lib/socket_srt.cpp +++ b/lib/socket_srt.cpp @@ -462,12 +462,11 @@ namespace Socket{ SRTServer::SRTServer(int fromSock){conn = SRTConnection(fromSock);} - SRTServer::SRTServer(int port, std::string hostname, bool nonblock, const std::string &_direction){ + SRTServer::SRTServer(int port, std::string hostname, std::map _params, bool nonblock, const std::string &_direction){ // We always create a server as listening - std::map listenMode; - listenMode["mode"] = "listener"; + _params["mode"] = "listener"; if (hostname == ""){hostname = "0.0.0.0";} - conn.connect(hostname, port, _direction, listenMode); + conn.connect(hostname, port, _direction, _params); conn.setBlocking(true); if (!conn){ ERROR_MSG("Unable to create socket"); diff --git a/lib/socket_srt.h b/lib/socket_srt.h index 5101d38d..07e947f8 100644 --- a/lib/socket_srt.h +++ b/lib/socket_srt.h @@ -97,7 +97,7 @@ namespace Socket{ public: SRTServer(); SRTServer(int existingSock); - SRTServer(int port, std::string hostname, bool nonblock = false, const std::string &_direction = "input"); + SRTServer(int port, std::string hostname, std::map params, bool nonblock = false, const std::string &_direction = "input"); SRTConnection accept(bool nonblock = false, const std::string &direction = "input"); void setBlocking(bool blocking); bool connected() const; diff --git a/src/input/input_tssrt.cpp b/src/input/input_tssrt.cpp index 4a7adcef..d12f9cba 100644 --- a/src/input/input_tssrt.cpp +++ b/src/input/input_tssrt.cpp @@ -137,7 +137,9 @@ namespace Mist{ HTTP::URL u(source); INFO_MSG("Parsed url: %s", u.getUrl().c_str()); if (Socket::interpretSRTMode(u) == "listener"){ - sSock = Socket::SRTServer(u.getPort(), u.host, false); + std::map arguments; + HTTP::parseVars(u.args, arguments); + sSock = Socket::SRTServer(u.getPort(), u.host, arguments, false); struct sigaction new_action; struct sigaction cur_action; new_action.sa_sigaction = signal_handler; diff --git a/src/output/output_tssrt.cpp b/src/output/output_tssrt.cpp index aa06189e..fb07b11e 100644 --- a/src/output/output_tssrt.cpp +++ b/src/output/output_tssrt.cpp @@ -434,7 +434,8 @@ int main(int argc, char *argv[]){ sigaction(SIGUSR1, &new_action, NULL); } if (conf.getInteger("port") && conf.getString("interface").size()){ - server_socket = Socket::SRTServer(conf.getInteger("port"), conf.getString("interface"), false, "output"); + std::map arguments; + server_socket = Socket::SRTServer(conf.getInteger("port"), conf.getString("interface"), arguments, false, "output"); } if (!server_socket.connected()){ DEVEL_MSG("Failure to open socket"); -- 2.25.1 From 508506c241b2980d1aed607cd269ea60e5265e41 Mon Sep 17 00:00:00 2001 From: Thulinma Date: Tue, 5 Jan 2021 15:14:04 +0100 Subject: [PATCH 02/38] Fixes to UDP socket behaviour --- lib/socket.cpp | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/lib/socket.cpp b/lib/socket.cpp index 22f5e32b..865c3be7 100644 --- a/lib/socket.cpp +++ b/lib/socket.cpp @@ -1713,15 +1713,19 @@ void Socket::UDPConnection::SetDestination(std::string destIp, uint32_t port){ memset(&hints, 0, sizeof(struct addrinfo)); hints.ai_family = family; hints.ai_socktype = SOCK_DGRAM; - hints.ai_flags = AI_ADDRCONFIG | AI_V4MAPPED; + hints.ai_flags = AI_ADDRCONFIG | AI_ALL; hints.ai_protocol = IPPROTO_UDP; hints.ai_canonname = NULL; hints.ai_addr = NULL; hints.ai_next = NULL; int s = getaddrinfo(destIp.c_str(), ss.str().c_str(), &hints, &result); if (s != 0){ - FAIL_MSG("Could not connect UDP socket to %s:%i! Error: %s", destIp.c_str(), port, gai_strmagic(s)); - return; + hints.ai_family = AF_UNSPEC; + s = getaddrinfo(destIp.c_str(), ss.str().c_str(), &hints, &result); + if (s != 0){ + FAIL_MSG("Could not connect UDP socket to %s:%i! Error: %s", destIp.c_str(), port, gai_strmagic(s)); + return; + } } for (rp = result; rp != NULL; rp = rp->ai_next){ @@ -1745,7 +1749,12 @@ void Socket::UDPConnection::SetDestination(std::string destIp, uint32_t port){ bind(boundPort, boundAddr, boundMulti); } } - HIGH_MSG("Set UDP destination: %s:%d (%s)", destIp.c_str(), port, addrFam(family)); + { + std::string trueDest; + uint32_t truePort; + GetDestination(trueDest, truePort); + HIGH_MSG("Set UDP destination: %s:%d => %s:%d (%s)", destIp.c_str(), port, trueDest.c_str(), truePort, addrFam(family)); + } freeaddrinfo(result); return; //\todo Possibly detect and handle failure @@ -1881,6 +1890,12 @@ uint16_t Socket::UDPConnection::bind(int port, std::string iface, const std::str for (rp = addr_result; rp != NULL; rp = rp->ai_next){ sock = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol); if (sock == -1){continue;} + if (rp->ai_family == AF_INET6){ + const int optval = 0; + if (setsockopt(sock, SOL_SOCKET, IPV6_V6ONLY, &optval, sizeof(optval)) < 0){ + WARN_MSG("Could not set IPv6 UDP socket to be dual-stack! %s", strerror(errno)); + } + } checkRecvBuf(); char human_addr[INET6_ADDRSTRLEN]; char human_port[16]; @@ -1909,7 +1924,7 @@ uint16_t Socket::UDPConnection::bind(int port, std::string iface, const std::str if (multicast){ const int optval = 1; if (setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval)) < 0){ - WARN_MSG("Could not set multicast UDP socket re-use!"); + WARN_MSG("Could not set multicast UDP socket re-use! %s", strerror(errno)); } } if (::bind(sock, rp->ai_addr, rp->ai_addrlen) == 0){ -- 2.25.1 From ed9910d587b5504d38e4f7ef2777f2ea7bd9b760 Mon Sep 17 00:00:00 2001 From: Thulinma Date: Wed, 3 Aug 2022 14:58:05 +0200 Subject: [PATCH 03/38] Fix for list limit in HLS/CMAF outputs when using non-live --- src/output/output_cmaf.cpp | 2 +- src/output/output_hls.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/output/output_cmaf.cpp b/src/output/output_cmaf.cpp index 756fceb8..4582a8b6 100644 --- a/src/output/output_cmaf.cpp +++ b/src/output/output_cmaf.cpp @@ -284,7 +284,7 @@ namespace Mist{ requestTid, M.biggestFragment(timingTid) / 1000, atol(H.GetVar("iMsn").c_str()), - config->getInteger("listlimit"), + M.getLive()? config->getInteger("listlimit") : 0, urlPrefix, systemBoot, bootMsOffset, diff --git a/src/output/output_hls.cpp b/src/output/output_hls.cpp index 93410f54..19d083e6 100644 --- a/src/output/output_hls.cpp +++ b/src/output/output_hls.cpp @@ -173,7 +173,7 @@ namespace Mist{ requestTid, M.biggestFragment(timingTid) / 1000, atol(H.GetVar("iMsn").c_str()), - config->getInteger("listlimit"), + M.getLive()? config->getInteger("listlimit") : 0, urlPrefix, systemBoot, bootMsOffset, -- 2.25.1 From c37aac8898484903da8c2095d29125bf69888c0b Mon Sep 17 00:00:00 2001 From: Thulinma Date: Thu, 4 Aug 2022 09:14:51 +0200 Subject: [PATCH 04/38] Remove "Rate=..." debug message from RTMP push output --- src/output/output_rtmp.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/output/output_rtmp.cpp b/src/output/output_rtmp.cpp index 69d5f91e..16a400d0 100644 --- a/src/output/output_rtmp.cpp +++ b/src/output/output_rtmp.cpp @@ -618,7 +618,6 @@ namespace Mist{ if (type == "audio"){ uint32_t rate = M.getRate(thisIdx); - WARN_MSG("Rate=%i", rate); rtmpheader[7] = 0x08; if (codec == "AAC"){ dataheader[0] += 0xA0; -- 2.25.1 From 4084768a3eb72026e901118e05fd17115dd785b3 Mon Sep 17 00:00:00 2001 From: Thulinma Date: Tue, 5 Jul 2022 15:21:04 +0200 Subject: [PATCH 05/38] Improved incoming push accept timing --- src/output/output.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/output/output.cpp b/src/output/output.cpp index 5c785bb0..1fa86eee 100644 --- a/src/output/output.cpp +++ b/src/output/output.cpp @@ -2050,7 +2050,7 @@ namespace Mist{ INFO_MSG("Waiting for stream reset before attempting push input accept (%" PRIu64 " <= %" PRIu64 "+500)", twoTime, oneTime); while (streamStatus != STRMSTAT_OFF && keepGoing()){ userSelect.clear(); - Util::wait(1000); + Util::wait(250); streamStatus = Util::getStreamStatus(streamName); } reconnect(); @@ -2059,14 +2059,15 @@ namespace Mist{ while (((streamStatus != STRMSTAT_WAIT && streamStatus != STRMSTAT_READY) || !meta) && keepGoing()){ INFO_MSG("Waiting for %s buffer to be ready... (%u)", streamName.c_str(), streamStatus); disconnect(); - Util::wait(1000); streamStatus = Util::getStreamStatus(streamName); if (streamStatus == STRMSTAT_OFF || streamStatus == STRMSTAT_WAIT || streamStatus == STRMSTAT_READY){ INFO_MSG("Reconnecting to %s buffer... (%u)", streamName.c_str(), streamStatus); - Util::wait(500); reconnect(); streamStatus = Util::getStreamStatus(streamName); } + if (((streamStatus != STRMSTAT_WAIT && streamStatus != STRMSTAT_READY) || !meta) && keepGoing()){ + Util::wait(100); + } } if (streamStatus == STRMSTAT_READY || streamStatus == STRMSTAT_WAIT){reconnect();} if (!meta){ -- 2.25.1 From 14bc94ece51affb7cb6d52a491b97921ed420587 Mon Sep 17 00:00:00 2001 From: Thulinma Date: Tue, 12 Jul 2022 15:47:29 +0200 Subject: [PATCH 06/38] Prevent eternal sleep when waiting for RelAccX structure to become ready --- lib/util.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/util.cpp b/lib/util.cpp index d522a151..8dd75727 100644 --- a/lib/util.cpp +++ b/lib/util.cpp @@ -446,7 +446,15 @@ namespace Util{ hdrOffset = (uint16_t*)(p+26); hdrEndPos = (uint64_t*)(p+28); if (waitReady){ - while (!isReady()){Util::sleep(50);} + uint64_t maxWait = Util::bootMS() + 10000; + while (!isReady()){ + if (Util::bootMS() > maxWait){ + FAIL_MSG("Waiting for RelAccX structure to be ready timed out, aborting"); + p = 0; + return; + } + Util::sleep(50); + } } if (isReady()){ uint16_t offset = RAXHDR_FIELDOFFSET; -- 2.25.1 From 5bdd4a416e964ae0cfcc56d99c062691d1b3931d Mon Sep 17 00:00:00 2001 From: Thulinma Date: Thu, 14 Jul 2022 10:45:53 +0200 Subject: [PATCH 07/38] Fix load balancer CPU usage --- src/input/input_balancer.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/input/input_balancer.cpp b/src/input/input_balancer.cpp index 448e6eff..36db5709 100644 --- a/src/input/input_balancer.cpp +++ b/src/input/input_balancer.cpp @@ -140,6 +140,9 @@ namespace Mist{ startTime = 0; // note success break; // break out of while loop } + }else{ + //Prevent 100% CPU usage if the response is slow + Util::sleep(25); } } if (startTime){ -- 2.25.1 From fffe98804cb9c126ec91b7a7b67f2c917561ce02 Mon Sep 17 00:00:00 2001 From: Thulinma Date: Thu, 30 Jun 2022 14:04:34 +0200 Subject: [PATCH 08/38] Fixed TS SRT input not closing the connection when stopping for internal reasons rather than external reasons --- src/input/input_tssrt.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/input/input_tssrt.cpp b/src/input/input_tssrt.cpp index d12f9cba..6cd614e7 100644 --- a/src/input/input_tssrt.cpp +++ b/src/input/input_tssrt.cpp @@ -241,6 +241,7 @@ namespace Mist{ } // If we are here: we have a proper connection (either accepted or pull input) and should start parsing it as such Input::streamMainLoop(); + srtConn.close(); } bool inputTSSRT::needsLock(){return false;} -- 2.25.1 From 267a74f0f6d44b928b99312b356a3080579fdc1c Mon Sep 17 00:00:00 2001 From: Thulinma Date: Thu, 4 Aug 2022 16:43:04 +0200 Subject: [PATCH 09/38] Fix track selector logic when multiple selections are considered --- lib/stream.cpp | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/lib/stream.cpp b/lib/stream.cpp index 6036a8fd..715015d6 100644 --- a/lib/stream.cpp +++ b/lib/stream.cpp @@ -1249,7 +1249,7 @@ std::set Util::wouldSelect(const DTSC::Meta &M, const std::map result; /*LTS-START*/ - bool noSelAudio = false, noSelVideo = false, noSelSub = false; + bool noSelAudio = false, noSelVideo = false, noSelSub = false, noSelMeta = false; // Then, select the tracks we've been asked to select. if (targetParams.count("audio") && targetParams.at("audio").size()){ if (targetParams.at("audio") != "-1" && targetParams.at("audio") != "none"){ @@ -1270,6 +1270,11 @@ std::set Util::wouldSelect(const DTSC::Meta &M, const std::map tracks = Util::findTracks(M, capa, "meta", targetParams.at("meta"), UA); + result.insert(tracks.begin(), tracks.end()); + noSelMeta = true; + } /*LTS-END*/ std::set validTracks = M.getValidTracks(); @@ -1291,6 +1296,7 @@ std::set Util::wouldSelect(const DTSC::Meta &M, const std::map Util::wouldSelect(const DTSC::Meta &M, const std::map Util::wouldSelect(const DTSC::Meta &M, const std::map 0){ jsonForEachConst((*it), itb){ if ((*itb).size() > 0){ @@ -1360,6 +1368,26 @@ std::set Util::wouldSelect(const DTSC::Meta &M, const std::map::iterator itd = validTracks.begin(); itd != validTracks.end(); itd++){ + if ((!byType && M.getCodec(*itd) == strRef.substr(shift)) || + (byType && M.getType(*itd) == strRef.substr(shift)) || strRef.substr(shift) == "*"){ + // user-agent-check + bool problems = false; + if (capa.isMember("exceptions") && capa["exceptions"].isObject() && + capa["exceptions"].size()){ + jsonForEachConst(capa["exceptions"], ex){ + if (ex.key() == "codec:" + strRef.substr(shift)){ + problems = !Util::checkException(*ex, UA); + break; + } + } + } + if (problems){break;} + extraCounter++; if (!multiSel){break;} } } @@ -1367,8 +1395,9 @@ std::set Util::wouldSelect(const DTSC::Meta &M, const std::map bestSoFarCount){ + if (selCounter > bestSoFarCount || (selCounter == bestSoFarCount && extraCounter > bestSoFarCountExtra)){ bestSoFarCount = selCounter; + bestSoFarCountExtra = extraCounter; bestSoFar = index; HIGH_MSG("Matched %u: %s", selCounter, (*it).toString().c_str()); } -- 2.25.1 From 3734c90544a68d03a1a69438f00f215c517fec01 Mon Sep 17 00:00:00 2001 From: Phencys Date: Sun, 20 Sep 2020 20:31:17 +0200 Subject: [PATCH 10/38] Added support for raw passthrough of MPEG2-TS data --- src/input/input_ts.cpp | 73 ++++++++++++++++++++++++++++++++--- src/input/input_ts.h | 5 +++ src/input/input_tsrist.cpp | 54 +++++++++++++++++++++++++- src/input/input_tsrist.h | 6 +++ src/input/input_tssrt.cpp | 35 +++++++++++++++++ src/input/input_tssrt.h | 4 ++ src/io.cpp | 7 ++++ src/io.h | 1 + src/output/output_httpts.cpp | 1 + src/output/output_ts.cpp | 1 + src/output/output_ts_base.cpp | 5 +++ src/output/output_tsrist.cpp | 1 + src/output/output_tssrt.cpp | 1 + 13 files changed, 187 insertions(+), 7 deletions(-) diff --git a/src/input/input_ts.cpp b/src/input/input_ts.cpp index 98300411..cfe3f7cd 100644 --- a/src/input/input_ts.cpp +++ b/src/input/input_ts.cpp @@ -134,6 +134,13 @@ void parseThread(void *mistIn){ } } } + + //On shutdown, make sure to clean up stream buffer + if (idx != INVALID_TRACK_ID){ + tthread::lock_guard guard(threadClaimMutex); + input->liveFinalize(idx); + } + std::string reason = "unknown reason"; if (!(Util::bootSecs() - threadTimer[tid] < THREAD_TIMEOUT)){reason = "thread timeout";} if (!cfgPointer->is_active){reason = "input shutting down";} @@ -155,6 +162,9 @@ namespace Mist{ /// Constructor of TS Input /// \arg cfg Util::Config that contains all current configurations. inputTS::inputTS(Util::Config *cfg) : Input(cfg){ + rawMode = false; + rawIdx = INVALID_TRACK_ID; + lastRawPacket = 0; capa["name"] = "TS"; capa["desc"] = "This input allows you to stream MPEG2-TS data from static files (/*.ts), streamed files " @@ -188,6 +198,7 @@ namespace Mist{ capa["codecs"][0u][1u].append("AC3"); capa["codecs"][0u][1u].append("MP2"); capa["codecs"][0u][1u].append("opus"); + capa["codecs"][1u][0u].append("rawts"); inFile = NULL; inputProcess = 0; isFinished = false; @@ -232,6 +243,16 @@ namespace Mist{ "Alternative stream to load for playback when there is no active broadcast"; capa["optional"]["fallback_stream"]["type"] = "str"; capa["optional"]["fallback_stream"]["default"] = ""; + + capa["optional"]["raw"]["name"] = "Raw input mode"; + capa["optional"]["raw"]["help"] = "Enable raw MPEG-TS passthrough mode"; + capa["optional"]["raw"]["option"] = "--raw"; + + JSON::Value option; + option["long"] = "raw"; + option["short"] = "R"; + option["help"] = "Enable raw MPEG-TS passthrough mode"; + config->addOption("raw", option); } inputTS::~inputTS(){ @@ -257,6 +278,10 @@ namespace Mist{ /// Live Setup of TS Input bool inputTS::preRun(){ INFO_MSG("Prerun: %s", config->getString("input").c_str()); + + rawMode = config->getBool("raw"); + if (rawMode){INFO_MSG("Entering raw mode");} + // streamed standard input if (config->getString("input") == "-"){ standAlone = false; @@ -520,9 +545,28 @@ namespace Mist{ } if (tcpCon.Received().available(188) && tcpCon.Received().get()[0] == 0x47){ std::string newData = tcpCon.Received().remove(188); - tsBuf.FromPointer(newData.data()); - liveStream.add(tsBuf); - if (!liveStream.isDataTrack(tsBuf.getPID())){liveStream.parse(tsBuf.getPID());} + if (rawMode){ + keepAlive(); + rawBuffer.append(newData); + if (rawBuffer.size() >= 1316 && (lastRawPacket == 0 || lastRawPacket != Util::bootMS())){ + if (rawIdx == INVALID_TRACK_ID){ + rawIdx = meta.addTrack(); + meta.setType(rawIdx, "meta"); + meta.setCodec(rawIdx, "rawts"); + meta.setID(rawIdx, 1); + userSelect[rawIdx].reload(streamName, rawIdx, COMM_STATUS_SOURCE); + } + uint64_t packetTime = Util::bootMS(); + thisPacket.genericFill(packetTime, 0, 1, rawBuffer, rawBuffer.size(), 0, 0); + bufferLivePacket(thisPacket); + lastRawPacket = packetTime; + rawBuffer.truncate(0); + } + }else { + tsBuf.FromPointer(newData.data()); + liveStream.add(tsBuf); + if (!liveStream.isDataTrack(tsBuf.getPID())){liveStream.parse(tsBuf.getPID());} + } } } noDataSince = Util::bootSecs(); @@ -543,7 +587,26 @@ namespace Mist{ gettingData = true; INFO_MSG("Now receiving UDP data..."); } - assembler.assemble(liveStream, udpCon.data, udpCon.data.size()); + if (rawMode){ + keepAlive(); + rawBuffer.append(udpCon.data, udpCon.data.size()); + if (rawBuffer.size() >= 1316 && (lastRawPacket == 0 || lastRawPacket != Util::bootMS())){ + if (rawIdx == INVALID_TRACK_ID){ + rawIdx = meta.addTrack(); + meta.setType(rawIdx, "meta"); + meta.setCodec(rawIdx, "rawts"); + meta.setID(rawIdx, 1); + userSelect[rawIdx].reload(streamName, rawIdx, COMM_STATUS_SOURCE); + } + uint64_t packetTime = Util::bootMS(); + thisPacket.genericFill(packetTime, 0, 1, rawBuffer, rawBuffer.size(), 0, 0); + bufferLivePacket(thisPacket); + lastRawPacket = packetTime; + rawBuffer.truncate(0); + } + }else{ + assembler.assemble(liveStream, udpCon.data, udpCon.data.size()); + } } if (!received){ Util::sleep(100); @@ -578,7 +641,7 @@ namespace Mist{ } std::set activeTracks = liveStream.getActiveTracks(); - { + if (!rawMode){ tthread::lock_guard guard(threadClaimMutex); if (hasStarted && !threadTimer.size()){ if (!isAlwaysOn()){ diff --git a/src/input/input_ts.h b/src/input/input_ts.h index e810ddb4..3116725d 100644 --- a/src/input/input_ts.h +++ b/src/input/input_ts.h @@ -41,6 +41,11 @@ namespace Mist{ pid_t inputProcess; size_t tmpIdx; bool isFinished; + + bool rawMode; + Util::ResizeablePointer rawBuffer; + size_t rawIdx; + uint64_t lastRawPacket; }; }// namespace Mist diff --git a/src/input/input_tsrist.cpp b/src/input/input_tsrist.cpp index 18b05c9d..5b95aa9f 100644 --- a/src/input/input_tsrist.cpp +++ b/src/input/input_tsrist.cpp @@ -66,6 +66,10 @@ namespace Mist{ /// Constructor of TS Input /// \arg cfg Util::Config that contains all current configurations. inputTSRIST::inputTSRIST(Util::Config *cfg) : Input(cfg){ + rawMode = false; + rawIdx = INVALID_TRACK_ID; + lastRawPacket = 0; + hasRaw = false; connPtr = this; cnfPtr = config; @@ -96,6 +100,7 @@ namespace Mist{ capa["codecs"][0u][1u].append("AC3"); capa["codecs"][0u][1u].append("MP2"); capa["codecs"][0u][1u].append("opus"); + capa["codecs"][1u][0u].append("rawts"); JSON::Value option; option["arg"] = "integer"; @@ -132,6 +137,15 @@ namespace Mist{ capa["optional"]["profile"]["type"] = "select"; capa["optional"]["profile"]["option"] = "--profile"; + capa["optional"]["raw"]["name"] = "Raw input mode"; + capa["optional"]["raw"]["help"] = "Enable raw MPEG-TS passthrough mode"; + capa["optional"]["raw"]["option"] = "--raw"; + + option["long"] = "raw"; + option["short"] = "R"; + option["help"] = "Enable raw MPEG-TS passthrough mode"; + config->addOption("raw", option); + lastTimeStamp = 0; timeStampOffset = 0; receiver_ctx = 0; @@ -146,6 +160,9 @@ namespace Mist{ /// Live Setup of SRT Input. Runs only if we are the "main" thread bool inputTSRIST::preRun(){ + rawMode = config->getBool("raw"); + if (rawMode){INFO_MSG("Entering raw mode");} + std::string source = config->getString("input"); standAlone = false; HTTP::URL u(source); @@ -161,6 +178,20 @@ namespace Mist{ // Retrieve the next packet to be played from the srt connection. void inputTSRIST::getNext(size_t idx){ thisPacket.null(); + if (rawMode){ + //Set to false so the other thread knows its safe to fill + hasRaw = false; + while (!hasRaw && config->is_active){ + Util::sleep(50); + if (!bufferActive()){ + Util::logExitReason("Buffer shut down"); + return; + } + } + //if hasRaw, thisPacket has been filled by the other thread + return; + } + while (!thisPacket && config->is_active){ if (tsStream.hasPacket()){ tsStream.getEarliestPacket(thisPacket); @@ -228,8 +259,27 @@ namespace Mist{ } void inputTSRIST::addData(const char * ptr, size_t len){ - for (size_t o = 0; o <= len-188; o += 188){ - tsStream.parse((char*)ptr+o, 0); + for (size_t o = 0; o+188 <= len; o += 188){ + if (rawMode){ + rawBuffer.append(ptr+o, 188); + if (!hasRaw && rawBuffer.size() >= 1316 && (lastRawPacket == 0 || lastRawPacket != Util::bootMS())){ + if (rawIdx == INVALID_TRACK_ID){ + rawIdx = meta.addTrack(); + meta.setType(rawIdx, "meta"); + meta.setCodec(rawIdx, "rawts"); + meta.setID(rawIdx, 1); + userSelect[rawIdx].reload(streamName, rawIdx, COMM_STATUS_SOURCE); + } + thisTime = Util::bootMS(); + thisIdx = rawIdx; + thisPacket.genericFill(thisTime, 0, 1, rawBuffer, rawBuffer.size(), 0, 0); + lastRawPacket = thisTime; + rawBuffer.truncate(0); + hasRaw = true; + } + }else{ + tsStream.parse((char*)ptr+o, 0); + } } } diff --git a/src/input/input_tsrist.h b/src/input/input_tsrist.h index 3bc29e1c..731f9b04 100644 --- a/src/input/input_tsrist.h +++ b/src/input/input_tsrist.h @@ -33,6 +33,12 @@ namespace Mist{ virtual void connStats(Comms::Statistics &statComm); struct rist_ctx *receiver_ctx; + + bool rawMode; + Util::ResizeablePointer rawBuffer; + size_t rawIdx; + uint64_t lastRawPacket; + bool hasRaw; }; }// namespace Mist diff --git a/src/input/input_tssrt.cpp b/src/input/input_tssrt.cpp index 6cd614e7..da69be29 100644 --- a/src/input/input_tssrt.cpp +++ b/src/input/input_tssrt.cpp @@ -25,6 +25,7 @@ Util::Config *cfgPointer = NULL; std::string baseStreamName; Socket::SRTServer sSock; +bool rawMode = false; void (*oldSignal)(int, siginfo_t *,void *) = 0; @@ -49,6 +50,8 @@ namespace Mist{ /// Constructor of TS Input /// \arg cfg Util::Config that contains all current configurations. inputTSSRT::inputTSSRT(Util::Config *cfg, SRTSOCKET s) : Input(cfg){ + rawIdx = INVALID_TRACK_ID; + lastRawPacket = 0; capa["name"] = "TSSRT"; capa["desc"] = "This input allows for processing MPEG2-TS-based SRT streams. Use mode=listener " "for push input."; @@ -66,6 +69,7 @@ namespace Mist{ capa["codecs"][0u][1u].append("AC3"); capa["codecs"][0u][1u].append("MP2"); capa["codecs"][0u][1u].append("opus"); + capa["codecs"][1u][0u].append("rawts"); JSON::Value option; option["arg"] = "integer"; @@ -103,7 +107,16 @@ namespace Mist{ capa["optional"]["acceptable"]["select"][2u][0u] = 2; capa["optional"]["acceptable"]["select"][2u][1u] = "Disallow non-matching streamid"; + capa["optional"]["raw"]["name"] = "Raw input mode"; + capa["optional"]["raw"]["help"] = "Enable raw MPEG-TS passthrough mode"; + capa["optional"]["raw"]["option"] = "--raw"; + option.null(); + option["long"] = "raw"; + option["short"] = "R"; + option["help"] = "Enable raw MPEG-TS passthrough mode"; + config->addOption("raw", option); + // Setup if we are called form with a thread for push-based input. if (s != -1){ srtConn = Socket::SRTConnection(s); @@ -131,6 +144,8 @@ namespace Mist{ /// Live Setup of SRT Input. Runs only if we are the "main" thread bool inputTSSRT::preRun(){ + rawMode = config->getBool("raw"); + if (rawMode){INFO_MSG("Entering raw mode");} if (srtConn.getSocket() == -1){ std::string source = config->getString("input"); standAlone = false; @@ -183,6 +198,26 @@ namespace Mist{ size_t recvSize = srtConn.RecvNow(); if (recvSize){ + if (rawMode){ + keepAlive(); + rawBuffer.append(srtConn.recvbuf, recvSize); + if (rawBuffer.size() >= 1316 && (lastRawPacket == 0 || lastRawPacket != Util::bootMS())){ + if (rawIdx == INVALID_TRACK_ID){ + rawIdx = meta.addTrack(); + meta.setType(rawIdx, "meta"); + meta.setCodec(rawIdx, "rawts"); + meta.setID(rawIdx, 1); + userSelect[rawIdx].reload(streamName, rawIdx, COMM_STATUS_SOURCE); + } + uint64_t packetTime = Util::bootMS(); + thisPacket.genericFill(packetTime, 0, 1, rawBuffer, rawBuffer.size(), 0, 0); + bufferLivePacket(thisPacket); + lastRawPacket = packetTime; + rawBuffer.truncate(0); + return; + } + continue; + } if (assembler.assemble(tsStream, srtConn.recvbuf, recvSize, true)){hasPacket = tsStream.hasPacket();} }else if (srtConn){ // This should not happen as the SRT socket is read blocking and won't return until there is diff --git a/src/input/input_tssrt.h b/src/input/input_tssrt.h index 40fa05c1..4f337b48 100644 --- a/src/input/input_tssrt.h +++ b/src/input/input_tssrt.h @@ -41,6 +41,10 @@ namespace Mist{ Socket::SRTConnection srtConn; bool singularFlag; virtual void connStats(Comms::Statistics &statComm); + + Util::ResizeablePointer rawBuffer; + size_t rawIdx; + uint64_t lastRawPacket; }; }// namespace Mist diff --git a/src/io.cpp b/src/io.cpp index 6a5844c4..fea32193 100644 --- a/src/io.cpp +++ b/src/io.cpp @@ -292,6 +292,13 @@ namespace Mist{ tPages.setInt("avail", pageOffset + packDataLen, pageIdx); } + /// Wraps up the buffering of a shared memory data page + /// \param idx The track index of the page to finalize + void InOutBase::liveFinalize(size_t idx){ + if (!livePage.count(idx)){return;} + bufferFinalize(idx, livePage[idx]); + } + /// Wraps up the buffering of a shared memory data page /// \param idx The track index of the page to finalize void InOutBase::bufferFinalize(size_t idx, IPC::sharedPage & page){ diff --git a/src/io.h b/src/io.h index d839952a..e6545929 100644 --- a/src/io.h +++ b/src/io.h @@ -21,6 +21,7 @@ namespace Mist{ bool bufferStart(size_t idx, uint32_t pageNumber, IPC::sharedPage & page, DTSC::Meta & aMeta); void bufferFinalize(size_t idx, IPC::sharedPage & page); + void liveFinalize(size_t idx); bool isCurrentLivePage(size_t idx, uint32_t pageNumber); void bufferRemove(size_t idx, uint32_t pageNumber); void bufferLivePacket(const DTSC::Packet &packet); diff --git a/src/output/output_httpts.cpp b/src/output/output_httpts.cpp index 6aab2b9b..9daa9dc3 100644 --- a/src/output/output_httpts.cpp +++ b/src/output/output_httpts.cpp @@ -135,6 +135,7 @@ namespace Mist{ capa["codecs"][0u][1u].append("+AC3"); capa["codecs"][0u][1u].append("+MP2"); capa["codecs"][0u][1u].append("+opus"); + capa["codecs"][1u][0u].append("rawts"); capa["methods"][0u]["handler"] = "http"; capa["methods"][0u]["type"] = "html5/video/mpeg"; capa["methods"][0u]["hrn"] = "TS HTTP progressive"; diff --git a/src/output/output_ts.cpp b/src/output/output_ts.cpp index d2addc24..561bf9aa 100644 --- a/src/output/output_ts.cpp +++ b/src/output/output_ts.cpp @@ -179,6 +179,7 @@ namespace Mist{ capa["codecs"][0u][1u].append("+AC3"); capa["codecs"][0u][1u].append("+MP2"); capa["codecs"][0u][1u].append("+opus"); + capa["codecs"][1u][0u].append("rawts"); cfg->addConnectorOptions(8888, capa); config = cfg; capa["push_urls"].append("tsudp://*"); diff --git a/src/output/output_ts_base.cpp b/src/output/output_ts_base.cpp index 17b2a8ce..c4163e7f 100644 --- a/src/output/output_ts_base.cpp +++ b/src/output/output_ts_base.cpp @@ -75,6 +75,11 @@ namespace Mist{ size_t dataLen = 0; thisPacket.getString("data", dataPointer, dataLen); // data + if (codec == "rawts"){ + for (size_t i = 0; i+188 <= dataLen; i+=188){sendTS(dataPointer+i, 188);} + return; + } + packTime *= 90; std::string bs; // prepare bufferstring diff --git a/src/output/output_tsrist.cpp b/src/output/output_tsrist.cpp index ee57d8ee..c4e03f80 100644 --- a/src/output/output_tsrist.cpp +++ b/src/output/output_tsrist.cpp @@ -202,6 +202,7 @@ namespace Mist{ capa["codecs"][0u][1u].append("+AC3"); capa["codecs"][0u][1u].append("+MP2"); capa["codecs"][0u][1u].append("+opus"); + capa["codecs"][1u][0u].append("rawts"); capa["optional"]["profile"]["name"] = "RIST profile"; capa["optional"]["profile"]["help"] = "RIST profile to use"; diff --git a/src/output/output_tssrt.cpp b/src/output/output_tssrt.cpp index fb07b11e..db07dc91 100644 --- a/src/output/output_tssrt.cpp +++ b/src/output/output_tssrt.cpp @@ -200,6 +200,7 @@ namespace Mist{ capa["codecs"][0u][1u].append("AC3"); capa["codecs"][0u][1u].append("MP2"); capa["codecs"][0u][1u].append("opus"); + capa["codecs"][1u][0u].append("rawts"); cfg->addConnectorOptions(8889, capa); config = cfg; capa["push_urls"].append("srt://*"); -- 2.25.1 From 6c117b63cfe85a204b48dfddddb355f416b39364 Mon Sep 17 00:00:00 2001 From: Alex Kordic Date: Thu, 4 Aug 2022 14:49:57 +0200 Subject: [PATCH 11/38] Add s3 protocol to `URIReader` --- lib/timing.cpp | 12 +++++ lib/timing.h | 1 + lib/urireader.cpp | 114 +++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 125 insertions(+), 2 deletions(-) diff --git a/lib/timing.cpp b/lib/timing.cpp index 036f0b29..f339f4fe 100644 --- a/lib/timing.cpp +++ b/lib/timing.cpp @@ -114,3 +114,15 @@ std::string Util::getUTCString(uint64_t epoch){ ptm->tm_mday, ptm->tm_hour, ptm->tm_min, ptm->tm_sec); return std::string(result); } + +std::string Util::getDateString(uint64_t epoch){ + char buffer[80]; + time_t rawtime = epoch; + if (!epoch) { + time(&rawtime); + } + struct tm * timeinfo; + timeinfo = localtime(&rawtime); + strftime(buffer, sizeof(buffer), "%a, %d %b %Y %H:%M:%S %z", timeinfo); + return std::string(buffer); +} diff --git a/lib/timing.h b/lib/timing.h index 107ccee5..b2a4d612 100644 --- a/lib/timing.h +++ b/lib/timing.h @@ -17,4 +17,5 @@ namespace Util{ uint64_t getNTP(); uint64_t epoch(); ///< Gets the amount of seconds since 01/01/1970. std::string getUTCString(uint64_t epoch = 0); + std::string getDateString(uint64_t epoch = 0); }// namespace Util diff --git a/lib/urireader.cpp b/lib/urireader.cpp index 80f204c8..ad85b052 100644 --- a/lib/urireader.cpp +++ b/lib/urireader.cpp @@ -3,11 +3,86 @@ #include "timing.h" #include "urireader.h" #include "util.h" +#include "encode.h" #include #include +#include +#include namespace HTTP{ + // When another protocol needs this, rename struct to HeaderOverride or similar + struct HTTPHeadThenGet { + bool continueOperation; + std::string date, headAuthorization, getAuthorization; + + HTTPHeadThenGet() : continueOperation(false) {} + + void prepareHeadHeaders(HTTP::Downloader& downloader) { + if(!continueOperation) return; + downloader.setHeader("Date", date); + downloader.setHeader("Authorization", headAuthorization); + } + + void prepareGetHeaders(HTTP::Downloader& downloader) { + if(!continueOperation) return; + // .setHeader() overwrites existing header value + downloader.setHeader("Date", date); + downloader.setHeader("Authorization", getAuthorization); + } + }; + +#ifndef NOSSL + inline bool s3CalculateSignature(std::string& signature, const std::string method, const std::string date, const std::string& requestPath, const std::string& accessKey, const std::string& secret) { + std::string toSign = method + "\n\n\n" + date + "\n" + requestPath; + unsigned char signatureBytes[MBEDTLS_MD_MAX_SIZE]; + const int sha1Size = 20; + mbedtls_md_context_t md_ctx = {0}; + // TODO: When we use MBEDTLS_MD_SHA512 ? Consult documentation/code + const mbedtls_md_info_t *md_info = mbedtls_md_info_from_type(MBEDTLS_MD_SHA1); + if (!md_info){ FAIL_MSG("error s3 MBEDTLS_MD_SHA1 unavailable"); return false; } + int status = mbedtls_md_setup(&md_ctx, md_info, 1); + if(status != 0) { FAIL_MSG("error s3 mbedtls_md_setup error %d", status); return false; } + status = mbedtls_md_hmac_starts(&md_ctx, (const unsigned char *)secret.c_str(), secret.size()); + if(status != 0) { FAIL_MSG("error s3 mbedtls_md_hmac_starts error %d", status); return false; } + status = mbedtls_md_hmac_update(&md_ctx, (const unsigned char *)toSign.c_str(), toSign.size()); + if(status != 0) { FAIL_MSG("error s3 mbedtls_md_hmac_update error %d", status); return false; } + status = mbedtls_md_hmac_finish(&md_ctx, signatureBytes); + if(status != 0) { FAIL_MSG("error s3 mbedtls_md_hmac_finish error %d", status); return false; } + std::string base64encoded = Encodings::Base64::encode(std::string((const char*)signatureBytes, sha1Size)); + signature = "AWS " + accessKey + ":" + base64encoded; + return true; + } + // Input url == s3+https://s3_key:secret@storage.googleapis.com/alexk-dms-upload-test/testvideo.ts + // Transform to: + // url=https://storage.googleapis.com/alexk-dms-upload-test/testvideo.ts + // header Date: ${Util::getDateString(()} + // header Authorization: AWS ${url.user}:${signature} + inline HTTPHeadThenGet s3TransformToHttp(HTTP::URL& url) { + HTTPHeadThenGet result; + result.date = Util::getDateString(); + // remove "s3+" prefix + url.protocol = url.protocol.erase(0, 3); + // Use user and pass to create signature and remove from HTTP request + std::string accessKey(url.user), secret(url.pass); + url.user = ""; + url.pass = ""; + std::string requestPath = "/" + Encodings::URL::encode(url.path, "/:=@[]#?&"); + if(url.args.size()) requestPath += "?" + url.args; + // Calculate Authorization data + if(!s3CalculateSignature(result.headAuthorization, "HEAD", result.date, requestPath, accessKey, secret)) { + result.continueOperation = false; + return result; + } + if(!s3CalculateSignature(result.getAuthorization, "GET", result.date, requestPath, accessKey, secret)) { + result.continueOperation = false; + return result; + } + result.continueOperation = true; + return result; + } +#endif // ifndef NOSSL + void URIReader::init(){ handle = -1; mapped = 0; @@ -97,12 +172,45 @@ namespace HTTP{ } } + // prepare for s3 and http + HTTPHeadThenGet httpHeaderOverride; + +#ifndef NOSSL + // In case of s3 URI we prepare HTTP request with AWS authorization and rely on HTTP logic below + if (myURI.protocol == "s3+https" || myURI.protocol == "s3+http"){ + // Check fallback to global credentials in env vars + bool haveCredentials = myURI.user.size() && myURI.pass.size(); + if(!haveCredentials) { + // Use environment variables + char * envValue = std::getenv("S3_ACCESS_KEY_ID"); + if(envValue == NULL) { + FAIL_MSG("error s3 uri without credentials. Consider setting S3_ACCESS_KEY_ID env var"); + return false; + } + myURI.user = envValue; + envValue = std::getenv("S3_SECRET_ACCESS_KEY"); + if(envValue == NULL) { + FAIL_MSG("error s3 uri without credentials. Consider setting S3_SECRET_ACCESS_KEY env var"); + return false; + } + myURI.pass = envValue; + } + // Transform s3 url to HTTP request: + httpHeaderOverride = s3TransformToHttp(myURI); + bool errorInSignatureCalculation = !httpHeaderOverride.continueOperation; + if(errorInSignatureCalculation) return false; + // Do not return, continue to HTTP case + } +#endif // ifndef NOSSL + // HTTP, stream or regular download? if (myURI.protocol == "http" || myURI.protocol == "https"){ - stateType = HTTP::HTTP; + stateType = HTTP; + downer.clearHeaders(); + // One set of headers specified for HEAD request + httpHeaderOverride.prepareHeadHeaders(downer); // Send HEAD request to determine range request is supported, and get total length - downer.clearHeaders(); if (userAgentOverride.size()){downer.setHeader("User-Agent", userAgentOverride);} if (!downer.head(myURI) || !downer.isOk()){ FAIL_MSG("Error getting URI info for '%s': %" PRIu32 " %s", myURI.getUrl().c_str(), @@ -120,6 +228,8 @@ namespace HTTP{ myURI = downer.lastURL(); } + // Other set of headers specified for GET request + httpHeaderOverride.prepareGetHeaders(downer); // streaming mode when size is unknown if (!supportRangeRequest){ MEDIUM_MSG("URI get without range request: %s, totalsize: %zu", myURI.getUrl().c_str(), totalSize); -- 2.25.1 From df4076a06eeb6fbc0fc20d9ace0091eb476420a2 Mon Sep 17 00:00:00 2001 From: Thulinma Date: Wed, 17 Aug 2022 14:57:29 +0200 Subject: [PATCH 12/38] Added ResizeablePointer::shift operator to shift data forward in buffer --- lib/util.cpp | 11 +++++++++++ lib/util.h | 1 + 2 files changed, 12 insertions(+) diff --git a/lib/util.cpp b/lib/util.cpp index 8dd75727..d4ff380f 100644 --- a/lib/util.cpp +++ b/lib/util.cpp @@ -191,6 +191,17 @@ namespace Util{ maxSize = 0; } + void ResizeablePointer::shift(size_t byteCount){ + //Shifting the entire buffer is easy, we do nothing and set size to zero + if (byteCount >= currSize){ + currSize = 0; + return; + } + //Shifting partial needs a memmove and size change + memmove(ptr, ((char*)ptr)+byteCount, currSize-byteCount); + currSize -= byteCount; + } + bool ResizeablePointer::assign(const void *p, uint32_t l){ if (!allocate(l)){return false;} memcpy(ptr, p, l); diff --git a/lib/util.h b/lib/util.h index 84b0d56c..d28a7f76 100644 --- a/lib/util.h +++ b/lib/util.h @@ -45,6 +45,7 @@ namespace Util{ bool append(const void *p, uint32_t l); bool append(const std::string &str); bool allocate(uint32_t l); + void shift(size_t byteCount); uint32_t rsize(); void truncate(const size_t newLen); inline operator char *(){return (char *)ptr;} -- 2.25.1 From 44c28097332b0a137c4674bcfd8d243b593b3cee Mon Sep 17 00:00:00 2001 From: Thulinma Date: Thu, 18 Aug 2022 02:44:11 +0200 Subject: [PATCH 13/38] Decreased verbosity of harmless data offset warnings, increased verbosity of harmful ones --- src/input/input.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/input/input.cpp b/src/input/input.cpp index 942a55f8..0a062392 100644 --- a/src/input/input.cpp +++ b/src/input/input.cpp @@ -1382,10 +1382,12 @@ namespace Mist{ size_t currPos = tPages.getInt("avail", pageIdx); if (currPos){ size_t keySize = keys.getSize(keyNum); - if (currPos-prevPos != keySize){ - INFO_MSG("Key %" PRIu32 " was %zu bytes but should've been %zu bytes! (differs %d)", keyNum, currPos-prevPos, keySize, (int)(currPos-prevPos-keySize)); - }else{ + if (currPos-prevPos == keySize){ VERYHIGH_MSG("Key %" PRIu32 " was %zu bytes", keyNum, currPos-prevPos); + }else if (currPos-prevPos > keySize){ + FAIL_MSG("Key %" PRIu32 " was %zu bytes but should've been %zu bytes! (differs %d)", keyNum, currPos-prevPos, keySize, (int)(currPos-prevPos-keySize)); + }else{ + MEDIUM_MSG("Key %" PRIu32 " was %zu bytes but should've been %zu bytes! (differs %d)", keyNum, currPos-prevPos, keySize, (int)(currPos-prevPos-keySize)); } ++keyNum; prevPos = currPos; @@ -1412,8 +1414,12 @@ namespace Mist{ size_t currPos = tPages.getInt("avail", pageIdx); if (currPos){ size_t keySize = keys.getSize(keyNum); - if (currPos-prevPos != keySize){ - INFO_MSG("Key %" PRIu32 " was %zu bytes but should've been %zu bytes! (differs %d)", keyNum, currPos-prevPos, keySize, (int)(currPos-prevPos-keySize)); + if (currPos-prevPos == keySize){ + VERYHIGH_MSG("Key %" PRIu32 " was %zu bytes", keyNum, currPos-prevPos); + }else if (currPos-prevPos > keySize){ + FAIL_MSG("Key %" PRIu32 " was %zu bytes but should've been %zu bytes! (differs %d)", keyNum, currPos-prevPos, keySize, (int)(currPos-prevPos-keySize)); + }else{ + MEDIUM_MSG("Key %" PRIu32 " was %zu bytes but should've been %zu bytes! (differs %d)", keyNum, currPos-prevPos, keySize, (int)(currPos-prevPos-keySize)); } ++keyNum; prevPos = currPos; -- 2.25.1 From 747438746c068f9217f804fab4d58bdf57156a43 Mon Sep 17 00:00:00 2001 From: Thulinma Date: Thu, 18 Aug 2022 15:45:46 +0200 Subject: [PATCH 14/38] Change VoD data page logic to use wallclock seconds rather than loop iterations for timeouts --- src/input/input.cpp | 30 ++++++++++++++++++++++-------- src/input/input.h | 2 +- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/input/input.cpp b/src/input/input.cpp index 0a062392..a86fe3cd 100644 --- a/src/input/input.cpp +++ b/src/input/input.cpp @@ -1022,28 +1022,42 @@ namespace Mist{ } void Input::removeUnused(){ + uint64_t cTime = Util::bootSecs(); std::set validTracks = M.getValidTracks(); + std::map > checkedPages; for (std::set::iterator it = validTracks.begin(); it != validTracks.end(); ++it){ Util::RelAccX &tPages = meta.pages(*it); for (size_t i = tPages.getDeleted(); i < tPages.getEndPos(); i++){ uint64_t pageNum = tPages.getInt("firstkey", i); + checkedPages[*it].insert(pageNum); if (pageCounter[*it].count(pageNum)){ // If the page is still being written to, reset the counter rather than potentially unloading it if (isCurrentLivePage(*it, pageNum)){ - pageCounter[*it][pageNum] = DEFAULT_PAGE_TIMEOUT; + pageCounter[*it][pageNum] = cTime; continue; } - --pageCounter[*it][pageNum]; - if (!pageCounter[*it][pageNum]){ + if (cTime > pageCounter[*it][pageNum] + DEFAULT_PAGE_TIMEOUT){ pageCounter[*it].erase(pageNum); bufferRemove(*it, pageNum); } + }else{ + pageCounter[*it][pageNum] = cTime; } - else{ - pageCounter[*it][pageNum] = DEFAULT_PAGE_TIMEOUT; + } + } + //Check pages we buffered but forgot about + for (std::map >::iterator it = pageCounter.begin(); + it != pageCounter.end(); it++){ + for (std::map::iterator it2 = it->second.begin(); it2 != it->second.end(); it2++){ + if (!checkedPages.count(it->first) || !checkedPages[it->first].count(it2->first)){ + INFO_MSG("Deleting forgotten page %zu:%" PRIu32, it->first, it2->first); + bufferRemove(it->first, it2->first); + it->second.erase(it2); + it2 = it->second.begin(); } } } + } std::string formatGUID(const std::string &val){ @@ -1290,8 +1304,8 @@ namespace Mist{ } uint32_t pageNumber = tPages.getInt("firstkey", pageIdx); if (isBuffered(idx, pageNumber, meta)){ - // Mark the page for removal after 15 seconds of no one watching it - pageCounter[idx][pageNumber] = DEFAULT_PAGE_TIMEOUT; + // Mark the page as still actively requested + pageCounter[idx][pageNumber] = Util::bootSecs(); DONTEVEN_MSG("Track %zu, key %" PRIu32 " is already buffered in page %" PRIu32 ". Cancelling bufferFrame", idx, keyNum, pageNumber); @@ -1432,7 +1446,7 @@ namespace Mist{ idx, pageNumber, tPages.getInt("firsttime", pageIdx), thisTime, bufferTimer); INFO_MSG(" (%" PRIu32 "/%" PRIu64 " parts, %" PRIu64 " bytes)", packCounter, tPages.getInt("parts", pageIdx), byteCounter); - pageCounter[idx][pageNumber] = DEFAULT_PAGE_TIMEOUT; + pageCounter[idx][pageNumber] = Util::bootSecs(); return true; } diff --git a/src/input/input.h b/src/input/input.h index 28880337..8d7e8891 100644 --- a/src/input/input.h +++ b/src/input/input.h @@ -87,7 +87,7 @@ namespace Mist{ IPC::sharedPage streamStatus; - std::map > pageCounter; + std::map > pageCounter; static Input *singleton; -- 2.25.1 From b210b4f5afc98b3cb4ea46e607267e12cf868831 Mon Sep 17 00:00:00 2001 From: Thulinma Date: Wed, 17 Aug 2022 17:06:25 +0200 Subject: [PATCH 15/38] Fixed seek-related bugs in URIReader for HTTP sources --- lib/urireader.cpp | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/lib/urireader.cpp b/lib/urireader.cpp index ad85b052..d4dd0e31 100644 --- a/lib/urireader.cpp +++ b/lib/urireader.cpp @@ -255,20 +255,30 @@ namespace HTTP{ // seek to pos, return true if succeeded. bool URIReader::seek(const uint64_t pos){ - if (isSeekable()){ - allData.truncate(0); - bufPos = 0; - if (stateType == HTTP::File){ - curPos = pos; - return true; - }else if (stateType == HTTP::HTTP && supportRangeRequest){ - INFO_MSG("SEEK: RangeRequest to %" PRIu64, pos); - if (!downer.getRangeNonBlocking(myURI.getUrl(), pos, 0)){ - FAIL_MSG("error loading request"); - } - } + //Seeking in a non-seekable source? No-op, always fails. + if (!isSeekable()){return false;} + + //Reset internal buffers + allData.truncate(0); + bufPos = 0; + + //Files always succeed because we use memmap + if (stateType == HTTP::File){ + curPos = pos; + return true; } + //HTTP-based needs to do a range request + if (stateType == HTTP::HTTP && supportRangeRequest){ + downer.getSocket().close(); + downer.getSocket().Received().clear(); + if (!downer.getRangeNonBlocking(myURI.getUrl(), pos, 0)){ + FAIL_MSG("Error making range request"); + return false; + } + curPos = pos; + return true; + } return false; } @@ -323,19 +333,13 @@ namespace HTTP{ }else if (stateType == HTTP::HTTP){ downer.continueNonBlocking(cb); - if (curPos == downer.const_data().size()){ - Util::sleep(50); - } - curPos = downer.const_data().size(); }else{// streaming mode int s; - static int totaal = 0; if ((downer.getSocket() && downer.getSocket().spool())){// || downer.getSocket().Received().size() > 0){ s = downer.getSocket().Received().bytes(wantedLen); std::string buf = downer.getSocket().Received().remove(s); cb.dataCallback(buf.data(), s); - totaal += s; }else{ Util::sleep(50); } -- 2.25.1 From 01a2ff54edb0cb544495bff57d8cc2a61440db2a Mon Sep 17 00:00:00 2001 From: Thulinma Date: Wed, 17 Aug 2022 14:57:48 +0200 Subject: [PATCH 16/38] Converted MP4 input to use URIReader --- src/input/input_mp4.cpp | 518 ++++++++++++++++++++++------------------ src/input/input_mp4.h | 15 +- 2 files changed, 298 insertions(+), 235 deletions(-) diff --git a/src/input/input_mp4.cpp b/src/input/input_mp4.cpp index 2e1c7923..f85911fb 100644 --- a/src/input/input_mp4.cpp +++ b/src/input/input_mp4.cpp @@ -159,11 +159,14 @@ namespace Mist{ } inputMP4::inputMP4(Util::Config *cfg) : Input(cfg){ - malSize = 4; // initialise data read buffer to 0; - data = (char *)malloc(malSize); capa["name"] = "MP4"; capa["desc"] = "This input allows streaming of MP4 files as Video on Demand."; - capa["source_match"] = "/*.mp4"; + capa["source_match"].append("/*.mp4"); + capa["source_match"].append("http://*.mp4"); + capa["source_match"].append("https://*.mp4"); + capa["source_match"].append("s3+http://*.mp4"); + capa["source_match"].append("s3+https://*.mp4"); + capa["source_match"].append("mp4:*"); capa["source_file"] = "$source"; capa["priority"] = 9; capa["codecs"][0u][0u].append("HEVC"); @@ -173,10 +176,9 @@ namespace Mist{ capa["codecs"][0u][1u].append("AAC"); capa["codecs"][0u][1u].append("AC3"); capa["codecs"][0u][1u].append("MP3"); + readPos = 0; } - inputMP4::~inputMP4(){free(data);} - bool inputMP4::checkArguments(){ if (config->getString("input") == "-"){ std::cerr << "Input from stdin not yet supported" << std::endl; @@ -199,255 +201,292 @@ namespace Mist{ bool inputMP4::preRun(){ // open File - inFile = fopen(config->getString("input").c_str(), "r"); + std::string inUrl = config->getString("input"); + if (inUrl.size() > 4 && inUrl.substr(0, 4) == "mp4:"){inUrl.erase(0, 4);} + inFile.open(inUrl); if (!inFile){return false;} + if (!inFile.isSeekable()){ + FAIL_MSG("MP4 input only supports seekable data sources, for now, and this source is not seekable: %s", config->getString("input").c_str()); + return false; + } return true; } + void inputMP4::dataCallback(const char *ptr, size_t size){readBuffer.append(ptr, size);} + bool inputMP4::readHeader(){ if (!inFile){ INFO_MSG("inFile failed!"); return false; } + bool hasMoov = false; + readBuffer.truncate(0); + readPos = 0; // first we get the necessary header parts size_t tNumber = 0; - while (!feof(inFile)){ - std::string boxType = MP4::readBoxType(inFile); - if (boxType == "erro"){break;} + activityCounter = Util::bootSecs(); + while (inFile && keepRunning()){ + //Read box header if needed + while (readBuffer.size() < 16 && inFile && keepRunning()){inFile.readSome(16, *this);} + //Failed? Abort. + if (readBuffer.size() < 16){ + FAIL_MSG("Could not read box header from input!"); + break; + } + //Box type is always on bytes 5-8 from the start of a box + std::string boxType = std::string(readBuffer+4, 4); + uint64_t boxSize = MP4::calcBoxSize(readBuffer); if (boxType == "moov"){ - MP4::MOOV moovBox; - moovBox.read(inFile); - // for all box in moov + while (readBuffer.size() < boxSize && inFile && keepRunning()){inFile.readSome(boxSize-readBuffer.size(), *this);} + if (readBuffer.size() < boxSize){ + FAIL_MSG("Could not read entire MOOV box into memory"); + break; + } + MP4::Box moovBox(readBuffer, false); - std::deque trak = moovBox.getChildren(); + // for all box in moov + std::deque trak = ((MP4::MOOV*)&moovBox)->getChildren(); for (std::deque::iterator trakIt = trak.begin(); trakIt != trak.end(); trakIt++){ trackHeaders.push_back(mp4TrackHeader()); trackHeaders.rbegin()->read(*trakIt); } - continue; + hasMoov = true; + break; } - if (!MP4::skipBox(inFile)){// moving on to next box - FAIL_MSG("Error in skipping box, exiting"); - return false; + activityCounter = Util::bootSecs(); + //Skip to next box + if (readBuffer.size() > boxSize){ + readBuffer.shift(boxSize); + readPos += boxSize; + }else{ + readBuffer.truncate(0); + if (!inFile.seek(readPos + boxSize)){ + FAIL_MSG("Seek to %" PRIu64 " failed! Aborting load", readPos+boxSize); + } + readPos = inFile.getPos(); } } - fseeko(inFile, 0, SEEK_SET); // See whether a separate header file exists. - if (readExistingHeader()){return true;} - HIGH_MSG("Not read existing header"); + if (readExistingHeader()){ + bps = 0; + std::set tracks = M.getValidTracks(); + for (std::set::iterator it = tracks.begin(); it != tracks.end(); it++){bps += M.getBps(*it);} + return true; + } + INFO_MSG("Not read existing header"); meta.reInit(isSingular() ? streamName : ""); + if (!hasMoov){ + FAIL_MSG("No MOOV box found; aborting header creation!"); + return false; + } tNumber = 0; // Create header file from MP4 data - while (!feof(inFile)){ - std::string boxType = MP4::readBoxType(inFile); - if (boxType == "erro"){break;} - if (boxType == "moov"){ - MP4::MOOV moovBox; - moovBox.read(inFile); + MP4::Box moovBox(readBuffer, false); - std::deque trak = moovBox.getChildren(); - HIGH_MSG("Obtained %zu trak Boxes", trak.size()); + std::deque trak = ((MP4::MOOV*)&moovBox)->getChildren(); + HIGH_MSG("Obtained %zu trak Boxes", trak.size()); - for (std::deque::iterator trakIt = trak.begin(); trakIt != trak.end(); trakIt++){ - MP4::MDIA mdiaBox = trakIt->getChild(); + for (std::deque::iterator trakIt = trak.begin(); trakIt != trak.end(); trakIt++){ + MP4::MDIA mdiaBox = trakIt->getChild(); - std::string hdlrType = mdiaBox.getChild().getHandlerType(); - if (hdlrType != "vide" && hdlrType != "soun" && hdlrType != "sbtl"){ - INFO_MSG("Unsupported handler: %s", hdlrType.c_str()); - continue; - } + std::string hdlrType = mdiaBox.getChild().getHandlerType(); + if (hdlrType != "vide" && hdlrType != "soun" && hdlrType != "sbtl"){ + INFO_MSG("Unsupported handler: %s", hdlrType.c_str()); + continue; + } - tNumber = meta.addTrack(); + tNumber = meta.addTrack(); - MP4::TKHD tkhdBox = trakIt->getChild(); - if (tkhdBox.getWidth() > 0){ - meta.setWidth(tNumber, tkhdBox.getWidth()); - meta.setHeight(tNumber, tkhdBox.getHeight()); - } - meta.setID(tNumber, tkhdBox.getTrackID()); - - MP4::MDHD mdhdBox = mdiaBox.getChild(); - uint64_t timescale = mdhdBox.getTimeScale(); - meta.setLang(tNumber, mdhdBox.getLanguage()); - - MP4::STBL stblBox = mdiaBox.getChild().getChild(); - - MP4::STSD stsdBox = stblBox.getChild(); - MP4::Box sEntryBox = stsdBox.getEntry(0); - std::string sType = sEntryBox.getType(); - HIGH_MSG("Found track %zu of type %s", tNumber, sType.c_str()); - - if (sType == "avc1" || sType == "h264" || sType == "mp4v"){ - MP4::VisualSampleEntry &vEntryBox = (MP4::VisualSampleEntry &)sEntryBox; - meta.setType(tNumber, "video"); - meta.setCodec(tNumber, "H264"); - if (!meta.getWidth(tNumber)){ - meta.setWidth(tNumber, vEntryBox.getWidth()); - meta.setHeight(tNumber, vEntryBox.getHeight()); - } - MP4::Box initBox = vEntryBox.getCLAP(); - if (initBox.isType("avcC")){ - meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); - } - initBox = vEntryBox.getPASP(); - if (initBox.isType("avcC")){ - meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); - } - /// this is a hacky way around invalid FLV data (since it gets ignored nearly - /// everywhere, but we do need correct data... - if (!meta.getWidth(tNumber)){ - h264::sequenceParameterSet sps; - sps.fromDTSCInit(meta.getInit(tNumber)); - h264::SPSMeta spsChar = sps.getCharacteristics(); - meta.setWidth(tNumber, spsChar.width); - meta.setHeight(tNumber, spsChar.height); - } - } - if (sType == "hev1" || sType == "hvc1"){ - MP4::VisualSampleEntry &vEntryBox = (MP4::VisualSampleEntry &)sEntryBox; - meta.setType(tNumber, "video"); - meta.setCodec(tNumber, "HEVC"); - if (!meta.getWidth(tNumber)){ - meta.setWidth(tNumber, vEntryBox.getWidth()); - meta.setHeight(tNumber, vEntryBox.getHeight()); - } - MP4::Box initBox = vEntryBox.getCLAP(); - if (initBox.isType("hvcC")){ - meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); - } - initBox = vEntryBox.getPASP(); - if (initBox.isType("hvcC")){ - meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); - } - } - if (sType == "mp4a" || sType == "aac " || sType == "ac-3"){ - MP4::AudioSampleEntry &aEntryBox = (MP4::AudioSampleEntry &)sEntryBox; - meta.setType(tNumber, "audio"); - meta.setChannels(tNumber, aEntryBox.getChannelCount()); - meta.setRate(tNumber, aEntryBox.getSampleRate()); - - if (sType == "ac-3"){ - meta.setCodec(tNumber, "AC3"); - }else{ - MP4::ESDS esdsBox = (MP4::ESDS &)(aEntryBox.getCodecBox()); - meta.setCodec(tNumber, esdsBox.getCodec()); - meta.setInit(tNumber, esdsBox.getInitData()); - } - meta.setSize(tNumber, 16); ///\todo this might be nice to calculate from mp4 file; - } - if (sType == "tx3g"){// plain text subtitles - meta.setType(tNumber, "meta"); - meta.setCodec(tNumber, "subtitle"); + MP4::TKHD tkhdBox = trakIt->getChild(); + if (tkhdBox.getWidth() > 0){ + meta.setWidth(tNumber, tkhdBox.getWidth()); + meta.setHeight(tNumber, tkhdBox.getHeight()); + } + meta.setID(tNumber, tkhdBox.getTrackID()); + + MP4::MDHD mdhdBox = mdiaBox.getChild(); + uint64_t timescale = mdhdBox.getTimeScale(); + meta.setLang(tNumber, mdhdBox.getLanguage()); + + MP4::STBL stblBox = mdiaBox.getChild().getChild(); + + MP4::STSD stsdBox = stblBox.getChild(); + MP4::Box sEntryBox = stsdBox.getEntry(0); + std::string sType = sEntryBox.getType(); + HIGH_MSG("Found track %zu of type %s", tNumber, sType.c_str()); + + if (sType == "avc1" || sType == "h264" || sType == "mp4v"){ + MP4::VisualSampleEntry &vEntryBox = (MP4::VisualSampleEntry &)sEntryBox; + meta.setType(tNumber, "video"); + meta.setCodec(tNumber, "H264"); + if (!meta.getWidth(tNumber)){ + meta.setWidth(tNumber, vEntryBox.getWidth()); + meta.setHeight(tNumber, vEntryBox.getHeight()); + } + MP4::Box initBox = vEntryBox.getCLAP(); + if (initBox.isType("avcC")){ + meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); + } + initBox = vEntryBox.getPASP(); + if (initBox.isType("avcC")){ + meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); + } + /// this is a hacky way around invalid FLV data (since it gets ignored nearly + /// everywhere, but we do need correct data... + if (!meta.getWidth(tNumber)){ + h264::sequenceParameterSet sps; + sps.fromDTSCInit(meta.getInit(tNumber)); + h264::SPSMeta spsChar = sps.getCharacteristics(); + meta.setWidth(tNumber, spsChar.width); + meta.setHeight(tNumber, spsChar.height); + } + } + if (sType == "hev1" || sType == "hvc1"){ + MP4::VisualSampleEntry &vEntryBox = (MP4::VisualSampleEntry &)sEntryBox; + meta.setType(tNumber, "video"); + meta.setCodec(tNumber, "HEVC"); + if (!meta.getWidth(tNumber)){ + meta.setWidth(tNumber, vEntryBox.getWidth()); + meta.setHeight(tNumber, vEntryBox.getHeight()); + } + MP4::Box initBox = vEntryBox.getCLAP(); + if (initBox.isType("hvcC")){ + meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); + } + initBox = vEntryBox.getPASP(); + if (initBox.isType("hvcC")){ + meta.setInit(tNumber, initBox.payload(), initBox.payloadSize()); + } + } + if (sType == "mp4a" || sType == "aac " || sType == "ac-3"){ + MP4::AudioSampleEntry &aEntryBox = (MP4::AudioSampleEntry &)sEntryBox; + meta.setType(tNumber, "audio"); + meta.setChannels(tNumber, aEntryBox.getChannelCount()); + meta.setRate(tNumber, aEntryBox.getSampleRate()); + + if (sType == "ac-3"){ + meta.setCodec(tNumber, "AC3"); + }else{ + MP4::ESDS esdsBox = (MP4::ESDS &)(aEntryBox.getCodecBox()); + meta.setCodec(tNumber, esdsBox.getCodec()); + meta.setInit(tNumber, esdsBox.getInitData()); + } + meta.setSize(tNumber, 16); ///\todo this might be nice to calculate from mp4 file; + } + if (sType == "tx3g"){// plain text subtitles + meta.setType(tNumber, "meta"); + meta.setCodec(tNumber, "subtitle"); + } + + MP4::STSS stssBox = stblBox.getChild(); + MP4::STTS sttsBox = stblBox.getChild(); + MP4::STSZ stszBox = stblBox.getChild(); + MP4::STCO stcoBox = stblBox.getChild(); + MP4::CO64 co64Box = stblBox.getChild(); + MP4::STSC stscBox = stblBox.getChild(); + MP4::CTTS cttsBox = stblBox.getChild(); // optional ctts box + + bool stco64 = co64Box.isType("co64"); + bool hasCTTS = cttsBox.isType("ctts"); + + uint64_t totaldur = 0; ///\todo note: set this to begin time + mp4PartBpos BsetPart; + + uint64_t entryNo = 0; + uint64_t sampleNo = 0; + + uint64_t stssIndex = 0; + uint64_t stcoIndex = 0; + uint64_t stscIndex = 0; + uint64_t cttsIndex = 0; // current ctts Index we are reading + uint64_t cttsEntryRead = 0; // current part of ctts we are reading + + uint64_t stssCount = stssBox.getEntryCount(); + uint64_t stscCount = stscBox.getEntryCount(); + uint64_t stszCount = stszBox.getSampleCount(); + uint64_t stcoCount = (stco64 ? co64Box.getEntryCount() : stcoBox.getEntryCount()); + + MP4::STTSEntry sttsEntry = sttsBox.getSTTSEntry(0); + + uint32_t fromSTCOinSTSC = 0; + uint64_t tmpOffset = (stco64 ? co64Box.getChunkOffset(0) : stcoBox.getChunkOffset(0)); + + uint64_t nextFirstChunk = (stscCount > 1 ? stscBox.getSTSCEntry(1).firstChunk - 1 : stcoCount); + + for (uint64_t stszIndex = 0; stszIndex < stszCount; ++stszIndex){ + if (stcoIndex >= nextFirstChunk){ + ++stscIndex; + nextFirstChunk = + (stscIndex + 1 < stscCount ? stscBox.getSTSCEntry(stscIndex + 1).firstChunk - 1 : stcoCount); + } + BsetPart.keyframe = (meta.getType(tNumber) == "video" && stssIndex < stssCount && + stszIndex + 1 == stssBox.getSampleNumber(stssIndex)); + if (BsetPart.keyframe){++stssIndex;} + // in bpos set + BsetPart.stcoNr = stcoIndex; + // bpos = chunkoffset[samplenr] in stco + BsetPart.bpos = tmpOffset; + ++fromSTCOinSTSC; + if (fromSTCOinSTSC < stscBox.getSTSCEntry(stscIndex).samplesPerChunk){// as long as we are still in this chunk + tmpOffset += stszBox.getEntrySize(stszIndex); + }else{ + ++stcoIndex; + fromSTCOinSTSC = 0; + tmpOffset = (stco64 ? co64Box.getChunkOffset(stcoIndex) : stcoBox.getChunkOffset(stcoIndex)); + } + BsetPart.time = (totaldur * 1000) / timescale; + totaldur += sttsEntry.sampleDelta; + sampleNo++; + if (sampleNo >= sttsEntry.sampleCount){ + ++entryNo; + sampleNo = 0; + if (entryNo < sttsBox.getEntryCount()){sttsEntry = sttsBox.getSTTSEntry(entryNo);} + } + + if (hasCTTS){ + MP4::CTTSEntry cttsEntry = cttsBox.getCTTSEntry(cttsIndex); + cttsEntryRead++; + if (cttsEntryRead >= cttsEntry.sampleCount){ + ++cttsIndex; + cttsEntryRead = 0; } + BsetPart.timeOffset = (cttsEntry.sampleOffset * 1000) / timescale; + }else{ + BsetPart.timeOffset = 0; + } - MP4::STSS stssBox = stblBox.getChild(); - MP4::STTS sttsBox = stblBox.getChild(); - MP4::STSZ stszBox = stblBox.getChild(); - MP4::STCO stcoBox = stblBox.getChild(); - MP4::CO64 co64Box = stblBox.getChild(); - MP4::STSC stscBox = stblBox.getChild(); - MP4::CTTS cttsBox = stblBox.getChild(); // optional ctts box - - bool stco64 = co64Box.isType("co64"); - bool hasCTTS = cttsBox.isType("ctts"); - - uint64_t totaldur = 0; ///\todo note: set this to begin time - mp4PartBpos BsetPart; - - uint64_t entryNo = 0; - uint64_t sampleNo = 0; - - uint64_t stssIndex = 0; - uint64_t stcoIndex = 0; - uint64_t stscIndex = 0; - uint64_t cttsIndex = 0; // current ctts Index we are reading - uint64_t cttsEntryRead = 0; // current part of ctts we are reading - - uint64_t stssCount = stssBox.getEntryCount(); - uint64_t stscCount = stscBox.getEntryCount(); - uint64_t stszCount = stszBox.getSampleCount(); - uint64_t stcoCount = (stco64 ? co64Box.getEntryCount() : stcoBox.getEntryCount()); - - MP4::STTSEntry sttsEntry = sttsBox.getSTTSEntry(0); - - uint32_t fromSTCOinSTSC = 0; - uint64_t tmpOffset = (stco64 ? co64Box.getChunkOffset(0) : stcoBox.getChunkOffset(0)); - - uint64_t nextFirstChunk = (stscCount > 1 ? stscBox.getSTSCEntry(1).firstChunk - 1 : stcoCount); - - for (uint64_t stszIndex = 0; stszIndex < stszCount; ++stszIndex){ - if (stcoIndex >= nextFirstChunk){ - ++stscIndex; - nextFirstChunk = - (stscIndex + 1 < stscCount ? stscBox.getSTSCEntry(stscIndex + 1).firstChunk - 1 : stcoCount); - } - BsetPart.keyframe = (meta.getType(tNumber) == "video" && stssIndex < stssCount && - stszIndex + 1 == stssBox.getSampleNumber(stssIndex)); - if (BsetPart.keyframe){++stssIndex;} - // in bpos set - BsetPart.stcoNr = stcoIndex; - // bpos = chunkoffset[samplenr] in stco - BsetPart.bpos = tmpOffset; - ++fromSTCOinSTSC; - if (fromSTCOinSTSC < stscBox.getSTSCEntry(stscIndex).samplesPerChunk){// as long as we are still in this chunk - tmpOffset += stszBox.getEntrySize(stszIndex); - }else{ - ++stcoIndex; - fromSTCOinSTSC = 0; - tmpOffset = (stco64 ? co64Box.getChunkOffset(stcoIndex) : stcoBox.getChunkOffset(stcoIndex)); - } - BsetPart.time = (totaldur * 1000) / timescale; - totaldur += sttsEntry.sampleDelta; - sampleNo++; - if (sampleNo >= sttsEntry.sampleCount){ - ++entryNo; - sampleNo = 0; - if (entryNo < sttsBox.getEntryCount()){sttsEntry = sttsBox.getSTTSEntry(entryNo);} - } - - if (hasCTTS){ - MP4::CTTSEntry cttsEntry = cttsBox.getCTTSEntry(cttsIndex); - cttsEntryRead++; - if (cttsEntryRead >= cttsEntry.sampleCount){ - ++cttsIndex; - cttsEntryRead = 0; - } - BsetPart.timeOffset = (cttsEntry.sampleOffset * 1000) / timescale; - }else{ - BsetPart.timeOffset = 0; - } - - if (sType == "tx3g"){ - if (stszBox.getEntrySize(stszIndex) <= 2 && false){ - FAIL_MSG("size <=2"); - }else{ - long long packSendSize = 0; - packSendSize = 24 + (BsetPart.timeOffset ? 17 : 0) + (BsetPart.bpos ? 15 : 0) + 19 + - stszBox.getEntrySize(stszIndex) + 11 - 2 + 19; - meta.update(BsetPart.time, BsetPart.timeOffset, tNumber, - stszBox.getEntrySize(stszIndex) - 2, BsetPart.bpos, true, packSendSize); - } - }else{ - meta.update(BsetPart.time, BsetPart.timeOffset, tNumber, - stszBox.getEntrySize(stszIndex), BsetPart.bpos, BsetPart.keyframe); - } + if (sType == "tx3g"){ + if (stszBox.getEntrySize(stszIndex) <= 2 && false){ + FAIL_MSG("size <=2"); + }else{ + long long packSendSize = 0; + packSendSize = 24 + (BsetPart.timeOffset ? 17 : 0) + (BsetPart.bpos ? 15 : 0) + 19 + + stszBox.getEntrySize(stszIndex) + 11 - 2 + 19; + meta.update(BsetPart.time, BsetPart.timeOffset, tNumber, + stszBox.getEntrySize(stszIndex) - 2, BsetPart.bpos, true, packSendSize); } + }else{ + meta.update(BsetPart.time, BsetPart.timeOffset, tNumber, + stszBox.getEntrySize(stszIndex), BsetPart.bpos, BsetPart.keyframe); } - continue; - } - if (!MP4::skipBox(inFile)){// moving on to next box - FAIL_MSG("Error in Skipping box, exiting"); - return false; } } - clearerr(inFile); // outputting dtsh file - M.toFile(config->getString("input") + ".dtsh"); + std::string inUrl = config->getString("input"); + if (inUrl.size() > 4 && inUrl.substr(0, 4) == "mp4:"){inUrl.erase(0, 4);} + if (inUrl != "-" && HTTP::URL(inUrl).isLocalPath()){ + M.toFile(inUrl + ".dtsh"); + }else{ + INFO_MSG("Skipping header write, as the source is not a local file"); + } + bps = 0; + std::set tracks = M.getValidTracks(); + for (std::set::iterator it = tracks.begin(); it != tracks.end(); it++){bps += M.getBps(*it);} return true; } @@ -474,23 +513,48 @@ namespace Mist{ ++nextKeyNum; } } - if (fseeko(inFile, curPart.bpos, SEEK_SET)){ - FAIL_MSG("seek unsuccessful @bpos %" PRIu64 ": %s", curPart.bpos, strerror(errno)); - thisPacket.null(); - return; + if (curPart.bpos < readPos || curPart.bpos > readPos + readBuffer.size() + 512*1024 + bps){ + INFO_MSG("Buffer contains %" PRIu64 "-%" PRIu64 ", but we need %" PRIu64 "; seeking!", readPos, readPos + readBuffer.size(), curPart.bpos); + readBuffer.truncate(0); + if (!inFile.seek(curPart.bpos)){ + FAIL_MSG("seek unsuccessful @bpos %" PRIu64 ": %s", curPart.bpos, strerror(errno)); + thisPacket.null(); + return; + } + readPos = inFile.getPos(); + }else{ + //If we have more than 5MiB buffered and are more than 5MiB into the buffer, shift the first 4MiB off the buffer. + //This prevents infinite growth of the read buffer for large files + if (readBuffer.size() >= 5*1024*1024 && curPart.bpos > readPos + 5*1024*1024 + bps){ + readBuffer.shift(4*1024*1024); + readPos += 4*1024*1024; + } } - if (curPart.size > malSize){ - data = (char *)realloc(data, curPart.size); - malSize = curPart.size; + + while (readPos+readBuffer.size() < curPart.bpos+curPart.size && inFile && keepRunning()){ + inFile.readSome((curPart.bpos+curPart.size) - (readPos+readBuffer.size()), *this); } - if (fread(data, curPart.size, 1, inFile) != 1){ - FAIL_MSG("read unsuccessful at %ld", ftell(inFile)); - thisPacket.null(); - return; + if (readPos+readBuffer.size() < curPart.bpos+curPart.size){ + FAIL_MSG("Read unsuccessful at %" PRIu64 ", seeking to retry...", readPos+readBuffer.size()); + readBuffer.truncate(0); + if (!inFile.seek(curPart.bpos)){ + FAIL_MSG("seek unsuccessful @bpos %" PRIu64 ": %s", curPart.bpos, strerror(errno)); + thisPacket.null(); + return; + } + readPos = inFile.getPos(); + while (readPos+readBuffer.size() < curPart.bpos+curPart.size && inFile && keepRunning()){ + inFile.readSome((curPart.bpos+curPart.size) - (readPos+readBuffer.size()), *this); + } + if (readPos+readBuffer.size() < curPart.bpos+curPart.size){ + FAIL_MSG("Read retry unsuccessful at %" PRIu64 ", aborting", readPos+readBuffer.size()); + thisPacket.null(); + return; + } } if (M.getCodec(curPart.trackID) == "subtitle"){ - unsigned int txtLen = Bit::btohs(data); + unsigned int txtLen = Bit::btohs(readBuffer + (curPart.bpos-readPos)); if (!txtLen && false){ curPart.index++; return getNext(idx); @@ -499,14 +563,14 @@ namespace Mist{ thisPack.null(); thisPack["trackid"] = curPart.trackID; thisPack["bpos"] = curPart.bpos; //(long long)fileSource.tellg(); - thisPack["data"] = std::string(data + 2, txtLen); + thisPack["data"] = std::string(readBuffer + (curPart.bpos-readPos) + 2, txtLen); thisPack["time"] = curPart.time; if (curPart.duration){thisPack["duration"] = curPart.duration;} thisPack["keyframe"] = true; std::string tmpStr = thisPack.toNetPacked(); thisPacket.reInit(tmpStr.data(), tmpStr.size()); }else{ - thisPacket.genericFill(curPart.time, curPart.offset, curPart.trackID, data, curPart.size, 0, isKeyframe); + thisPacket.genericFill(curPart.time, curPart.offset, curPart.trackID, readBuffer + (curPart.bpos-readPos), curPart.size, 0, isKeyframe); } thisTime = curPart.time; thisIdx = curPart.trackID; diff --git a/src/input/input_mp4.h b/src/input/input_mp4.h index b5987d3b..4ab9f36e 100644 --- a/src/input/input_mp4.h +++ b/src/input/input_mp4.h @@ -1,5 +1,6 @@ #include "input.h" #include +#include #include #include namespace Mist{ @@ -70,10 +71,10 @@ namespace Mist{ bool stco64; }; - class inputMP4 : public Input{ + class inputMP4 : public Input, public Util::DataCallback { public: inputMP4(Util::Config *cfg); - ~inputMP4(); + void dataCallback(const char *ptr, size_t size); protected: // Private Functions @@ -85,7 +86,10 @@ namespace Mist{ void seek(uint64_t seekTime, size_t idx = INVALID_TRACK_ID); void handleSeek(uint64_t seekTime, size_t idx); - FILE *inFile; + HTTP::URIReader inFile; + Util::ResizeablePointer readBuffer; + uint64_t readPos; + uint64_t bps; mp4TrackHeader &headerData(size_t trackID); @@ -94,11 +98,6 @@ namespace Mist{ // remember last seeked keyframe; std::map nextKeyframe; - - // these next two variables keep a buffer for reading from filepointer inFile; - uint64_t malSize; - char *data; ///\todo rename this variable to a more sensible name, it is a temporary piece of - /// memory to read from files }; }// namespace Mist -- 2.25.1 From 2740d65a0f5715c42eb6de24cbf610ba6271b85a Mon Sep 17 00:00:00 2001 From: Thulinma Date: Wed, 17 Aug 2022 10:54:31 +0200 Subject: [PATCH 17/38] Improved RTP timestamp logging, fixed bug related to firstTime value --- lib/rtp.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/rtp.cpp b/lib/rtp.cpp index 3ed9b7c6..5a5ed829 100644 --- a/lib/rtp.cpp +++ b/lib/rtp.cpp @@ -877,7 +877,7 @@ namespace RTP{ if (!firstTime){ milliSync = Util::bootMS(); firstTime = pTime + 1; - INFO_MSG("RTP timestamp rollover expected in " PRETTY_PRINT_TIME, + INFO_MSG("RTP timestamp rollover for %" PRIu64 " (%s) expected in " PRETTY_PRINT_TIME, trackId, codec.c_str(), PRETTY_ARG_TIME((0xFFFFFFFFul - firstTime) / multiplier / 1000)); }else{ if (recentWrap){ @@ -886,14 +886,15 @@ namespace RTP{ }else{ if (prevTime > pTime && pTime < 0x40000000lu && prevTime > 0x80000000lu){ ++wrapArounds; + INFO_MSG("RTP timestamp rollover %" PRIu32 " for %" PRIu64 " (%s) happened; next should be in " PRETTY_PRINT_TIME, wrapArounds, trackId, codec.c_str(), PRETTY_ARG_TIME((0xFFFFFFFFul) / multiplier / 1000)); recentWrap = true; } } } // When there are B-frames, the firstTime can be higher than the current time // causing msTime to become negative and thus overflow - if (firstTime > pTime + 1){ - WARN_MSG("firstTime was higher than current packet time. Readjusting firsTime..."); + if (!wrapArounds && firstTime > pTime + 1){ + WARN_MSG("firstTime was higher than current packet time. Readjusting firstTime..."); firstTime = pTime + 1; } prevTime = pkt.getTimeStamp(); -- 2.25.1 From 2870ae1cedd202755bbe9d0e6808086d40656157 Mon Sep 17 00:00:00 2001 From: Thulinma Date: Wed, 24 Aug 2022 15:02:19 +0200 Subject: [PATCH 18/38] =?UTF-8?q?SRT=20RAW=20fixup=20=F0=9F=A4=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/input/input_tssrt.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/input/input_tssrt.cpp b/src/input/input_tssrt.cpp index da69be29..4219ebbc 100644 --- a/src/input/input_tssrt.cpp +++ b/src/input/input_tssrt.cpp @@ -211,7 +211,6 @@ namespace Mist{ } uint64_t packetTime = Util::bootMS(); thisPacket.genericFill(packetTime, 0, 1, rawBuffer, rawBuffer.size(), 0, 0); - bufferLivePacket(thisPacket); lastRawPacket = packetTime; rawBuffer.truncate(0); return; -- 2.25.1 From a9ddc377895f5feb898c162de0f6b0363cf94276 Mon Sep 17 00:00:00 2001 From: Thulinma Date: Thu, 25 Aug 2022 10:26:38 +0200 Subject: [PATCH 19/38] =?UTF-8?q?Why,=20Haivision,=20why=3F=20=F0=9F=98=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/socket_srt.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/socket_srt.cpp b/lib/socket_srt.cpp index 3e03bce4..6d1d5695 100644 --- a/lib/socket_srt.cpp +++ b/lib/socket_srt.cpp @@ -392,6 +392,10 @@ namespace Socket{ } } params["payloadsize"] = asString(chunkTransmitSize); + //This line forces the transmission type to live if unset. + //Live is actually the default, but not explicitly setting the option means + //that all other defaults do not get applied either, which is bad. + if (!params.count("transtype")){params["transtype"] = "live";} } int SRTConnection::preConfigureSocket(){ -- 2.25.1 From 8aceff951d3e320786bf33c9ff558a506df372c6 Mon Sep 17 00:00:00 2001 From: Marco van Dijk Date: Mon, 29 Aug 2022 14:36:00 +0200 Subject: [PATCH 20/38] Fix connections getting closed on interrupted system call --- lib/socket.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/socket.cpp b/lib/socket.cpp index 865c3be7..6d63af00 100644 --- a/lib/socket.cpp +++ b/lib/socket.cpp @@ -1032,6 +1032,7 @@ unsigned int Socket::Connection::iwrite(const void *buffer, int len){ case MBEDTLS_ERR_SSL_WANT_WRITE: return 0; break; case MBEDTLS_ERR_SSL_WANT_READ: return 0; break; case EWOULDBLOCK: return 0; break; + case EINTR: return 0; break; default: Error = true; lastErr = strerror(errno); @@ -1071,6 +1072,7 @@ unsigned int Socket::Connection::iwrite(const void *buffer, int len){ if (r < 0){ switch (errno){ case EWOULDBLOCK: return 0; break; + case EINTR: return 0; break; default: Error = true; lastErr = strerror(errno); -- 2.25.1 From 54a46146c2527004a96b78fe8b7341eab4de6f62 Mon Sep 17 00:00:00 2001 From: Thulinma Date: Thu, 11 Aug 2022 13:17:13 +0200 Subject: [PATCH 21/38] Fix TS inputs taking more and more memory over time in some cases --- lib/ts_stream.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/ts_stream.cpp b/lib/ts_stream.cpp index 4ab20c59..8da72493 100644 --- a/lib/ts_stream.cpp +++ b/lib/ts_stream.cpp @@ -190,7 +190,8 @@ namespace TS{ uint32_t tid = newPack.getPID(); bool unitStart = newPack.getUnitStart(); static uint32_t wantPrev = 0; - bool wantTrack = ((wantPrev == tid) || (tid == 0 || newPack.isPMT(pmtTracks) || pidToCodec.count(tid))); + bool isData = pidToCodec.count(tid); + bool wantTrack = ((wantPrev == tid) || (tid == 0 || newPack.isPMT(pmtTracks) || isData)); if (!wantTrack){return;} if (psCacheTid != tid || !psCache){ psCache = &(pesStreams[tid]); @@ -199,7 +200,7 @@ namespace TS{ if (unitStart || !psCache->empty()){ wantPrev = tid; psCache->push_back(newPack); - if (unitStart){ + if (unitStart && isData){ pesPositions[tid].push_back(bytePos); ++(seenUnitStart[tid]); } @@ -210,7 +211,7 @@ namespace TS{ if (tid == 0){return false;} { tthread::lock_guard guard(tMutex); - return !pmtTracks.count(tid); + return pidToCodec.count(tid); } } @@ -280,6 +281,11 @@ namespace TS{ } if (!pidToCodec.count(tid)){ + pesStreams.erase(tid); + pesPositions.erase(tid); + seenUnitStart.erase(tid); + psCacheTid = 0; + psCache = 0; return; // skip unknown codecs } -- 2.25.1 From c10d5a7ec1fdc71cc4af2329a9030b422735fd34 Mon Sep 17 00:00:00 2001 From: Marco van Dijk Date: Wed, 16 Mar 2022 13:45:37 +0100 Subject: [PATCH 22/38] onFail on a WS connection, send the error back using the websocket request handler --- src/output/output_http_internal.cpp | 20 ++++++++++++++++++-- src/output/output_http_internal.h | 1 + 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/output/output_http_internal.cpp b/src/output/output_http_internal.cpp index c8ca2e6c..97ca455f 100644 --- a/src/output/output_http_internal.cpp +++ b/src/output/output_http_internal.cpp @@ -36,6 +36,7 @@ namespace Mist{ OutHTTP::OutHTTP(Socket::Connection &conn) : HTTPOutput(conn){ stayConnected = false; + thisError = ""; // If this connection is a socket and not already connected to stdio, connect it to stdio. if (myConn.getPureSocket() != -1 && myConn.getSocket() != STDIN_FILENO && myConn.getSocket() != STDOUT_FILENO){ std::string host = getConnectedHost(); @@ -63,6 +64,11 @@ namespace Mist{ bool OutHTTP::listenMode(){return !(config->getString("ip").size());} void OutHTTP::onFail(const std::string &msg, bool critical){ + // If we are connected through WS, the websockethandler should return the error message + if (stayConnected){ + thisError = msg; + return; + } if (responded){ HTTPOutput::onFail(msg, critical); return; @@ -78,7 +84,6 @@ namespace Mist{ return; } if (H.url.size() >= 3 && H.url.substr(H.url.size() - 3) == ".js"){ - if (websocketHandler()){return;} JSON::Value json_resp; json_resp["error"] = "Could not retrieve stream. Sorry."; json_resp["error_guru"] = msg; @@ -1149,12 +1154,23 @@ namespace Mist{ if (meta){meta.reloadReplacedPagesIfNeeded();} if (newState != prevState || (newState == STRMSTAT_READY && M.getValidTracks() != prevTracks)){ if (newState == STRMSTAT_READY){ + thisError = ""; reconnect(); prevTracks = M.getValidTracks(); }else{ disconnect(); } - JSON::Value resp = getStatusJSON(reqHost, useragent); + JSON::Value resp; + // Check if we have an error message set + if (thisError == ""){ + resp = getStatusJSON(reqHost, useragent); + }else{ + resp["error"] = "Could not retrieve stream. Sorry."; + resp["error_guru"] = thisError; + if (config->getString("nostreamtext") != ""){ + resp["on_error"] = config->getString("nostreamtext"); + } + } if (currStreamName != streamName){ currStreamName = streamName; snprintf(pageName, NAME_BUFFER_SIZE, SHM_STREAM_STATE, streamName.c_str()); diff --git a/src/output/output_http_internal.h b/src/output/output_http_internal.h index 0f939e78..774eb186 100644 --- a/src/output/output_http_internal.h +++ b/src/output/output_http_internal.h @@ -21,6 +21,7 @@ namespace Mist{ private: std::string origStreamName; std::string mistPath; + std::string thisError; }; }// namespace Mist -- 2.25.1 From f418fed81c0c46fd338e238b558085f408456ed9 Mon Sep 17 00:00:00 2001 From: Thulinma Date: Thu, 24 Mar 2022 15:20:52 +0100 Subject: [PATCH 23/38] Fix HLS output not setting "responded" value appropriately --- src/output/output_hls.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/output/output_hls.cpp b/src/output/output_hls.cpp index 19d083e6..ad810aa7 100644 --- a/src/output/output_hls.cpp +++ b/src/output/output_hls.cpp @@ -232,6 +232,7 @@ namespace Mist{ H.setCORSHeaders(); if (H.method == "OPTIONS" || H.method == "HEAD"){ H.SendResponse("200", "OK", myConn); + responded = true; return; } H.SetBody(""); H.SendResponse("200", "OK", myConn); + responded = true; return; }// crossdomain.xml @@ -262,6 +264,7 @@ namespace Mist{ } H.SetBody(""); H.SendResponse("200", "OK", myConn); + responded = true; return; } @@ -311,6 +314,7 @@ namespace Mist{ targetTime = HLS::getPartTargetTime(M, idx, mTrack, startTime, msn, part); if (!targetTime){ H.SendResponse("404", "Partial fragment does not exist", myConn); + responded = true; return; } startTime += part * HLS::partDurationMaxMs; @@ -372,6 +376,7 @@ namespace Mist{ "served.\n"); myConn.SendNow(H.BuildResponse("404", "Fragment out of range")); WARN_MSG("Fragment @ %" PRIu64 " too old", startTime); + responded = true; return; } @@ -389,10 +394,12 @@ namespace Mist{ } if (H.method == "OPTIONS" || H.method == "HEAD"){ H.SendResponse("200", "OK", myConn); + responded = true; return; } H.StartResponse(H, myConn, VLCworkaround || config->getBool("nonchunked")); + responded = true; // we assume whole fragments - but timestamps may be altered at will contPAT = fragmentIndex; // PAT continuity counter contPMT = fragmentIndex; // PMT continuity counter @@ -419,6 +426,7 @@ namespace Mist{ // Strip /hls// from url std::string url = H.url.substr(H.url.find('/', 5) + 1); sendHlsManifest(url); + responded = true; } } -- 2.25.1 From f3c003481d572c140643e9bbfb292942d8e2ed38 Mon Sep 17 00:00:00 2001 From: Thulinma Date: Mon, 26 Sep 2022 12:44:30 +0200 Subject: [PATCH 24/38] Made FLV memory-based loader functions use const pointers --- lib/flv_tag.cpp | 6 +++--- lib/flv_tag.h | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/flv_tag.cpp b/lib/flv_tag.cpp index 9ccb6c41..780c6c5a 100644 --- a/lib/flv_tag.cpp +++ b/lib/flv_tag.cpp @@ -49,7 +49,7 @@ bool FLV::check_header(char *header){ /// Checks the first 3 bytes for the string "FLV". Implementing a basic FLV header check, /// returning true if it is, false if not. -bool FLV::is_header(char *header){ +bool FLV::is_header(const char *header){ if (header[0] != 'F') return false; if (header[1] != 'L') return false; if (header[2] != 'V') return false; @@ -624,7 +624,7 @@ bool FLV::Tag::ChunkLoader(const RTMPStream::Chunk &O){ /// \param S The size of the data buffer. /// \param P The current position in the data buffer. Will be updated to reflect new position. /// \return True if count bytes are read succesfully, false otherwise. -bool FLV::Tag::MemReadUntil(char *buffer, unsigned int count, unsigned int &sofar, char *D, +bool FLV::Tag::MemReadUntil(char *buffer, unsigned int count, unsigned int &sofar, const char *D, unsigned int S, unsigned int &P){ if (sofar >= count){return true;} int r = 0; @@ -646,7 +646,7 @@ bool FLV::Tag::MemReadUntil(char *buffer, unsigned int count, unsigned int &sofa /// location of the data buffer. \param S The size of the data buffer. \param P The current position /// in the data buffer. Will be updated to reflect new position. \return True if a whole tag is /// succesfully read, false otherwise. -bool FLV::Tag::MemLoader(char *D, unsigned int S, unsigned int &P){ +bool FLV::Tag::MemLoader(const char *D, unsigned int S, unsigned int &P){ if (len < 15){len = 15;} if (!checkBufferSize()){return false;} if (done){ diff --git a/lib/flv_tag.h b/lib/flv_tag.h index ca27f472..10f616b9 100644 --- a/lib/flv_tag.h +++ b/lib/flv_tag.h @@ -22,7 +22,7 @@ namespace FLV{ // functions bool check_header(char *header); ///< Checks a FLV Header for validness. - bool is_header(char *header); ///< Checks the first 3 bytes for the string "FLV". + bool is_header(const char *header); ///< Checks the first 3 bytes for the string "FLV". /// Helper function that can quickly skip through a file looking for a particular tag type bool seekToTagType(FILE *f, uint8_t type); @@ -55,7 +55,7 @@ namespace FLV{ bool DTSCMetaInit(const DTSC::Meta &M, std::set &selTracks); void toMeta(DTSC::Meta &meta, AMF::Object &amf_storage); void toMeta(DTSC::Meta &meta, AMF::Object &amf_storage, size_t &reTrack, const std::map &targetParams); - bool MemLoader(char *D, unsigned int S, unsigned int &P); + bool MemLoader(const char *D, unsigned int S, unsigned int &P); bool FileLoader(FILE *f); unsigned int getTrackID(); char *getData(); @@ -68,7 +68,7 @@ namespace FLV{ void setLen(); bool checkBufferSize(); // loader helper functions - bool MemReadUntil(char *buffer, unsigned int count, unsigned int &sofar, char *D, + bool MemReadUntil(char *buffer, unsigned int count, unsigned int &sofar, const char *D, unsigned int S, unsigned int &P); bool FileReadUntil(char *buffer, unsigned int count, unsigned int &sofar, FILE *f); }; -- 2.25.1 From 3e85da2afd1b3d36290763eebf84dd66ad08636c Mon Sep 17 00:00:00 2001 From: Thulinma Date: Mon, 19 Sep 2022 17:26:15 +0200 Subject: [PATCH 25/38] Match libav's RTMP receive window rollover behaviour --- lib/rtmpchunks.cpp | 5 +++++ src/output/output_rtmp.cpp | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/rtmpchunks.cpp b/lib/rtmpchunks.cpp index e7c151f0..5f9d33f0 100644 --- a/lib/rtmpchunks.cpp +++ b/lib/rtmpchunks.cpp @@ -475,6 +475,11 @@ bool RTMPStream::Chunk::Parse(Socket::Buffer &buffer){ } lastrecv[cs_id] = *this; RTMPStream::rec_cnt += i + real_len; + if (RTMPStream::rec_cnt >= 0xf0000000){ + INFO_MSG("Resetting receive window due to impending rollover"); + RTMPStream::rec_cnt -= 0xf0000000; + RTMPStream::rec_window_at = 0; + } if (len_left == 0){ return true; }else{ diff --git a/src/output/output_rtmp.cpp b/src/output/output_rtmp.cpp index 16a400d0..23194f4a 100644 --- a/src/output/output_rtmp.cpp +++ b/src/output/output_rtmp.cpp @@ -1599,7 +1599,7 @@ namespace Mist{ while (next.Parse(inputBuffer)){ // send ACK if we received a whole window - if ((RTMPStream::rec_cnt - RTMPStream::rec_window_at > RTMPStream::rec_window_size) || Util::bootSecs() > lastAck+15){ + if ((RTMPStream::rec_cnt - RTMPStream::rec_window_at > RTMPStream::rec_window_size / 4) || Util::bootSecs() > lastAck+15){ lastAck = Util::bootSecs(); RTMPStream::rec_window_at = RTMPStream::rec_cnt; myConn.SendNow(RTMPStream::SendCTL(3, RTMPStream::rec_cnt)); // send ack (msg 3) -- 2.25.1 From 074e7570284a094c12b00b88aa5cdf4a8f7fe5e4 Mon Sep 17 00:00:00 2001 From: Ramkoemar Date: Mon, 18 Oct 2021 14:29:13 +0200 Subject: [PATCH 26/38] Sessions rework --- CMakeLists.txt | 11 + lib/comms.cpp | 428 +++++--- lib/comms.h | 50 +- lib/defines.h | 7 +- src/controller/controller.cpp | 1 - src/controller/controller_api.cpp | 1 + src/controller/controller_statistics.cpp | 1185 ++++++++-------------- src/controller/controller_statistics.h | 61 +- src/controller/controller_storage.cpp | 22 +- src/input/input.cpp | 12 +- src/input/input.h | 2 +- src/input/input_rtsp.cpp | 5 +- src/input/input_sdp.cpp | 5 +- src/input/input_ts.cpp | 5 +- src/input/input_tssrt.cpp | 2 +- src/input/input_tssrt.h | 2 +- src/output/output.cpp | 125 +-- src/output/output.h | 9 +- src/output/output_cmaf.cpp | 2 +- src/output/output_cmaf.h | 3 +- src/output/output_http.cpp | 28 +- src/output/output_tssrt.cpp | 2 +- src/output/output_tssrt.h | 2 +- src/output/output_webrtc.cpp | 2 +- src/output/output_webrtc.h | 2 +- src/process/process_exec.cpp | 4 +- src/session.cpp | 367 +++++++ 27 files changed, 1192 insertions(+), 1153 deletions(-) create mode 100644 src/session.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e9bd644..6af16808 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -592,6 +592,17 @@ makeOutput(RTSP rtsp)#LTS makeOutput(WAV wav)#LTS makeOutput(SDP sdp http) +add_executable(MistSession + ${BINARY_DIR}/mist/.headers + src/session.cpp +) +install( + TARGETS MistSession + DESTINATION bin +) +target_link_libraries(MistSession mist) + + add_executable(MistProcFFMPEG ${BINARY_DIR}/mist/.headers src/process/process_ffmpeg.cpp diff --git a/lib/comms.cpp b/lib/comms.cpp index 108c686a..85f1e6d7 100644 --- a/lib/comms.cpp +++ b/lib/comms.cpp @@ -7,6 +7,7 @@ #include "timing.h" #include #include +#include "config.h" namespace Comms{ Comms::Comms(){ @@ -141,22 +142,197 @@ namespace Comms{ } } - Statistics::Statistics() : Comms(){sem.open(SEM_STATISTICS, O_CREAT | O_RDWR, ACCESSPERMS, 1);} + Sessions::Sessions() : Connections(){sem.open(SEM_STATISTICS, O_CREAT | O_RDWR, ACCESSPERMS, 1);} - void Statistics::unload(){ - if (index != INVALID_RECORD_INDEX){ - setStatus(COMM_STATUS_DISCONNECT | getStatus()); + void Sessions::reload(bool _master, bool reIssue){ + Comms::reload(COMMS_STATISTICS, COMMS_STATISTICS_INITSIZE, _master, reIssue); + } + + std::string Sessions::getSessId() const{return sessId.string(index);} + std::string Sessions::getSessId(size_t idx) const{return (master ? sessId.string(idx) : 0);} + void Sessions::setSessId(std::string _sid){sessId.set(_sid, index);} + void Sessions::setSessId(std::string _sid, size_t idx){ + if (!master){return;} + sessId.set(_sid, idx); + } + + bool Sessions::sessIdExists(std::string _sid){ + for (size_t i = 0; i < recordCount(); i++){ + if (getStatus(i) == COMM_STATUS_INVALID || (getStatus(i) & COMM_STATUS_DISCONNECT)){continue;} + if (getSessId(i) == _sid){ + if (Util::Procs::isRunning(getPid(i))){ + return true; + } + } } - index = INVALID_RECORD_INDEX; + return false; } - void Statistics::reload(bool _master, bool reIssue){ - Comms::reload(COMMS_STATISTICS, COMMS_STATISTICS_INITSIZE, _master, reIssue); + void Sessions::addFields(){ + Connections::addFields(); + dataAccX.addField("sessid", RAX_STRING, 80); } - void Statistics::addFields(){ + void Sessions::nullFields(){ + Connections::nullFields(); + setSessId(""); + } + + void Sessions::fieldAccess(){ + Connections::fieldAccess(); + sessId = dataAccX.getFieldAccX("sessid"); + } + + Users::Users() : Comms(){} + + Users::Users(const Users &rhs) : Comms(){ + if (rhs){ + reload(rhs.streamName, (size_t)rhs.getTrack()); + if (*this){ + setKeyNum(rhs.getKeyNum()); + setTrack(rhs.getTrack()); + } + } + } + + void Users::reload(const std::string &_streamName, bool _master, bool reIssue){ + streamName = _streamName; + + char semName[NAME_BUFFER_SIZE]; + snprintf(semName, NAME_BUFFER_SIZE, SEM_USERS, streamName.c_str()); + sem.open(semName, O_CREAT | O_RDWR, ACCESSPERMS, 1); + + char userPageName[NAME_BUFFER_SIZE]; + snprintf(userPageName, NAME_BUFFER_SIZE, COMMS_USERS, streamName.c_str()); + + Comms::reload(userPageName, COMMS_USERS_INITSIZE, _master, reIssue); + } + + void Users::addFields(){ + Comms::addFields(); + dataAccX.addField("track", RAX_64UINT); + dataAccX.addField("keynum", RAX_64UINT); + } + + void Users::nullFields(){ + Comms::nullFields(); + setTrack(0); + setKeyNum(0); + } + + void Users::fieldAccess(){ + Comms::fieldAccess(); + track = dataAccX.getFieldAccX("track"); + keyNum = dataAccX.getFieldAccX("keynum"); + } + + void Users::reload(const std::string &_streamName, size_t idx, uint8_t initialState){ + reload(_streamName); + if (dataPage){ + setTrack(idx); + setKeyNum(0); + setStatus(initialState); + } + } + + uint32_t Users::getTrack() const{return track.uint(index);} + uint32_t Users::getTrack(size_t idx) const{return (master ? track.uint(idx) : 0);} + void Users::setTrack(uint32_t _track){track.set(_track, index);} + void Users::setTrack(uint32_t _track, size_t idx){ + if (!master){return;} + track.set(_track, idx); + } + + size_t Users::getKeyNum() const{return keyNum.uint(index);} + size_t Users::getKeyNum(size_t idx) const{return (master ? keyNum.uint(idx) : 0);} + void Users::setKeyNum(size_t _keyNum){keyNum.set(_keyNum, index);} + void Users::setKeyNum(size_t _keyNum, size_t idx){ + if (!master){return;} + keyNum.set(_keyNum, idx); + } + + /// \brief Claims a spot on the connections page for the input/output which calls this function + /// Starts the MistSession binary for each session, which handles the statistics + /// and the USER_NEW and USER_END triggers + /// \param streamName: Name of the stream the input is providing or an output is making available to viewers + /// \param ip: IP address of the viewer which wants to access streamName. For inputs this value can be set to any value + /// \param sid: Session ID given by the player or randomly generated + /// \param protocol: Protocol currently in use for this connection + /// \param sessionMode: Determines how a viewer session is defined: + // If set to 0, all connections with the same viewer IP and stream name are bundled. + // If set to 1, all connections with the same viewer IP and player ID are bundled. + // If set to 2, all connections with the same player ID and stream name are bundled. + // If set to 3, all connections with the same viewer IP, player ID and stream name are bundled. + /// \param _master: If True, we are reading from this page. If False, we are writing (to our entry) on this page + /// \param reIssue: If True, claim a new entry on this page + void Connections::reload(std::string streamName, std::string ip, std::string sid, std::string protocol, std::string reqUrl, uint64_t sessionMode, bool _master, bool reIssue){ + if (sessionMode == 0xFFFFFFFFFFFFFFFFull){ + FAIL_MSG("The session mode was not initialised properly. Assuming default behaviour of bundling by viewer IP, stream name and player id"); + sessionMode = SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID; + } + // Generate a unique session ID for each viewer, input or output + sessionId = generateSession(streamName, ip, sid, protocol, sessionMode); + if (protocol.size() >= 6 && protocol.substr(0, 6) == "INPUT:"){ + sessionId = "I" + sessionId; + }else if (protocol.size() >= 7 && protocol.substr(0, 7) == "OUTPUT:"){ + sessionId = "O" + sessionId; + } + char userPageName[NAME_BUFFER_SIZE]; + snprintf(userPageName, NAME_BUFFER_SIZE, COMMS_SESSIONS, sessionId.c_str()); + // Check if the page exists, if not, spawn new session process + if (!_master){ + dataPage.init(userPageName, 0, false, false); + if (!dataPage){ + pid_t thisPid; + std::deque args; + args.push_back(Util::getMyPath() + "MistSession"); + args.push_back(sessionId); + args.push_back("--sessionmode"); + args.push_back(JSON::Value(sessionMode).asString()); + args.push_back("--streamname"); + args.push_back(streamName); + args.push_back("--ip"); + args.push_back(ip); + args.push_back("--sid"); + args.push_back(sid); + args.push_back("--protocol"); + args.push_back(protocol); + args.push_back("--requrl"); + args.push_back(reqUrl); + int err = fileno(stderr); + thisPid = Util::Procs::StartPiped(args, 0, 0, &err); + Util::Procs::forget(thisPid); + HIGH_MSG("Spawned new session executeable (pid %u) for sessionId '%s', corresponding to host %s and stream %s", thisPid, sessionId.c_str(), ip.c_str(), streamName.c_str()); + } + } + // Open SEM_SESSION + if(!sem){ + char semName[NAME_BUFFER_SIZE]; + snprintf(semName, NAME_BUFFER_SIZE, SEM_SESSION, sessionId.c_str()); + sem.open(semName, O_RDWR, ACCESSPERMS, 1); + } + Comms::reload(userPageName, COMMS_SESSIONS_INITSIZE, _master, reIssue); + VERYHIGH_MSG("Reloading connection. Claimed record %lu", index); + } + + /// \brief Marks the data page as closed, so that we longer write any new data to is + void Connections::setExit(){ + if (!master){return;} + dataAccX.setExit(); + } + + bool Connections::getExit(){ + return dataAccX.isExit(); + } + + void Connections::unload(){ + if (index != INVALID_RECORD_INDEX){ + setStatus(COMM_STATUS_DISCONNECT | getStatus()); + } + index = INVALID_RECORD_INDEX; + } + void Connections::addFields(){ Comms::addFields(); - dataAccX.addField("sync", RAX_UINT); dataAccX.addField("now", RAX_64UINT); dataAccX.addField("time", RAX_64UINT); dataAccX.addField("lastsecond", RAX_64UINT); @@ -165,15 +341,15 @@ namespace Comms{ dataAccX.addField("host", RAX_RAW, 16); dataAccX.addField("stream", RAX_STRING, 100); dataAccX.addField("connector", RAX_STRING, 20); - dataAccX.addField("crc", RAX_32UINT); + dataAccX.addField("tags", RAX_STRING, 512); dataAccX.addField("pktcount", RAX_64UINT); dataAccX.addField("pktloss", RAX_64UINT); dataAccX.addField("pktretrans", RAX_64UINT); } - void Statistics::nullFields(){ + void Connections::nullFields(){ Comms::nullFields(); - setCRC(0); + setTags(""); setConnector(""); setStream(""); setHost(""); @@ -182,15 +358,13 @@ namespace Comms{ setLastSecond(0); setTime(0); setNow(0); - setSync(0); setPacketCount(0); setPacketLostCount(0); setPacketRetransmitCount(0); } - void Statistics::fieldAccess(){ + void Connections::fieldAccess(){ Comms::fieldAccess(); - sync = dataAccX.getFieldAccX("sync"); now = dataAccX.getFieldAccX("now"); time = dataAccX.getFieldAccX("time"); lastSecond = dataAccX.getFieldAccX("lastsecond"); @@ -199,209 +373,159 @@ namespace Comms{ host = dataAccX.getFieldAccX("host"); stream = dataAccX.getFieldAccX("stream"); connector = dataAccX.getFieldAccX("connector"); - crc = dataAccX.getFieldAccX("crc"); + tags = dataAccX.getFieldAccX("tags"); pktcount = dataAccX.getFieldAccX("pktcount"); pktloss = dataAccX.getFieldAccX("pktloss"); pktretrans = dataAccX.getFieldAccX("pktretrans"); } - uint8_t Statistics::getSync() const{return sync.uint(index);} - uint8_t Statistics::getSync(size_t idx) const{return (master ? sync.uint(idx) : 0);} - void Statistics::setSync(uint8_t _sync){sync.set(_sync, index);} - void Statistics::setSync(uint8_t _sync, size_t idx){ - if (!master){return;} - sync.set(_sync, idx); - } - - uint64_t Statistics::getNow() const{return now.uint(index);} - uint64_t Statistics::getNow(size_t idx) const{return (master ? now.uint(idx) : 0);} - void Statistics::setNow(uint64_t _now){now.set(_now, index);} - void Statistics::setNow(uint64_t _now, size_t idx){ + uint64_t Connections::getNow() const{return now.uint(index);} + uint64_t Connections::getNow(size_t idx) const{return (master ? now.uint(idx) : 0);} + void Connections::setNow(uint64_t _now){now.set(_now, index);} + void Connections::setNow(uint64_t _now, size_t idx){ if (!master){return;} now.set(_now, idx); } - uint64_t Statistics::getTime() const{return time.uint(index);} - uint64_t Statistics::getTime(size_t idx) const{return (master ? time.uint(idx) : 0);} - void Statistics::setTime(uint64_t _time){time.set(_time, index);} - void Statistics::setTime(uint64_t _time, size_t idx){ + uint64_t Connections::getTime() const{return time.uint(index);} + uint64_t Connections::getTime(size_t idx) const{return (master ? time.uint(idx) : 0);} + void Connections::setTime(uint64_t _time){time.set(_time, index);} + void Connections::setTime(uint64_t _time, size_t idx){ if (!master){return;} time.set(_time, idx); } - uint64_t Statistics::getLastSecond() const{return lastSecond.uint(index);} - uint64_t Statistics::getLastSecond(size_t idx) const{ + uint64_t Connections::getLastSecond() const{return lastSecond.uint(index);} + uint64_t Connections::getLastSecond(size_t idx) const{ return (master ? lastSecond.uint(idx) : 0); } - void Statistics::setLastSecond(uint64_t _lastSecond){lastSecond.set(_lastSecond, index);} - void Statistics::setLastSecond(uint64_t _lastSecond, size_t idx){ + void Connections::setLastSecond(uint64_t _lastSecond){lastSecond.set(_lastSecond, index);} + void Connections::setLastSecond(uint64_t _lastSecond, size_t idx){ if (!master){return;} lastSecond.set(_lastSecond, idx); } - uint64_t Statistics::getDown() const{return down.uint(index);} - uint64_t Statistics::getDown(size_t idx) const{return (master ? down.uint(idx) : 0);} - void Statistics::setDown(uint64_t _down){down.set(_down, index);} - void Statistics::setDown(uint64_t _down, size_t idx){ + uint64_t Connections::getDown() const{return down.uint(index);} + uint64_t Connections::getDown(size_t idx) const{return (master ? down.uint(idx) : 0);} + void Connections::setDown(uint64_t _down){down.set(_down, index);} + void Connections::setDown(uint64_t _down, size_t idx){ if (!master){return;} down.set(_down, idx); } - uint64_t Statistics::getUp() const{return up.uint(index);} - uint64_t Statistics::getUp(size_t idx) const{return (master ? up.uint(idx) : 0);} - void Statistics::setUp(uint64_t _up){up.set(_up, index);} - void Statistics::setUp(uint64_t _up, size_t idx){ + uint64_t Connections::getUp() const{return up.uint(index);} + uint64_t Connections::getUp(size_t idx) const{return (master ? up.uint(idx) : 0);} + void Connections::setUp(uint64_t _up){up.set(_up, index);} + void Connections::setUp(uint64_t _up, size_t idx){ if (!master){return;} up.set(_up, idx); } - std::string Statistics::getHost() const{return std::string(host.ptr(index), 16);} - std::string Statistics::getHost(size_t idx) const{ + std::string Connections::getHost() const{return std::string(host.ptr(index), 16);} + std::string Connections::getHost(size_t idx) const{ if (!master){return std::string((size_t)16, (char)'\000');} return std::string(host.ptr(idx), 16); } - void Statistics::setHost(std::string _host){host.set(_host, index);} - void Statistics::setHost(std::string _host, size_t idx){ + void Connections::setHost(std::string _host){host.set(_host, index);} + void Connections::setHost(std::string _host, size_t idx){ if (!master){return;} host.set(_host, idx); } - std::string Statistics::getStream() const{return stream.string(index);} - std::string Statistics::getStream(size_t idx) const{return (master ? stream.string(idx) : "");} - void Statistics::setStream(std::string _stream){stream.set(_stream, index);} - void Statistics::setStream(std::string _stream, size_t idx){ + std::string Connections::getStream() const{return stream.string(index);} + std::string Connections::getStream(size_t idx) const{return (master ? stream.string(idx) : "");} + void Connections::setStream(std::string _stream){stream.set(_stream, index);} + void Connections::setStream(std::string _stream, size_t idx){ if (!master){return;} stream.set(_stream, idx); } - std::string Statistics::getConnector() const{return connector.string(index);} - std::string Statistics::getConnector(size_t idx) const{ + std::string Connections::getConnector() const{return connector.string(index);} + std::string Connections::getConnector(size_t idx) const{ return (master ? connector.string(idx) : ""); } - void Statistics::setConnector(std::string _connector){connector.set(_connector, index);} - void Statistics::setConnector(std::string _connector, size_t idx){ + void Connections::setConnector(std::string _connector){connector.set(_connector, index);} + void Connections::setConnector(std::string _connector, size_t idx){ if (!master){return;} connector.set(_connector, idx); } - uint32_t Statistics::getCRC() const{return crc.uint(index);} - uint32_t Statistics::getCRC(size_t idx) const{return (master ? crc.uint(idx) : 0);} - void Statistics::setCRC(uint32_t _crc){crc.set(_crc, index);} - void Statistics::setCRC(uint32_t _crc, size_t idx){ + bool Connections::hasConnector(size_t idx, std::string protocol){ + std::stringstream sstream(connector.string(idx)); + std::string _conn; + while (std::getline(sstream, _conn, ',')){ + if (_conn == protocol){ + return true; + } + } + return false; + } + + std::string Connections::getTags() const{return tags.string(index);} + std::string Connections::getTags(size_t idx) const{return (master ? tags.string(idx) : 0);} + void Connections::setTags(std::string _sid){tags.set(_sid, index);} + void Connections::setTags(std::string _sid, size_t idx){ if (!master){return;} - crc.set(_crc, idx); + tags.set(_sid, idx); } - uint64_t Statistics::getPacketCount() const{return pktcount.uint(index);} - uint64_t Statistics::getPacketCount(size_t idx) const{ + uint64_t Connections::getPacketCount() const{return pktcount.uint(index);} + uint64_t Connections::getPacketCount(size_t idx) const{ return (master ? pktcount.uint(idx) : 0); } - void Statistics::setPacketCount(uint64_t _count){pktcount.set(_count, index);} - void Statistics::setPacketCount(uint64_t _count, size_t idx){ + void Connections::setPacketCount(uint64_t _count){pktcount.set(_count, index);} + void Connections::setPacketCount(uint64_t _count, size_t idx){ if (!master){return;} pktcount.set(_count, idx); } - uint64_t Statistics::getPacketLostCount() const{return pktloss.uint(index);} - uint64_t Statistics::getPacketLostCount(size_t idx) const{ + uint64_t Connections::getPacketLostCount() const{return pktloss.uint(index);} + uint64_t Connections::getPacketLostCount(size_t idx) const{ return (master ? pktloss.uint(idx) : 0); } - void Statistics::setPacketLostCount(uint64_t _lost){pktloss.set(_lost, index);} - void Statistics::setPacketLostCount(uint64_t _lost, size_t idx){ + void Connections::setPacketLostCount(uint64_t _lost){pktloss.set(_lost, index);} + void Connections::setPacketLostCount(uint64_t _lost, size_t idx){ if (!master){return;} pktloss.set(_lost, idx); } - uint64_t Statistics::getPacketRetransmitCount() const{return pktretrans.uint(index);} - uint64_t Statistics::getPacketRetransmitCount(size_t idx) const{ + uint64_t Connections::getPacketRetransmitCount() const{return pktretrans.uint(index);} + uint64_t Connections::getPacketRetransmitCount(size_t idx) const{ return (master ? pktretrans.uint(idx) : 0); } - void Statistics::setPacketRetransmitCount(uint64_t _retrans){pktretrans.set(_retrans, index);} - void Statistics::setPacketRetransmitCount(uint64_t _retrans, size_t idx){ + void Connections::setPacketRetransmitCount(uint64_t _retrans){pktretrans.set(_retrans, index);} + void Connections::setPacketRetransmitCount(uint64_t _retrans, size_t idx){ if (!master){return;} pktretrans.set(_retrans, idx); } - std::string Statistics::getSessId() const{return getSessId(index);} - - std::string Statistics::getSessId(size_t idx) const{ - char res[140]; - memset(res, 0, 140); - std::string tmp = host.string(idx); - memcpy(res, tmp.c_str(), (tmp.size() > 16 ? 16 : tmp.size())); - tmp = stream.string(idx); - memcpy(res + 16, tmp.c_str(), (tmp.size() > 100 ? 100 : tmp.size())); - tmp = connector.string(idx); - memcpy(res + 116, tmp.c_str(), (tmp.size() > 20 ? 20 : tmp.size())); - Bit::htobl(res + 136, crc.uint(idx)); - return Secure::md5(res, 140); - } - - Users::Users() : Comms(){} - - Users::Users(const Users &rhs) : Comms(){ - if (rhs){ - reload(rhs.streamName, (size_t)rhs.getTrack()); - if (*this){ - setKeyNum(rhs.getKeyNum()); - setTrack(rhs.getTrack()); - } + /// \brief Generates a session ID which is unique per viewer + /// \return generated session ID as string + std::string Connections::generateSession(std::string streamName, std::string ip, std::string sid, std::string connector, uint64_t sessionMode){ + std::string concat; + // First bit defines whether to include stream name + if (sessionMode > 7){ + concat += streamName; + sessionMode -= 8; } - } - - void Users::reload(const std::string &_streamName, bool _master, bool reIssue){ - streamName = _streamName; - - char semName[NAME_BUFFER_SIZE]; - snprintf(semName, NAME_BUFFER_SIZE, SEM_USERS, streamName.c_str()); - sem.open(semName, O_CREAT | O_RDWR, ACCESSPERMS, 1); - - char userPageName[NAME_BUFFER_SIZE]; - snprintf(userPageName, NAME_BUFFER_SIZE, COMMS_USERS, streamName.c_str()); - - Comms::reload(userPageName, COMMS_USERS_INITSIZE, _master, reIssue); - } - - void Users::addFields(){ - Comms::addFields(); - dataAccX.addField("track", RAX_64UINT); - dataAccX.addField("keynum", RAX_64UINT); - } - - void Users::nullFields(){ - Comms::nullFields(); - setTrack(0); - setKeyNum(0); - } - - void Users::fieldAccess(){ - Comms::fieldAccess(); - track = dataAccX.getFieldAccX("track"); - keyNum = dataAccX.getFieldAccX("keynum"); - } - - void Users::reload(const std::string &_streamName, size_t idx, uint8_t initialState){ - reload(_streamName); - if (dataPage){ - setTrack(idx); - setKeyNum(0); - setStatus(initialState); + // Second bit defines whether to include viewer ip + if (sessionMode > 3){ + concat += ip; + sessionMode -= 4; } - } - - uint32_t Users::getTrack() const{return track.uint(index);} - uint32_t Users::getTrack(size_t idx) const{return (master ? track.uint(idx) : 0);} - void Users::setTrack(uint32_t _track){track.set(_track, index);} - void Users::setTrack(uint32_t _track, size_t idx){ - if (!master){return;} - track.set(_track, idx); - } - - size_t Users::getKeyNum() const{return keyNum.uint(index);} - size_t Users::getKeyNum(size_t idx) const{return (master ? keyNum.uint(idx) : 0);} - void Users::setKeyNum(size_t _keyNum){keyNum.set(_keyNum, index);} - void Users::setKeyNum(size_t _keyNum, size_t idx){ - if (!master){return;} - keyNum.set(_keyNum, idx); + // Third bit defines whether to include player ip + if (sessionMode > 1){ + concat += sid; + sessionMode -= 2; + } + // Fourth bit defines whether to include protocol + if (sessionMode == 1){ + concat += connector; + sessionMode = 0; + } + if (sessionMode > 0){ + WARN_MSG("Could not resolve session mode of value %lu", sessionMode); + } + return Secure::sha256(concat.c_str(), concat.length()); } }// namespace Comms diff --git a/lib/comms.h b/lib/comms.h index 9f459422..9a5c0ea9 100644 --- a/lib/comms.h +++ b/lib/comms.h @@ -64,21 +64,21 @@ namespace Comms{ Util::FieldAccX pid; }; - class Statistics : public Comms{ + class Connections : public Comms{ public: - Statistics(); - operator bool() const{return dataPage.mapped && (master || index != INVALID_RECORD_INDEX);} + void reload(std::string streamName, std::string ip, std::string sid, std::string protocol, std::string reqUrl, uint64_t sessionMode, bool _master = false, bool reIssue = false); void unload(); - void reload(bool _master = false, bool reIssue = false); + operator bool() const{return dataPage.mapped && (master || index != INVALID_RECORD_INDEX);} + std::string generateSession(std::string streamName, std::string ip, std::string sid, std::string connector, uint64_t sessionMode); + std::string sessionId; + + void setExit(); + bool getExit(); + virtual void addFields(); virtual void nullFields(); virtual void fieldAccess(); - uint8_t getSync() const; - uint8_t getSync(size_t idx) const; - void setSync(uint8_t _sync); - void setSync(uint8_t _sync, size_t idx); - uint64_t getNow() const; uint64_t getNow(size_t idx) const; void setNow(uint64_t _now); @@ -118,11 +118,12 @@ namespace Comms{ std::string getConnector(size_t idx) const; void setConnector(std::string _connector); void setConnector(std::string _connector, size_t idx); + bool hasConnector(size_t idx, std::string protocol); - uint32_t getCRC() const; - uint32_t getCRC(size_t idx) const; - void setCRC(uint32_t _crc); - void setCRC(uint32_t _crc, size_t idx); + std::string getTags() const; + std::string getTags(size_t idx) const; + void setTags(std::string _sid); + void setTags(std::string _sid, size_t idx); uint64_t getPacketCount() const; uint64_t getPacketCount(size_t idx) const; @@ -139,11 +140,7 @@ namespace Comms{ void setPacketRetransmitCount(uint64_t _retransmit); void setPacketRetransmitCount(uint64_t _retransmit, size_t idx); - std::string getSessId() const; - std::string getSessId(size_t index) const; - - private: - Util::FieldAccX sync; + protected: Util::FieldAccX now; Util::FieldAccX time; Util::FieldAccX lastSecond; @@ -152,7 +149,8 @@ namespace Comms{ Util::FieldAccX host; Util::FieldAccX stream; Util::FieldAccX connector; - Util::FieldAccX crc; + Util::FieldAccX sessId; + Util::FieldAccX tags; Util::FieldAccX pktcount; Util::FieldAccX pktloss; Util::FieldAccX pktretrans; @@ -186,4 +184,18 @@ namespace Comms{ Util::FieldAccX track; Util::FieldAccX keyNum; }; + + class Sessions : public Connections{ + public: + Sessions(); + void reload(bool _master = false, bool reIssue = false); + std::string getSessId() const; + std::string getSessId(size_t idx) const; + void setSessId(std::string _sid); + void setSessId(std::string _sid, size_t idx); + bool sessIdExists(std::string _sid); + virtual void addFields(); + virtual void nullFields(); + virtual void fieldAccess(); + }; }// namespace Comms diff --git a/lib/defines.h b/lib/defines.h index 10c4b9b1..203b75db 100644 --- a/lib/defines.h +++ b/lib/defines.h @@ -196,11 +196,14 @@ static inline void show_stackframe(){} #define TRACK_PAGE_RECORDSIZE 36 #define COMMS_STATISTICS "MstStat" -#define COMMS_STATISTICS_INITSIZE 8 * 1024 * 1024 +#define COMMS_STATISTICS_INITSIZE 16 * 1024 * 1024 #define COMMS_USERS "MstUser%s" //%s stream name #define COMMS_USERS_INITSIZE 512 * 1024 +#define COMMS_SESSIONS "MstSession%s" +#define COMMS_SESSIONS_INITSIZE 8 * 1024 * 1024 + #define SEM_STATISTICS "/MstStat" #define SEM_USERS "/MstUser%s" //%s stream name @@ -226,7 +229,9 @@ static inline void show_stackframe(){} #define SEM_LIVE "/MstLIVE%s" //%s stream name #define SEM_INPUT "/MstInpt%s" //%s stream name #define SEM_TRACKLIST "/MstTRKS%s" //%s stream name +#define SEM_SESSION "MstSess%s" #define SEM_SESSCACHE "/MstSessCacheLock" +#define SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID 14 #define SHM_CAPA "MstCapa" #define SHM_PROTO "MstProt" #define SHM_PROXY "MstProx" diff --git a/src/controller/controller.cpp b/src/controller/controller.cpp index a0ff2ee8..9e17ca6b 100644 --- a/src/controller/controller.cpp +++ b/src/controller/controller.cpp @@ -306,7 +306,6 @@ int main_loop(int argc, char **argv){ if (Controller::Storage["config"].isMember("accesslog")){ Controller::conf.getOption("accesslog", true)[0u] = Controller::Storage["config"]["accesslog"]; } - Controller::maxConnsPerIP = Controller::conf.getInteger("maxconnsperip"); Controller::Storage["config"]["prometheus"] = Controller::conf.getString("prometheus"); Controller::Storage["config"]["accesslog"] = Controller::conf.getString("accesslog"); Controller::normalizeTrustedProxies(Controller::Storage["config"]["trustedproxy"]); diff --git a/src/controller/controller_api.cpp b/src/controller/controller_api.cpp index 4d94bb11..02491065 100644 --- a/src/controller/controller_api.cpp +++ b/src/controller/controller_api.cpp @@ -594,6 +594,7 @@ void Controller::handleAPICommands(JSON::Value &Request, JSON::Value &Response){ out["prometheus"] = in["prometheus"]; Controller::prometheus = out["prometheus"].asStringRef(); } + if (in.isMember("sessionMode")){out["sessionMode"] = in["sessionMode"];} if (in.isMember("defaultStream")){out["defaultStream"] = in["defaultStream"];} if (in.isMember("location") && in["location"].isObject()){ out["location"]["lat"] = in["location"]["lat"].asDouble(); diff --git a/src/controller/controller_statistics.cpp b/src/controller/controller_statistics.cpp index c5b7f643..6e51f4ee 100644 --- a/src/controller/controller_statistics.cpp +++ b/src/controller/controller_statistics.cpp @@ -15,6 +15,7 @@ #include #include //for fstatvfs #include +#include #ifndef KILL_ON_EXIT #define KILL_ON_EXIT false @@ -30,7 +31,6 @@ #define STAT_CLI_UP 64 #define STAT_CLI_BPS_DOWN 128 #define STAT_CLI_BPS_UP 256 -#define STAT_CLI_CRC 512 #define STAT_CLI_SESSID 1024 #define STAT_CLI_PKTCOUNT 2048 #define STAT_CLI_PKTLOST 4096 @@ -46,59 +46,18 @@ #define STAT_TOT_PERCRETRANS 64 #define STAT_TOT_ALL 0xFF -#define COUNTABLE_BYTES 128 * 1024 - -std::map Controller::sessions; ///< list of sessions that have statistics data available -std::map Controller::connToSession; ///< Map of socket IDs to session info. +// Mapping of sessId -> session statistics +std::map sessions; std::map Controller::triggerStats; ///< Holds prometheus stats for trigger executions bool Controller::killOnExit = KILL_ON_EXIT; tthread::mutex Controller::statsMutex; -unsigned int Controller::maxConnsPerIP = 0; uint64_t Controller::statDropoff = 0; static uint64_t cpu_use = 0; char noBWCountMatches[1717]; uint64_t bwLimit = 128 * 1024 * 1024; // gigabit default limit -/// Session cache shared memory page -IPC::sharedPage *shmSessions = 0; -/// Lock for the session cache shared memory page -IPC::semaphore *cacheLock = 0; - -/// Convert bandwidth config into memory format -void Controller::updateBandwidthConfig(){ - size_t offset = 0; - bwLimit = 128 * 1024 * 1024; // gigabit default limit - memset(noBWCountMatches, 0, 1717); - if (Storage.isMember("bandwidth")){ - if (Storage["bandwidth"].isMember("limit")){bwLimit = Storage["bandwidth"]["limit"].asInt();} - if (Storage["bandwidth"].isMember("exceptions")){ - jsonForEach(Storage["bandwidth"]["exceptions"], j){ - std::string newbins = Socket::getBinForms(j->asStringRef()); - if (offset + newbins.size() < 1700){ - memcpy(noBWCountMatches + offset, newbins.data(), newbins.size()); - offset += newbins.size(); - } - } - } - } - //Localhost is always excepted from counts - { - std::string newbins = Socket::getBinForms("::1"); - if (offset + newbins.size() < 1700){ - memcpy(noBWCountMatches + offset, newbins.data(), newbins.size()); - offset += newbins.size(); - } - } - { - std::string newbins = Socket::getBinForms("127.0.0.1/8"); - if (offset + newbins.size() < 1700){ - memcpy(noBWCountMatches + offset, newbins.data(), newbins.size()); - offset += newbins.size(); - } - } -} // For server-wide totals. Local to this file only. struct streamTotals{ @@ -116,11 +75,30 @@ struct streamTotals{ uint64_t packLoss; uint64_t packRetrans; }; + +Comms::Sessions statComm; +bool statCommActive = false; +// Global server wide statistics +static uint64_t servUpBytes = 0; +static uint64_t servDownBytes = 0; +static uint64_t servUpOtherBytes = 0; +static uint64_t servDownOtherBytes = 0; +static uint64_t servInputs = 0; +static uint64_t servOutputs = 0; +static uint64_t servViewers = 0; +static uint64_t servSeconds = 0; +static uint64_t servPackSent = 0; +static uint64_t servPackLoss = 0; +static uint64_t servPackRetrans = 0; +// Total time watched for all sessions which are no longer active +static uint64_t viewSecondsTotal = 0; +// Mapping of streamName -> summary of stream-wide statistics static std::map streamStats; -static void createEmptyStatsIfNeeded(const std::string & strm){ - if (streamStats.count(strm)){return;} - streamTotals & sT = streamStats[strm]; +// If sessId does not exist yet in streamStats, create and init an entry for it +static void createEmptyStatsIfNeeded(const std::string & sessId){ + if (streamStats.count(sessId)){return;} + streamTotals & sT = streamStats[sessId]; sT.upBytes = 0; sT.downBytes = 0; sT.inputs = 0; @@ -136,68 +114,38 @@ static void createEmptyStatsIfNeeded(const std::string & strm){ sT.packRetrans = 0; } - -static uint64_t servUpBytes = 0; -static uint64_t servDownBytes = 0; -static uint64_t servUpOtherBytes = 0; -static uint64_t servDownOtherBytes = 0; -static uint64_t servInputs = 0; -static uint64_t servOutputs = 0; -static uint64_t servViewers = 0; -static uint64_t servSeconds = 0; -static uint64_t servPackSent = 0; -static uint64_t servPackLoss = 0; -static uint64_t servPackRetrans = 0; - -Controller::sessIndex::sessIndex(){ - crc = 0; -} - -/// Initializes a sessIndex from a statistics object + index, converting binary format IP addresses -/// into strings. This extracts the host, stream name, connector and crc field, ignoring everything -/// else. -Controller::sessIndex::sessIndex(const Comms::Statistics &statComm, size_t id){ - Socket::hostBytesToStr(statComm.getHost(id).data(), 16, host); - streamName = statComm.getStream(id); - connector = statComm.getConnector(id); - crc = statComm.getCRC(id); - ID = statComm.getSessId(id); -} - -std::string Controller::sessIndex::toStr(){ - std::stringstream s; - s << ID << "(" << host << " " << crc << " " << streamName << " " << connector << ")"; - return s.str(); -} - -bool Controller::sessIndex::operator==(const Controller::sessIndex &b) const{ - return (host == b.host && crc == b.crc && streamName == b.streamName && connector == b.connector); -} - -bool Controller::sessIndex::operator!=(const Controller::sessIndex &b) const{ - return !(*this == b); -} - -bool Controller::sessIndex::operator>(const Controller::sessIndex &b) const{ - return host > b.host || - (host == b.host && - (crc > b.crc || (crc == b.crc && (streamName > b.streamName || - (streamName == b.streamName && connector > b.connector))))); -} - -bool Controller::sessIndex::operator<(const Controller::sessIndex &b) const{ - return host < b.host || - (host == b.host && - (crc < b.crc || (crc == b.crc && (streamName < b.streamName || - (streamName == b.streamName && connector < b.connector))))); -} - -bool Controller::sessIndex::operator<=(const Controller::sessIndex &b) const{ - return !(*this > b); -} - -bool Controller::sessIndex::operator>=(const Controller::sessIndex &b) const{ - return !(*this < b); +/// Convert bandwidth config into memory format +void Controller::updateBandwidthConfig(){ + size_t offset = 0; + bwLimit = 128 * 1024 * 1024; // gigabit default limit + memset(noBWCountMatches, 0, 1717); + if (Storage.isMember("bandwidth")){ + if (Storage["bandwidth"].isMember("limit")){bwLimit = Storage["bandwidth"]["limit"].asInt();} + if (Storage["bandwidth"].isMember("exceptions")){ + jsonForEach(Storage["bandwidth"]["exceptions"], j){ + std::string newbins = Socket::getBinForms(j->asStringRef()); + if (offset + newbins.size() < 1700){ + memcpy(noBWCountMatches + offset, newbins.data(), newbins.size()); + offset += newbins.size(); + } + } + } + } + //Localhost is always excepted from counts + { + std::string newbins = Socket::getBinForms("::1"); + if (offset + newbins.size() < 1700){ + memcpy(noBWCountMatches + offset, newbins.data(), newbins.size()); + offset += newbins.size(); + } + } + { + std::string newbins = Socket::getBinForms("127.0.0.1/8"); + if (offset + newbins.size() < 1700){ + memcpy(noBWCountMatches + offset, newbins.data(), newbins.size()); + offset += newbins.size(); + } + } } /// This function is ran whenever a stream becomes active. @@ -211,9 +159,6 @@ void Controller::streamStopped(std::string stream){ INFO_MSG("Stream %s became inactive", stream.c_str()); } -Comms::Statistics statComm; -bool statCommActive = false; - /// Invalidates all current sessions for the given streamname /// Updates the session cache, afterwards. void Controller::sessions_invalidate(const std::string &streamname){ @@ -221,18 +166,17 @@ void Controller::sessions_invalidate(const std::string &streamname){ FAIL_MSG("In shutdown procedure - cannot invalidate sessions."); return; } - unsigned int invalidated = 0; unsigned int sessCount = 0; - tthread::lock_guard guard(statsMutex); - for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ - if (it->first.streamName == streamname){ + // Find all matching streams in statComm + for (size_t i = 0; i < statComm.recordCount(); i++){ + if (statComm.getStatus(i) == COMM_STATUS_INVALID || (statComm.getStatus(i) & COMM_STATUS_DISCONNECT)){continue;} + if (statComm.getStream(i) == streamname){ sessCount++; - invalidated += it->second.invalidate(); + // Re-trigger USER_NEW trigger for this session + kill(statComm.getPid(i), SIGUSR1); } } - Controller::writeSessionCache(); - INFO_MSG("Invalidated %u connections in %u sessions for stream %s", invalidated, sessCount, - streamname.c_str()); + INFO_MSG("Invalidated %u session(s) for stream %s", sessCount, streamname.c_str()); } /// Shuts down all current sessions for the given streamname @@ -256,18 +200,8 @@ void Controller::sessId_shutdown(const std::string &sessId){ FAIL_MSG("In controller shutdown procedure - cannot shutdown sessions."); return; } - unsigned int murdered = 0; - unsigned int sessCount = 0; - tthread::lock_guard guard(statsMutex); - for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ - if (it->first.ID == sessId){ - sessCount++; - murdered += it->second.kill(); - break; - } - } - Controller::writeSessionCache(); - INFO_MSG("Shut down %u connections in %u session(s) for ID %s", murdered, sessCount, sessId.c_str()); + killConnections(sessId); + INFO_MSG("Shut down session with session ID %s", sessId.c_str()); } /// Tags the given session @@ -277,8 +211,8 @@ void Controller::sessId_tag(const std::string &sessId, const std::string &tag){ return; } tthread::lock_guard guard(statsMutex); - for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ - if (it->first.ID == sessId){ + for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ + if (it->first == sessId){ it->second.tags.insert(tag); return; } @@ -295,17 +229,15 @@ void Controller::tag_shutdown(const std::string &tag){ FAIL_MSG("In controller shutdown procedure - cannot shutdown sessions."); return; } - unsigned int murdered = 0; unsigned int sessCount = 0; tthread::lock_guard guard(statsMutex); - for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ + for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ if (it->second.tags.count(tag)){ sessCount++; - murdered += it->second.kill(); + killConnections(it->first); } } - Controller::writeSessionCache(); - INFO_MSG("Shut down %u connections in %u session(s) for tag %s", murdered, sessCount, tag.c_str()); + INFO_MSG("Shut down %u session(s) for tag %s", sessCount, tag.c_str()); } /// Shuts down all current sessions for the given streamname @@ -315,48 +247,23 @@ void Controller::sessions_shutdown(const std::string &streamname, const std::str FAIL_MSG("In controller shutdown procedure - cannot shutdown sessions."); return; } - unsigned int murdered = 0; unsigned int sessCount = 0; tthread::lock_guard guard(statsMutex); - for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ - if ((!streamname.size() || it->first.streamName == streamname) && - (!protocol.size() || it->first.connector == protocol)){ + // Find all matching streams in statComm and get their sessId + for (size_t i = 0; i < statComm.recordCount(); i++){ + if (statComm.getStatus(i) == COMM_STATUS_INVALID || (statComm.getStatus(i) & COMM_STATUS_DISCONNECT)){continue;} + if ((!streamname.size() || statComm.getStream(i) == streamname) && + (!protocol.size() || statComm.hasConnector(i, protocol))){ + uint32_t pid = statComm.getPid(i); sessCount++; - murdered += it->second.kill(); - } - } - Controller::writeSessionCache(); - INFO_MSG("Shut down %u connections in %u sessions for stream %s/%s", murdered, sessCount, - streamname.c_str(), protocol.c_str()); -} - -/// Writes the session cache to shared memory. -/// Assumes the config mutex, stats mutex and session cache semaphore are already locked. -/// Does nothing if the session cache could not be initialized on the first try -/// Does no error checking after first open attempt (fails silently)! -void Controller::writeSessionCache(){ - uint32_t shmOffset = 0; - if (shmSessions && shmSessions->mapped){ - if (cacheLock){cacheLock->wait(16);} - if (sessions.size()){ - for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ - if (it->second.hasData()){ - // store an entry in the shmSessions page, if it fits - if (it->second.sync > 2 && shmOffset + SHM_SESSIONS_ITEM < SHM_SESSIONS_SIZE){ - *((uint32_t *)(shmSessions->mapped + shmOffset)) = it->first.crc; - strncpy(shmSessions->mapped + shmOffset + 4, it->first.streamName.c_str(), 100); - strncpy(shmSessions->mapped + shmOffset + 104, it->first.connector.c_str(), 20); - strncpy(shmSessions->mapped + shmOffset + 124, it->first.host.c_str(), 40); - shmSessions->mapped[shmOffset + 164] = it->second.sync; - shmOffset += SHM_SESSIONS_ITEM; - } - } + if (pid > 1){ + Util::Procs::Stop(pid); + INFO_MSG("Killing PID %" PRIu32, pid); } } - // set a final shmSessions entry to all zeroes - memset(shmSessions->mapped + shmOffset, 0, SHM_SESSIONS_ITEM); - if (cacheLock){cacheLock->post(16);} } + INFO_MSG("Shut down %u sessions for stream %s/%s", sessCount, + streamname.c_str(), protocol.c_str()); } /// This function runs as a thread and roughly once per second retrieves @@ -366,14 +273,6 @@ void Controller::SharedMemStats(void *config){ HIGH_MSG("Starting stats thread"); statComm.reload(true); statCommActive = true; - shmSessions = new IPC::sharedPage(SHM_SESSIONS, SHM_SESSIONS_SIZE, false, false); - if (!shmSessions || !shmSessions->mapped){ - if (shmSessions){delete shmSessions;} - shmSessions = new IPC::sharedPage(SHM_SESSIONS, SHM_SESSIONS_SIZE, true); - } - cacheLock = new IPC::semaphore(SEM_SESSCACHE, O_CREAT | O_RDWR, ACCESSPERMS, 16); - cacheLock->unlink(); - cacheLock->open(SEM_SESSCACHE, O_CREAT | O_RDWR, ACCESSPERMS, 16); std::set inactiveStreams; Controller::initState(); bool shiftWrites = true; @@ -401,7 +300,6 @@ void Controller::SharedMemStats(void *config){ } } { - tthread::lock_guard guard(Controller::configMutex); tthread::lock_guard guard2(statsMutex); // parse current users @@ -429,29 +327,47 @@ void Controller::SharedMemStats(void *config){ it->second.packRetrans = 0; } } - // wipe old statistics + unsigned int tOut = Util::bootSecs() - STATS_DELAY; + unsigned int tIn = Util::bootSecs() - STATS_INPUT_DELAY; + if (streamStats.size()){ + for (std::map::iterator it = streamStats.begin(); + it != streamStats.end(); ++it){ + it->second.currViews = 0; + it->second.currIns = 0; + it->second.currOuts = 0; + } + } + // wipe old statistics and set session type counters if (sessions.size()){ - std::list mustWipe; + std::list mustWipe; uint64_t cutOffPoint = Util::bootSecs() - STAT_CUTOFF; - uint64_t disconnectPointIn = Util::bootSecs() - STATS_INPUT_DELAY; - uint64_t disconnectPointOut = Util::bootSecs() - STATS_DELAY; - for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ - uint64_t dPoint = it->second.getSessType() == SESS_INPUT ? disconnectPointIn : disconnectPointOut; - if (it->second.sync == 100){ - // Denied entries are connection-entry-wiped as soon as they become boring - it->second.wipeOld(dPoint); - }else{ - // Normal entries are summarized after STAT_CUTOFF seconds - it->second.wipeOld(cutOffPoint); - } + for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ // This part handles ending sessions, keeping them in cache for now - if (it->second.isTracked() && !it->second.isConnected() && it->second.getEnd() < dPoint){ - it->second.dropSession(it->first); - } - // This part handles wiping from the session cache - if (!it->second.hasData()){ - it->second.dropSession(it->first); // End the session, just in case it wasn't yet + if (it->second.getEnd() < cutOffPoint && it->second.newestDataPoint() < cutOffPoint){ + viewSecondsTotal += it->second.getConnTime(); mustWipe.push_back(it->first); + // Don't count this session as a viewer + continue; + } + // Recount input, output and viewer type sessions + switch (it->second.getSessType()){ + case SESS_UNSET: break; + case SESS_VIEWER: + if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){ + streamStats[it->first].currViews++; + } + servSeconds += it->second.getConnTime(); + break; + case SESS_INPUT: + if (it->second.hasDataFor(tIn) && it->second.isViewerOn(tIn)){ + streamStats[it->first].currIns++; + } + break; + case SESS_OUTPUT: + if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){ + streamStats[it->first].currOuts++; + } + break; } } while (mustWipe.size()){ @@ -513,7 +429,6 @@ void Controller::SharedMemStats(void *config){ shiftWrites = true; } /*LTS-START*/ - Controller::writeSessionCache(); Controller::checkServerLimits(); /*LTS-END*/ } @@ -523,7 +438,6 @@ void Controller::SharedMemStats(void *config){ HIGH_MSG("Stopping stats thread"); if (Util::Config::is_restarting){ statComm.setMaster(false); - shmSessions->master = false; }else{/*LTS-START*/ if (Controller::killOnExit){ WARN_MSG("Killing all connected clients to force full shutdown"); @@ -532,10 +446,6 @@ void Controller::SharedMemStats(void *config){ /*LTS-END*/ } Controller::deinitState(Util::Config::is_restarting); - delete shmSessions; - shmSessions = 0; - delete cacheLock; - cacheLock = 0; } /// Gets a complete list of all streams currently in active state, with optional prefix matching @@ -559,97 +469,53 @@ std::set Controller::getActiveStreams(const std::string &prefix){ return ret; } -/// Forces a re-sync of the session -/// Assumes the session cache will be updated separately - may not work correctly if this is forgotten! -uint32_t Controller::statSession::invalidate(){ - uint32_t ret = 0; - sync = 1; - if (curConns.size() && statCommActive){ - for (std::map::iterator jt = curConns.begin(); jt != curConns.end(); ++jt){ - if (statComm.getStatus(jt->first) != COMM_STATUS_INVALID){ - statComm.setSync(2, jt->first); - ret++; - } - } - } - return ret; -} - -/// Kills all active connections, sets the session state to denied (sync=100). +/// Kills all connection of a given session /// Assumes the session cache will be updated separately - may not work correctly if this is forgotten! -uint32_t Controller::statSession::kill(){ - uint32_t ret = 0; - sync = 100; - if (curConns.size() && statCommActive){ - for (std::map::iterator jt = curConns.begin(); jt != curConns.end(); ++jt){ - if (statComm.getStatus(jt->first) != COMM_STATUS_INVALID){ - statComm.setSync(100, jt->first); - uint32_t pid = statComm.getPid(jt->first); +void Controller::killConnections(std::string sessId){ + if (statCommActive){ + // Find a matching stream in statComm with a matching sessID and kill it + for (size_t i = 0; i < statComm.recordCount(); i++){ + if (statComm.getStatus(i) == COMM_STATUS_INVALID || (statComm.getStatus(i) & COMM_STATUS_DISCONNECT)){continue;} + if (statComm.getSessId(i) == sessId){ + uint32_t pid = statComm.getPid(i); if (pid > 1){ Util::Procs::Stop(pid); INFO_MSG("Killing PID %" PRIu32, pid); } - ret++; } } } - return ret; } /// Updates the given active connection with new stats data. -void Controller::statSession::update(uint64_t index, Comms::Statistics &statComm){ - std::string myHost; - Socket::hostBytesToStr(statComm.getHost(index).data(), 16, myHost); - std::string myStream = statComm.getStream(index); - std::string myConnector = statComm.getConnector(index); - // update the sync byte: 0 = requesting fill, 2 = requesting refill, 1 = needs checking, > 2 = - // state known (100=denied, 10=accepted) - if (!statComm.getSync(index)){ - sessIndex tmpidx(statComm, index); - // if we have a maximum connection count per IP, enforce it - if (maxConnsPerIP && !statComm.getSync(index)){ - unsigned int currConns = 1; - long long shortly = Util::bootSecs(); - for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ - - if (&it->second != this && it->first.host == myHost && - (it->second.hasDataFor(shortly - STATS_DELAY) || it->second.hasDataFor(shortly) || - it->second.hasDataFor(shortly - 1) || it->second.hasDataFor(shortly - 2) || - it->second.hasDataFor(shortly - 3) || it->second.hasDataFor(shortly - 4) || - it->second.hasDataFor(shortly - 5)) && - ++currConns > maxConnsPerIP){ - break; - } - } - if (currConns > maxConnsPerIP){ - WARN_MSG("Disconnecting session from %s: exceeds max connection count of %u", myHost.c_str(), maxConnsPerIP); - statComm.setSync(100, index); - } - } - if (statComm.getSync(index) != 100){ - // only set the sync if this is the first connection in the list - // we also catch the case that there are no connections, which is an error-state - if (!sessions[tmpidx].curConns.size() || sessions[tmpidx].curConns.begin()->first == index){ - MEDIUM_MSG("Requesting sync to %u for %s, %s, %s, %" PRIu32, sync, myStream.c_str(), - myConnector.c_str(), myHost.c_str(), statComm.getCRC(index) & 0xFFFFFFFFu); - statComm.setSync(sync, index); - } - // and, always set the sync if it is > 2 - if (sync > 2){ - MEDIUM_MSG("Setting sync to %u for %s, %s, %s, %" PRIu32, sync, myStream.c_str(), - myConnector.c_str(), myHost.c_str(), statComm.getCRC(index) & 0xFFFFFFFFu); - statComm.setSync(sync, index); - } +void Controller::statSession::update(uint64_t index, Comms::Sessions &statComm){ + if (host == ""){ + Socket::hostBytesToStr(statComm.getHost(index).data(), 16, host); + } + if (streamName == ""){ + streamName = statComm.getStream(index); + } + if (curConnector == ""){ + curConnector = statComm.getConnector(index); + } + if (sessId == ""){ + sessId = statComm.getSessId(index); + } + // Export tags to session + if (tags.size()){ + std::stringstream tagStream; + for (std::set::iterator it = tags.begin(); it != tags.end(); ++it){ + tagStream << "[" << *it << "]"; } - }else{ - if (sync < 2 && statComm.getSync(index) > 2){sync = statComm.getSync(index);} + statComm.setTags(tagStream.str(), index); } + long long prevDown = getDown(); long long prevUp = getUp(); uint64_t prevPktSent = getPktCount(); uint64_t prevPktLost = getPktLost(); uint64_t prevPktRetrans = getPktRetransmit(); - curConns[index].update(statComm, index); + curData.update(statComm, index); // store timestamp of first received data, if older if (firstSec > statComm.getNow(index)){firstSec = statComm.getNow(index);} uint64_t secIncr = 0; @@ -671,7 +537,7 @@ void Controller::statSession::update(uint64_t index, Comms::Statistics &statComm uint64_t currPktRetrans = getPktRetransmit(); if (currUp - prevUp < 0 || currDown - prevDown < 0){ INFO_MSG("Negative data usage! %lldu/%lldd (u%lld->%lld) in %s over %s, #%" PRIu64, currUp - prevUp, - currDown - prevDown, prevUp, currUp, myStream.c_str(), myConnector.c_str(), index); + currDown - prevDown, prevUp, currUp, streamName.c_str(), curConnector.c_str(), index); }else{ if (!noBWCount){ size_t bwMatchOffset = 0; @@ -701,56 +567,39 @@ void Controller::statSession::update(uint64_t index, Comms::Statistics &statComm servPackRetrans += currPktRetrans - prevPktRetrans; } } - if (currDown + currUp >= COUNTABLE_BYTES){ - if (sessionType == SESS_UNSET){ - if (myConnector.size() >= 5 && myConnector.substr(0, 5) == "INPUT"){ - ++servInputs; - createEmptyStatsIfNeeded(myStream); - streamStats[myStream].inputs++; - streamStats[myStream].currIns++; - sessionType = SESS_INPUT; - }else if (myConnector.size() >= 6 && myConnector.substr(0, 6) == "OUTPUT"){ - ++servOutputs; - createEmptyStatsIfNeeded(myStream); - streamStats[myStream].outputs++; - streamStats[myStream].currOuts++; - sessionType = SESS_OUTPUT; - }else{ - ++servViewers; - createEmptyStatsIfNeeded(myStream); - streamStats[myStream].viewers++; - streamStats[myStream].currViews++; - sessionType = SESS_VIEWER; - } + if (sessionType == SESS_UNSET){ + if (curConnector.size() >= 5 && curConnector.substr(0, 5) == "INPUT"){ + ++servInputs; + createEmptyStatsIfNeeded(streamName); + streamStats[streamName].inputs++; + streamStats[streamName].currIns++; + sessionType = SESS_INPUT; + }else if (curConnector.size() >= 6 && curConnector.substr(0, 6) == "OUTPUT"){ + ++servOutputs; + createEmptyStatsIfNeeded(streamName); + streamStats[streamName].outputs++; + streamStats[streamName].currOuts++; + sessionType = SESS_OUTPUT; + }else{ + ++servViewers; + createEmptyStatsIfNeeded(streamName); + streamStats[streamName].viewers++; + streamStats[streamName].currViews++; + sessionType = SESS_VIEWER; } - // If previous < COUNTABLE_BYTES, we haven't counted any data so far. - // We need to count all the data in that case, otherwise we only count the difference. - if (noBWCount != 2){ //only count connections that are countable - if (prevUp + prevDown < COUNTABLE_BYTES){ - if (!myStream.size() || myStream[0] == 0){ - if (streamStats.count(myStream)){streamStats.erase(myStream);} - }else{ - createEmptyStatsIfNeeded(myStream); - streamStats[myStream].upBytes += currUp; - streamStats[myStream].downBytes += currDown; - streamStats[myStream].packSent += currPktSent; - streamStats[myStream].packLoss += currPktLost; - streamStats[myStream].packRetrans += currPktRetrans; - if (sessionType == SESS_VIEWER){streamStats[myStream].viewSeconds += lastSec - firstSec;} - } - }else{ - if (!myStream.size() || myStream[0] == 0){ - if (streamStats.count(myStream)){streamStats.erase(myStream);} - }else{ - createEmptyStatsIfNeeded(myStream); - streamStats[myStream].upBytes += currUp - prevUp; - streamStats[myStream].downBytes += currDown - prevDown; - streamStats[myStream].packSent += currPktSent - prevPktSent; - streamStats[myStream].packLoss += currPktLost - prevPktLost; - streamStats[myStream].packRetrans += currPktRetrans - prevPktRetrans; - if (sessionType == SESS_VIEWER){streamStats[myStream].viewSeconds += secIncr;} - } - } + } + // Only count connections that are countable + if (noBWCount != 2){ + if (!streamName.size() || streamName[0] == 0){ + if (streamStats.count(streamName)){streamStats.erase(streamName);} + }else{ + createEmptyStatsIfNeeded(streamName); + streamStats[streamName].upBytes += currUp - prevUp; + streamStats[streamName].downBytes += currDown - prevDown; + streamStats[streamName].packSent += currPktSent - prevPktSent; + streamStats[streamName].packLoss += currPktLost - prevPktLost; + streamStats[streamName].packRetrans += currPktRetrans - prevPktRetrans; + if (sessionType == SESS_VIEWER){streamStats[streamName].viewSeconds += secIncr;} } } } @@ -759,53 +608,19 @@ Controller::sessType Controller::statSession::getSessType(){ return sessionType; } -/// Archives connection log entries older than the given cutOff point. -void Controller::statSession::wipeOld(uint64_t cutOff){ - if (firstSec > cutOff){return;} - firstSec = 0xFFFFFFFFFFFFFFFFull; - if (oldConns.size()){ - for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ - while (it->log.size() && it->log.begin()->first < cutOff){ - if (it->log.size() == 1){ - wipedDown += it->log.begin()->second.down; - wipedUp += it->log.begin()->second.up; - wipedPktCount += it->log.begin()->second.pktCount; - wipedPktLost += it->log.begin()->second.pktLost; - wipedPktRetransmit += it->log.begin()->second.pktRetransmit; - } - it->log.erase(it->log.begin()); - } - if (it->log.size()){ - if (firstSec > it->log.begin()->first){firstSec = it->log.begin()->first;} - } - } - while (oldConns.size() && !oldConns.begin()->log.size()){oldConns.pop_front();} - } - if (curConns.size()){ - for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ - while (it->second.log.size() > 1 && it->second.log.begin()->first < cutOff){ - it->second.log.erase(it->second.log.begin()); - } - if (it->second.log.size()){ - if (firstSec > it->second.log.begin()->first){firstSec = it->second.log.begin()->first;} - } - } - } -} - -void Controller::statSession::dropSession(const Controller::sessIndex &index){ - if (!tracked || curConns.size()){return;} +Controller::statSession::~statSession(){ + if (!tracked){return;} switch (sessionType){ - case SESS_INPUT: - if (streamStats.count(index.streamName) && streamStats[index.streamName].currIns){streamStats[index.streamName].currIns--;} - break; - case SESS_OUTPUT: - if (streamStats.count(index.streamName) && streamStats[index.streamName].currOuts){streamStats[index.streamName].currOuts--;} - break; - case SESS_VIEWER: - if (streamStats.count(index.streamName) && streamStats[index.streamName].currViews){streamStats[index.streamName].currViews--;} - break; - default: break; + case SESS_INPUT: + if (streamStats.count(streamName) && streamStats[streamName].currIns){streamStats[streamName].currIns--;} + break; + case SESS_OUTPUT: + if (streamStats.count(streamName) && streamStats[streamName].currOuts){streamStats[streamName].currOuts--;} + break; + case SESS_VIEWER: + if (streamStats.count(streamName) && streamStats[streamName].currViews){streamStats[streamName].currViews--;} + break; + default: break; } uint64_t duration = lastSec - firstActive; if (duration < 1){duration = 1;} @@ -815,13 +630,13 @@ void Controller::statSession::dropSession(const Controller::sessIndex &index){ tagStream << "[" << *it << "]"; } } - Controller::logAccess(index.ID, index.streamName, index.connector, index.host, duration, getUp(), + Controller::logAccess(sessId, streamName, curConnector, host, duration, getUp(), getDown(), tagStream.str()); if (Controller::accesslog.size()){ if (Controller::accesslog == "LOG"){ std::stringstream accessStr; - accessStr << "Session <" << index.ID << "> " << index.streamName << " (" << index.connector - << ") from " << index.host << " ended after " << duration << "s, avg " + accessStr << "Session <" << sessId << "> " << streamName << " (" << curConnector + << ") from " << host << " ended after " << duration << "s, avg " << getUp() / duration / 1024 << "KB/s up " << getDown() / duration / 1024 << "KB/s down."; if (tags.size()){accessStr << " Tags: " << tagStream.str();} Controller::Log("ACCS", accessStr.str()); @@ -845,8 +660,8 @@ void Controller::statSession::dropSession(const Controller::sessIndex &index){ time(&rawtime); timeinfo = localtime_r(&rawtime, &tmptime); strftime(buffer, 100, "%F %H:%M:%S", timeinfo); - accLogFile << buffer << ", " << index.ID << ", " << index.streamName << ", " - << index.connector << ", " << index.host << ", " << duration << ", " + accLogFile << buffer << ", " << sessId << ", " << streamName << ", " + << curConnector << ", " << host << ", " << duration << ", " << getUp() / duration / 1024 << ", " << getDown() / duration / 1024 << ", "; if (tags.size()){accLogFile << tagStream.str();} accLogFile << std::endl; @@ -857,77 +672,21 @@ void Controller::statSession::dropSession(const Controller::sessIndex &index){ firstActive = 0; firstSec = 0xFFFFFFFFFFFFFFFFull; lastSec = 0; - wipedUp = 0; - wipedDown = 0; - wipedPktCount = 0; - wipedPktLost = 0; - wipedPktRetransmit = 0; - oldConns.clear(); sessionType = SESS_UNSET; } -/// Archives the given connection. -void Controller::statSession::finish(uint64_t index){ - oldConns.push_back(curConns[index]); - curConns.erase(index); -} - /// Constructs an empty session Controller::statSession::statSession(){ firstActive = 0; tracked = false; firstSec = 0xFFFFFFFFFFFFFFFFull; lastSec = 0; - sync = 1; - wipedUp = 0; - wipedDown = 0; - wipedPktCount = 0; - wipedPktLost = 0; - wipedPktRetransmit = 0; sessionType = SESS_UNSET; noBWCount = 0; -} - -/// Moves the given connection to the given session -void Controller::statSession::switchOverTo(statSession &newSess, uint64_t index){ - // add to the given session first - newSess.curConns[index] = curConns[index]; - // if this connection has data, update firstSec/lastSec if needed - if (curConns[index].log.size()){ - if (newSess.firstSec > curConns[index].log.begin()->first){ - newSess.firstSec = curConns[index].log.begin()->first; - } - if (newSess.lastSec < curConns[index].log.rbegin()->first){ - newSess.lastSec = curConns[index].log.rbegin()->first; - } - } - // remove from current session - curConns.erase(index); - // if there was any data, recalculate this session's firstSec and lastSec. - if (newSess.curConns[index].log.size()){ - firstSec = 0xFFFFFFFFFFFFFFFFull; - lastSec = 0; - if (oldConns.size()){ - for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ - if (it->log.size()){ - if (firstSec > it->log.begin()->first){firstSec = it->log.begin()->first;} - if (lastSec < it->log.rbegin()->first){lastSec = it->log.rbegin()->first;} - } - } - } - if (curConns.size()){ - for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ - if (it->second.log.size()){ - if (firstSec > it->second.log.begin()->first){ - firstSec = it->second.log.begin()->first; - } - if (lastSec < it->second.log.rbegin()->first){ - lastSec = it->second.log.rbegin()->first; - } - } - } - } - } + streamName = ""; + host = ""; + curConnector = ""; + sessId = ""; } /// Returns the first measured timestamp in this session. @@ -944,39 +703,34 @@ uint64_t Controller::statSession::getEnd(){ bool Controller::statSession::hasDataFor(uint64_t t){ if (lastSec < t){return false;} if (firstSec > t){return false;} - if (oldConns.size()){ - for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ - if (it->hasDataFor(t)){return true;} - } - } - if (curConns.size()){ - for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ - if (it->second.hasDataFor(t)){return true;} - } - } - return false; -} - -/// Returns true if there is any data for this session. -bool Controller::statSession::hasData(){ - if (!firstSec && !lastSec){return false;} - if (curConns.size()){return true;} - if (oldConns.size()){ - for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ - if (it->log.size()){return true;} - } - } + if (curData.hasDataFor(t)){return true;} return false; } /// Returns true if this session should count as a viewer on the given timestamp. bool Controller::statSession::isViewerOn(uint64_t t){ - return getUp(t) + getDown(t) > COUNTABLE_BYTES; + return getUp(t) + getDown(t); +} + +std::string Controller::statSession::getStreamName(){ + return streamName; +} + +std::string Controller::statSession::getHost(){ + return host; +} + +std::string Controller::statSession::getSessId(){ + return sessId; +} + +std::string Controller::statSession::getCurrentProtocols(){ + return curConnector; } /// Returns true if this session should be considered connected -bool Controller::statSession::isConnected(){ - return curConns.size(); +uint64_t Controller::statSession::newestDataPoint(){ + return lastSec; } /// Returns true if this session has started (tracked == true) but not yet ended (log entry written) @@ -986,188 +740,103 @@ bool Controller::statSession::isTracked(){ /// Returns the cumulative connected time for this session at timestamp t. uint64_t Controller::statSession::getConnTime(uint64_t t){ - uint64_t retVal = 0; - if (oldConns.size()){ - for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ - if (it->hasDataFor(t)){retVal += it->getDataFor(t).time;} - } + if (curData.hasDataFor(t)){ + return curData.getDataFor(t).time; } - if (curConns.size()){ - for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ - if (it->second.hasDataFor(t)){retVal += it->second.getDataFor(t).time;} - } + return 0; +} + +/// Returns the cumulative connected time for this session. +uint64_t Controller::statSession::getConnTime(){ + if (curData.log.size()){ + return curData.log.rbegin()->second.time; } - return retVal; + return 0; } /// Returns the last requested media timestamp for this session at timestamp t. uint64_t Controller::statSession::getLastSecond(uint64_t t){ - if (curConns.size()){ - for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ - if (it->second.hasDataFor(t)){return it->second.getDataFor(t).lastSecond;} - } - } - if (oldConns.size()){ - for (std::deque::reverse_iterator it = oldConns.rbegin(); it != oldConns.rend(); ++it){ - if (it->hasDataFor(t)){return it->getDataFor(t).lastSecond;} - } + if (curData.hasDataFor(t)){ + return curData.getDataFor(t).lastSecond; } return 0; } /// Returns the cumulative downloaded bytes for this session at timestamp t. uint64_t Controller::statSession::getDown(uint64_t t){ - uint64_t retVal = wipedDown; - if (oldConns.size()){ - for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ - if (it->hasDataFor(t)){retVal += it->getDataFor(t).down;} - } + if (curData.hasDataFor(t)){ + return curData.getDataFor(t).down; } - if (curConns.size()){ - for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ - if (it->second.hasDataFor(t)){retVal += it->second.getDataFor(t).down;} - } - } - return retVal; + return 0; } /// Returns the cumulative uploaded bytes for this session at timestamp t. uint64_t Controller::statSession::getUp(uint64_t t){ - uint64_t retVal = wipedUp; - if (oldConns.size()){ - for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ - if (it->hasDataFor(t)){retVal += it->getDataFor(t).up;} - } - } - if (curConns.size()){ - for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ - if (it->second.hasDataFor(t)){retVal += it->second.getDataFor(t).up;} - } + if (curData.hasDataFor(t)){ + return curData.getDataFor(t).up; } - return retVal; + return 0; } /// Returns the cumulative downloaded bytes for this session at timestamp t. uint64_t Controller::statSession::getDown(){ - uint64_t retVal = wipedDown; - if (oldConns.size()){ - for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ - if (it->log.size()){retVal += it->log.rbegin()->second.down;} - } + if (curData.log.size()){ + return curData.log.rbegin()->second.down; } - if (curConns.size()){ - for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ - if (it->second.log.size()){retVal += it->second.log.rbegin()->second.down;} - } - } - return retVal; + return 0; } /// Returns the cumulative uploaded bytes for this session at timestamp t. uint64_t Controller::statSession::getUp(){ - uint64_t retVal = wipedUp; - if (oldConns.size()){ - for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ - if (it->log.size()){retVal += it->log.rbegin()->second.up;} - } - } - if (curConns.size()){ - for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ - if (it->second.log.size()){retVal += it->second.log.rbegin()->second.up;} - } + if (curData.log.size()){ + return curData.log.rbegin()->second.up; } - return retVal; + return 0; } uint64_t Controller::statSession::getPktCount(uint64_t t){ - uint64_t retVal = wipedPktCount; - if (oldConns.size()){ - for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ - if (it->hasDataFor(t)){retVal += it->getDataFor(t).pktCount;} - } + if (curData.hasDataFor(t)){ + return curData.getDataFor(t).pktCount; } - if (curConns.size()){ - for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ - if (it->second.hasDataFor(t)){retVal += it->second.getDataFor(t).pktCount;} - } - } - return retVal; + return 0; } /// Returns the cumulative uploaded bytes for this session at timestamp t. uint64_t Controller::statSession::getPktCount(){ - uint64_t retVal = wipedPktCount; - if (oldConns.size()){ - for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ - if (it->log.size()){retVal += it->log.rbegin()->second.pktCount;} - } - } - if (curConns.size()){ - for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ - if (it->second.log.size()){retVal += it->second.log.rbegin()->second.pktCount;} - } + if (curData.log.size()){ + return curData.log.rbegin()->second.pktCount; } - return retVal; + return 0; } + uint64_t Controller::statSession::getPktLost(uint64_t t){ - uint64_t retVal = wipedPktLost; - if (oldConns.size()){ - for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ - if (it->hasDataFor(t)){retVal += it->getDataFor(t).pktLost;} - } + if (curData.hasDataFor(t)){ + return curData.getDataFor(t).pktLost; } - if (curConns.size()){ - for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ - if (it->second.hasDataFor(t)){retVal += it->second.getDataFor(t).pktLost;} - } - } - return retVal; + return 0; } /// Returns the cumulative uploaded bytes for this session at timestamp t. uint64_t Controller::statSession::getPktLost(){ - uint64_t retVal = wipedPktLost; - if (oldConns.size()){ - for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ - if (it->log.size()){retVal += it->log.rbegin()->second.pktLost;} - } - } - if (curConns.size()){ - for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ - if (it->second.log.size()){retVal += it->second.log.rbegin()->second.pktLost;} - } + if (curData.log.size()){ + return curData.log.rbegin()->second.pktLost; } - return retVal; + return 0; } + uint64_t Controller::statSession::getPktRetransmit(uint64_t t){ - uint64_t retVal = wipedPktRetransmit; - if (oldConns.size()){ - for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ - if (it->hasDataFor(t)){retVal += it->getDataFor(t).pktRetransmit;} - } + if (curData.hasDataFor(t)){ + return curData.getDataFor(t).pktRetransmit; } - if (curConns.size()){ - for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ - if (it->second.hasDataFor(t)){retVal += it->second.getDataFor(t).pktRetransmit;} - } - } - return retVal; + return 0; } /// Returns the cumulative uploaded bytes for this session at timestamp t. uint64_t Controller::statSession::getPktRetransmit(){ - uint64_t retVal = wipedPktRetransmit; - if (oldConns.size()){ - for (std::deque::iterator it = oldConns.begin(); it != oldConns.end(); ++it){ - if (it->log.size()){retVal += it->log.rbegin()->second.pktRetransmit;} - } - } - if (curConns.size()){ - for (std::map::iterator it = curConns.begin(); it != curConns.end(); ++it){ - if (it->second.log.size()){retVal += it->second.log.rbegin()->second.pktRetransmit;} - } + if (curData.log.size()){ + return curData.log.rbegin()->second.pktRetransmit; } - return retVal; + return 0; } /// Returns the cumulative downloaded bytes per second for this session at timestamp t. @@ -1207,6 +876,7 @@ Controller::statLog &Controller::statStorage::getDataFor(unsigned long long t){ empty.pktCount = 0; empty.pktLost = 0; empty.pktRetransmit = 0; + empty.connectors = ""; return empty; } std::map::iterator it = log.upper_bound(t); @@ -1216,7 +886,7 @@ Controller::statLog &Controller::statStorage::getDataFor(unsigned long long t){ /// This function is called by parseStatistics. /// It updates the internally saved statistics data. -void Controller::statStorage::update(Comms::Statistics &statComm, size_t index){ +void Controller::statStorage::update(Comms::Sessions &statComm, size_t index){ statLog tmp; tmp.time = statComm.getTime(index); tmp.lastSecond = statComm.getLastSecond(index); @@ -1225,56 +895,58 @@ void Controller::statStorage::update(Comms::Statistics &statComm, size_t index){ tmp.pktCount = statComm.getPacketCount(index); tmp.pktLost = statComm.getPacketLostCount(index); tmp.pktRetransmit = statComm.getPacketRetransmitCount(index); + tmp.connectors = statComm.getConnector(index); log[statComm.getNow(index)] = tmp; - // wipe data older than approx. STAT_CUTOFF seconds - /// \todo Remove least interesting data first. - if (log.size() > STAT_CUTOFF){log.erase(log.begin());} + // wipe data older than STAT_CUTOFF seconds + while (log.size() && log.begin()->first < Util::bootSecs() - STAT_CUTOFF){log.erase(log.begin());} } void Controller::statLeadIn(){ statDropoff = Util::bootSecs() - 3; } -void Controller::statOnActive(size_t id){ - // calculate the current session index, store as idx. - sessIndex idx(statComm, id); +void Controller::statOnActive(size_t id){ if (statComm.getNow(id) >= statDropoff){ - // if the connection was already indexed and it has changed, move it - if (connToSession.count(id) && connToSession[id] != idx){ - if (sessions[connToSession[id]].getSessType() != SESS_UNSET){ - INFO_MSG("Switching connection %zu from active session %s over to %s", id, - connToSession[id].toStr().c_str(), idx.toStr().c_str()); - }else{ - INFO_MSG("Switching connection %zu from inactive session %s over to %s", id, - connToSession[id].toStr().c_str(), idx.toStr().c_str()); - } - sessions[connToSession[id]].switchOverTo(sessions[idx], id); - // Destroy this session without calling dropSession, because it was merged into another. What session? We never made it. Stop asking hard questions. Go, shoo. *sprays water* - if (!sessions[connToSession[id]].hasData()){sessions.erase(connToSession[id]);} - } - if (!connToSession.count(id)){ - INSANE_MSG("New connection: %zu as %s", id, idx.toStr().c_str()); - } - // store the index for later comparison - connToSession[id] = idx; // update the session with the latest data - sessions[idx].update(id, statComm); + sessions[statComm.getSessId(id)].update(id, statComm); } } + void Controller::statOnDisconnect(size_t id){ - sessIndex idx(statComm, id); - INSANE_MSG("Ended connection: %zu as %s", id, idx.toStr().c_str()); - sessions[idx].finish(id); - connToSession.erase(id); + // Check to see if cleanup is required (when a Session binary fails) + const std::string thisSessionId = statComm.getSessId(id); + // Try to lock to see if the session crashed during boot + IPC::semaphore sessionLock; + char semName[NAME_BUFFER_SIZE]; + snprintf(semName, NAME_BUFFER_SIZE, SEM_SESSION, thisSessionId.c_str()); + sessionLock.open(semName, O_CREAT | O_RDWR, ACCESSPERMS, 1); + if (!sessionLock.tryWaitOneSecond()){ + // Session likely crashed during boot. Remove the session lock which was created on bootup of the session + sessionLock.unlink(); + }else if (!statComm.sessIdExists(thisSessionId)){ + // There is no running process managing this session, so check if the data page still exists + IPC::sharedPage dataPage; + char userPageName[NAME_BUFFER_SIZE]; + snprintf(userPageName, NAME_BUFFER_SIZE, COMMS_SESSIONS, thisSessionId.c_str()); + dataPage.init(userPageName, 1, false, false); + if(dataPage){ + // Session likely crashed while it was running + dataPage.init(userPageName, 1, true); + FAIL_MSG("Session '%s' got canceled unexpectedly. Hoovering up the left overs...", thisSessionId.c_str()); + } + // Finally remove the session lock which was created on bootup of the session + sessionLock.unlink(); + } } + void Controller::statLeadOut(){} /// Returns true if this stream has at least one connected client. bool Controller::hasViewers(std::string streamName){ if (sessions.size()){ long long currTime = Util::bootSecs(); - for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ - if (it->first.streamName == streamName && + for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ + if (it->second.getStreamName() == streamName && (it->second.hasDataFor(currTime) || it->second.hasDataFor(currTime - 1))){ return true; } @@ -1377,7 +1049,6 @@ void Controller::fillClients(JSON::Value &req, JSON::Value &rep){ if (fields & STAT_CLI_UP){rep["fields"].append("up");} if (fields & STAT_CLI_BPS_DOWN){rep["fields"].append("downbps");} if (fields & STAT_CLI_BPS_UP){rep["fields"].append("upbps");} - if (fields & STAT_CLI_CRC){rep["fields"].append("crc");} if (fields & STAT_CLI_SESSID){rep["fields"].append("sessid");} if (fields & STAT_CLI_PKTCOUNT){rep["fields"].append("pktcount");} if (fields & STAT_CLI_PKTLOST){rep["fields"].append("pktlost");} @@ -1386,26 +1057,25 @@ void Controller::fillClients(JSON::Value &req, JSON::Value &rep){ rep["data"].null(); // loop over all sessions if (sessions.size()){ - for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ + for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ unsigned long long time = reqTime; if (now && reqTime - it->second.getEnd() < 5){time = it->second.getEnd();} // data present and wanted? insert it! if ((it->second.getEnd() >= time && it->second.getStart() <= time) && - (!streams.size() || streams.count(it->first.streamName)) && - (!protos.size() || protos.count(it->first.connector))){ + (!streams.size() || streams.count(it->second.getStreamName())) && + (!protos.size() || protos.count(it->second.getCurrentProtocols()))){ if (it->second.hasDataFor(time)){ JSON::Value d; - if (fields & STAT_CLI_HOST){d.append(it->first.host);} - if (fields & STAT_CLI_STREAM){d.append(it->first.streamName);} - if (fields & STAT_CLI_PROTO){d.append(it->first.connector);} + if (fields & STAT_CLI_HOST){d.append(it->second.getHost());} + if (fields & STAT_CLI_STREAM){d.append(it->second.getStreamName());} + if (fields & STAT_CLI_PROTO){d.append(it->second.getCurrentProtocols());} if (fields & STAT_CLI_CONNTIME){d.append(it->second.getConnTime(time));} if (fields & STAT_CLI_POSITION){d.append(it->second.getLastSecond(time));} if (fields & STAT_CLI_DOWN){d.append(it->second.getDown(time));} if (fields & STAT_CLI_UP){d.append(it->second.getUp(time));} if (fields & STAT_CLI_BPS_DOWN){d.append(it->second.getBpsDown(time));} if (fields & STAT_CLI_BPS_UP){d.append(it->second.getBpsUp(time));} - if (fields & STAT_CLI_CRC){d.append(it->first.crc);} - if (fields & STAT_CLI_SESSID){d.append(it->first.ID);} + if (fields & STAT_CLI_SESSID){d.append(it->second.getSessId());} if (fields & STAT_CLI_PKTCOUNT){d.append(it->second.getPktCount(time));} if (fields & STAT_CLI_PKTLOST){d.append(it->second.getPktLost(time));} if (fields & STAT_CLI_PKTRETRANSMIT){d.append(it->second.getPktRetransmit(time));} @@ -1463,12 +1133,12 @@ void Controller::fillHasStats(JSON::Value &req, JSON::Value &rep){ { tthread::lock_guard guard(statsMutex); if (sessions.size()){ - for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ + for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ if (it->second.getSessType() == SESS_INPUT){ - streams.insert(it->first.streamName); + streams.insert(it->second.getStreamName()); }else{ - streams.insert(it->first.streamName); - if (it->second.getSessType() == SESS_VIEWER){clients[it->first.streamName]++;} + streams.insert(it->second.getStreamName()); + if (it->second.getSessType() == SESS_VIEWER){clients[it->second.getStreamName()]++;} } } } @@ -1742,12 +1412,12 @@ void Controller::fillTotals(JSON::Value &req, JSON::Value &rep){ // loop over all sessions /// \todo Make the interval configurable instead of 1 second if (sessions.size()){ - for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ + for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ // data present and wanted? insert it! if ((it->second.getEnd() >= (unsigned long long)reqStart || it->second.getStart() <= (unsigned long long)reqEnd) && - (!streams.size() || streams.count(it->first.streamName)) && - (!protos.size() || protos.count(it->first.connector))){ + (!streams.size() || streams.count(it->second.getStreamName())) && + (!protos.size() || protos.count(it->second.getCurrentProtocols()))){ for (unsigned long long i = reqStart; i <= reqEnd; ++i){ if (it->second.hasDataFor(i)){ totalsCount[i].add(it->second.getBpsDown(i), it->second.getBpsUp(i), it->second.getSessType(), it->second.getPktCount(), it->second.getPktLost(), it->second.getPktRetransmit()); @@ -1831,6 +1501,25 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int H.SetHeader("Server", APPIDENT); H.StartResponse("200", "OK", H, conn, true); + // Counters of current active viewers, inputs and outputs of the Session stats cache + std::map outputs; + uint32_t totViewers = 0; + uint32_t totInputs = 0; + uint32_t totOutputs = 0; + for (uint64_t idx = 0; idx < statComm.recordCount(); idx++){ + if (statComm.getStatus(idx) == COMM_STATUS_INVALID || statComm.getStatus(idx) & COMM_STATUS_DISCONNECT){continue;} + const std::string thisSessId = statComm.getSessId(idx); + // Count active viewers, inputs, outputs and protocols + if (thisSessId[0] == 'I'){ + totInputs++; + }else if (thisSessId[0] == 'O'){ + totOutputs++; + outputs[statComm.getConnector(idx)]++; + }else{ + totViewers++; + } + } + // Collect core server stats uint64_t mem_total = 0, mem_free = 0, mem_bufcache = 0; uint64_t bw_up_total = 0, bw_down_total = 0; @@ -1904,109 +1593,69 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int response << "# TYPE mist_shm_used gauge\n"; response << "mist_shm_used " << (shm_total - shm_free) << "\n\n"; - if (Controller::triggerStats.size()){ - response << "# HELP mist_trigger_count Total executions for the given trigger\n"; - response << "# HELP mist_trigger_time Total execution time in millis for the given trigger\n"; - response << "# HELP mist_trigger_fails Total failed executions for the given trigger\n"; - for (std::map::iterator it = Controller::triggerStats.begin(); - it != Controller::triggerStats.end(); it++){ - response << "mist_trigger_count{trigger=\"" << it->first << "\"}" << it->second.totalCount << "\n"; - response << "mist_trigger_time{trigger=\"" << it->first << "\"}" << it->second.ms << "\n"; - response << "mist_trigger_fails{trigger=\"" << it->first << "\"}" << it->second.failCount << "\n"; + response << "# HELP mist_viewseconds_total Number of seconds any media was received by a viewer.\n"; + response << "# TYPE mist_viewseconds_total counter\n"; + response << "mist_viewseconds_total " << servSeconds + viewSecondsTotal << "\n"; + + response << "\n# HELP mist_sessions_count Counts of unique sessions by type since server " + "start.\n"; + response << "# TYPE mist_sessions_count counter\n"; + response << "mist_sessions_count{sessType=\"viewers\"}" << servViewers << "\n"; + response << "mist_sessions_count{sessType=\"incoming\"}" << servInputs << "\n"; + response << "mist_sessions_count{sessType=\"outgoing\"}" << servOutputs << "\n\n"; + + response << "# HELP mist_bw_total Count of bytes handled since server start, by direction.\n"; + response << "# TYPE mist_bw_total counter\n"; + response << "stat_bw_total{direction=\"up\"}" << bw_up_total << "\n"; + response << "stat_bw_total{direction=\"down\"}" << bw_down_total << "\n\n"; + response << "mist_bw_total{direction=\"up\"}" << servUpBytes << "\n"; + response << "mist_bw_total{direction=\"down\"}" << servDownBytes << "\n\n"; + response << "mist_bw_other{direction=\"up\"}" << servUpOtherBytes << "\n"; + response << "mist_bw_other{direction=\"down\"}" << servDownOtherBytes << "\n\n"; + response << "mist_bw_limit " << bwLimit << "\n\n"; + + response << "# HELP mist_packets_total Total number of packets sent/received/lost over lossy protocols, server-wide.\n"; + response << "# TYPE mist_packets_total counter\n"; + response << "mist_packets_total{pkttype=\"sent\"}" << servPackSent << "\n"; + response << "mist_packets_total{pkttype=\"lost\"}" << servPackLoss << "\n"; + response << "mist_packets_total{pkttype=\"retrans\"}" << servPackRetrans << "\n"; + + if (outputs.size()){ + response << "# HELP mist_outputs Number of viewers active right now, server-wide, by output type.\n"; + response << "# TYPE mist_outputs gauge\n"; + for (std::map::iterator it = outputs.begin(); it != outputs.end(); ++it){ + response << "mist_outputs{output=\"" << it->first << "\"}" << it->second << "\n"; } response << "\n"; } {// Scope for shortest possible blocking of statsMutex tthread::lock_guard guard(statsMutex); - // collect the data first - std::map outputs; - unsigned long totViewers = 0, totInputs = 0, totOutputs = 0; - unsigned int tOut = Util::bootSecs() - STATS_DELAY; - unsigned int tIn = Util::bootSecs() - STATS_INPUT_DELAY; - // check all sessions - if (sessions.size()){ - for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ - switch (it->second.getSessType()){ - case SESS_UNSET: break; - case SESS_VIEWER: - if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){ - outputs[it->first.connector]++; - totViewers++; - } - break; - case SESS_INPUT: - if (it->second.hasDataFor(tIn) && it->second.isViewerOn(tIn)){totInputs++;} - break; - case SESS_OUTPUT: - if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){totOutputs++;} - break; - } - } - } - response << "# HELP mist_sessions_total Number of sessions active right now, server-wide, by " - "type.\n"; + response << "# HELP mist_sessions_total Number of sessions active right now, server-wide, by type.\n"; response << "# TYPE mist_sessions_total gauge\n"; response << "mist_sessions_total{sessType=\"viewers\"}" << totViewers << "\n"; response << "mist_sessions_total{sessType=\"incoming\"}" << totInputs << "\n"; response << "mist_sessions_total{sessType=\"outgoing\"}" << totOutputs << "\n"; - response << "mist_sessions_total{sessType=\"cached\"}" << sessions.size() << "\n\n"; - - response << "# HELP mist_viewseconds_total Number of seconds any media was received by a viewer.\n"; - response << "# TYPE mist_viewseconds_total counter\n"; - response << "mist_viewseconds_total " << servSeconds << "\n"; + response << "mist_sessions_total{sessType=\"cached\"}" << sessions.size() << "\n"; - response << "# HELP mist_outputs Number of viewers active right now, server-wide, by output type.\n"; - response << "# TYPE mist_outputs gauge\n"; - for (std::map::iterator it = outputs.begin(); it != outputs.end(); ++it){ - response << "mist_outputs{output=\"" << it->first << "\"}" << it->second << "\n"; - } - response << "\n"; - - response << "# HELP mist_sessions_count Counts of unique sessions by type since server " - "start.\n"; - response << "# TYPE mist_sessions_count counter\n"; - response << "mist_sessions_count{sessType=\"viewers\"}" << servViewers << "\n"; - response << "mist_sessions_count{sessType=\"incoming\"}" << servInputs << "\n"; - response << "mist_sessions_count{sessType=\"outgoing\"}" << servOutputs << "\n\n"; - - response << "# HELP mist_bw_total Count of bytes handled since server start, by direction.\n"; - response << "# TYPE mist_bw_total counter\n"; - response << "stat_bw_total{direction=\"up\"}" << bw_up_total << "\n"; - response << "stat_bw_total{direction=\"down\"}" << bw_down_total << "\n\n"; - response << "mist_bw_total{direction=\"up\"}" << servUpBytes << "\n"; - response << "mist_bw_total{direction=\"down\"}" << servDownBytes << "\n\n"; - response << "mist_bw_other{direction=\"up\"}" << servUpOtherBytes << "\n"; - response << "mist_bw_other{direction=\"down\"}" << servDownOtherBytes << "\n\n"; - response << "mist_bw_limit " << bwLimit << "\n\n"; - - response << "# HELP mist_packets_total Total number of packets sent/received/lost over lossy protocols, server-wide.\n"; - response << "# TYPE mist_packets_total counter\n"; - response << "mist_packets_total{pkttype=\"sent\"}" << servPackSent << "\n"; - response << "mist_packets_total{pkttype=\"lost\"}" << servPackLoss << "\n"; - response << "mist_packets_total{pkttype=\"retrans\"}" << servPackRetrans << "\n"; - - response << "\n# HELP mist_viewers Number of sessions by type and stream active right now.\n"; - response << "# TYPE mist_viewers gauge\n"; - response << "# HELP mist_viewcount Count of unique viewer sessions since stream start, per " + response << "\n# HELP mist_viewcount Count of unique viewer sessions since stream start, per " "stream.\n"; response << "# TYPE mist_viewcount counter\n"; - response << "# HELP mist_bw Count of bytes handled since stream start, by direction.\n"; - response << "# TYPE mist_bw counter\n"; response << "# HELP mist_viewseconds Number of seconds any media was received by a viewer.\n"; response << "# TYPE mist_viewseconds counter\n"; + response << "# HELP mist_bw Count of bytes handled since stream start, by direction.\n"; + response << "# TYPE mist_bw counter\n"; response << "# HELP mist_packets Total number of packets sent/received/lost over lossy protocols.\n"; response << "# TYPE mist_packets counter\n"; - response << "mist_viewseconds_total " << servSeconds << "\n"; for (std::map::iterator it = streamStats.begin(); - it != streamStats.end(); ++it){ + it != streamStats.end(); ++it){ response << "mist_sessions{stream=\"" << it->first << "\",sessType=\"viewers\"}" - << it->second.currViews << "\n"; + << it->second.currViews << "\n"; response << "mist_sessions{stream=\"" << it->first << "\",sessType=\"incoming\"}" - << it->second.currIns << "\n"; + << it->second.currIns << "\n"; response << "mist_sessions{stream=\"" << it->first << "\",sessType=\"outgoing\"}" - << it->second.currOuts << "\n"; + << it->second.currOuts << "\n"; response << "mist_viewcount{stream=\"" << it->first << "\"}" << it->second.viewers << "\n"; response << "mist_viewseconds{stream=\"" << it->first << "\"} " << it->second.viewSeconds << "\n"; response << "mist_bw{stream=\"" << it->first << "\",direction=\"up\"}" << it->second.upBytes << "\n"; @@ -2015,6 +1664,19 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int response << "mist_packets{stream=\"" << it->first << "\",pkttype=\"lost\"}" << it->second.packLoss << "\n"; response << "mist_packets{stream=\"" << it->first << "\",pkttype=\"retrans\"}" << it->second.packRetrans << "\n"; } + + if (Controller::triggerStats.size()){ + response << "\n# HELP mist_trigger_count Total executions for the given trigger\n"; + response << "# HELP mist_trigger_time Total execution time in millis for the given trigger\n"; + response << "# HELP mist_trigger_fails Total failed executions for the given trigger\n"; + for (std::map::iterator it = Controller::triggerStats.begin(); + it != Controller::triggerStats.end(); it++){ + response << "mist_trigger_count{trigger=\"" << it->first << "\"}" << it->second.totalCount << "\n"; + response << "mist_trigger_time{trigger=\"" << it->first << "\"}" << it->second.ms << "\n"; + response << "mist_trigger_fails{trigger=\"" << it->first << "\"}" << it->second.failCount << "\n"; + } + response << "\n"; + } } H.Chunkify(response.str(), conn); } @@ -2026,58 +1688,33 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int resp["shm_total"] = shm_total; resp["shm_used"] = (shm_total - shm_free); resp["logs"] = Controller::logCounter; - if (Controller::triggerStats.size()){ - for (std::map::iterator it = Controller::triggerStats.begin(); - it != Controller::triggerStats.end(); it++){ - JSON::Value &tVal = resp["triggers"][it->first]; - tVal["count"] = it->second.totalCount; - tVal["ms"] = it->second.ms; - tVal["fails"] = it->second.failCount; - } - } + resp["curr"].append(totViewers); + resp["curr"].append(totInputs); + resp["curr"].append(totOutputs); + resp["tot"].append(servViewers); + resp["tot"].append(servInputs); + resp["tot"].append(servOutputs); + resp["st"].append(bw_up_total); + resp["st"].append(bw_down_total); + resp["bw"].append(servUpBytes); + resp["bw"].append(servDownBytes); + resp["pkts"].append(servPackSent); + resp["pkts"].append(servPackLoss); + resp["pkts"].append(servPackRetrans); + resp["bwlimit"] = bwLimit; {// Scope for shortest possible blocking of statsMutex tthread::lock_guard guard(statsMutex); - // collect the data first - std::map outputs; - uint64_t totViewers = 0, totInputs = 0, totOutputs = 0; - uint64_t tOut = Util::bootSecs() - STATS_DELAY; - uint64_t tIn = Util::bootSecs() - STATS_INPUT_DELAY; - // check all sessions - if (sessions.size()){ - for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ - switch (it->second.getSessType()){ - case SESS_UNSET: break; - case SESS_VIEWER: - if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){ - outputs[it->first.connector]++; - totViewers++; - } - break; - case SESS_INPUT: - if (it->second.hasDataFor(tIn) && it->second.isViewerOn(tIn)){totInputs++;} - break; - case SESS_OUTPUT: - if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){totOutputs++;} - break; - } + resp["curr"].append((uint64_t)sessions.size()); + + if (Controller::triggerStats.size()){ + for (std::map::iterator it = Controller::triggerStats.begin(); + it != Controller::triggerStats.end(); it++){ + JSON::Value &tVal = resp["triggers"][it->first]; + tVal["count"] = it->second.totalCount; + tVal["ms"] = it->second.ms; + tVal["fails"] = it->second.failCount; } } - - resp["curr"].append(totViewers); - resp["curr"].append(totInputs); - resp["curr"].append(totOutputs); - resp["curr"].append((uint64_t)sessions.size()); - resp["tot"].append(servViewers); - resp["tot"].append(servInputs); - resp["tot"].append(servOutputs); - resp["st"].append(bw_up_total); - resp["st"].append(bw_down_total); - resp["bw"].append(servUpBytes); - resp["bw"].append(servDownBytes); - resp["pkts"].append(servPackSent); - resp["pkts"].append(servPackLoss); - resp["pkts"].append(servPackRetrans); - resp["bwlimit"] = bwLimit; if (Storage["config"].isMember("location") && Storage["config"]["location"].isMember("lat") && Storage["config"]["location"].isMember("lon")){ resp["loc"]["lat"] = Storage["config"]["location"]["lat"].asDouble(); resp["loc"]["lon"] = Storage["config"]["location"]["lon"].asDouble(); diff --git a/src/controller/controller_statistics.h b/src/controller/controller_statistics.h index 1cc4a82d..f798f811 100644 --- a/src/controller/controller_statistics.h +++ b/src/controller/controller_statistics.h @@ -18,8 +18,7 @@ namespace Controller{ extern bool killOnExit; - extern unsigned int maxConnsPerIP; - + /// This function is ran whenever a stream becomes active. void streamStarted(std::string stream); /// This function is ran whenever a stream becomes inactive. @@ -35,34 +34,14 @@ namespace Controller{ uint64_t pktCount; uint64_t pktLost; uint64_t pktRetransmit; + std::string connectors; }; enum sessType{SESS_UNSET = 0, SESS_INPUT, SESS_OUTPUT, SESS_VIEWER}; - /// This is a comparison and storage class that keeps sessions apart from each other. - /// Whenever two of these objects are not equal, it will create a new session. - class sessIndex{ - public: - sessIndex(); - sessIndex(const Comms::Statistics &statComm, size_t id); - std::string ID; - std::string host; - unsigned int crc; - std::string streamName; - std::string connector; - - bool operator==(const sessIndex &o) const; - bool operator!=(const sessIndex &o) const; - bool operator>(const sessIndex &o) const; - bool operator<=(const sessIndex &o) const; - bool operator<(const sessIndex &o) const; - bool operator>=(const sessIndex &o) const; - std::string toStr(); - }; - class statStorage{ public: - void update(Comms::Statistics &statComm, size_t index); + void update(Comms::Sessions &statComm, size_t index); bool hasDataFor(unsigned long long); statLog &getDataFor(unsigned long long); std::map log; @@ -75,36 +54,33 @@ namespace Controller{ uint64_t firstActive; uint64_t firstSec; uint64_t lastSec; - uint64_t wipedUp; - uint64_t wipedDown; - uint64_t wipedPktCount; - uint64_t wipedPktLost; - uint64_t wipedPktRetransmit; - std::deque oldConns; sessType sessionType; bool tracked; uint8_t noBWCount; ///< Set to 2 when not to count for external bandwidth + std::string streamName; + std::string host; + std::string curConnector; + std::string sessId; + public: statSession(); - uint32_t invalidate(); - uint32_t kill(); - char sync; - std::map curConns; + ~statSession(); + statStorage curData; std::set tags; sessType getSessType(); - void wipeOld(uint64_t); - void finish(uint64_t index); - void switchOverTo(statSession &newSess, uint64_t index); - void update(uint64_t index, Comms::Statistics &data); - void dropSession(const sessIndex &index); + void update(uint64_t index, Comms::Sessions &data); uint64_t getStart(); uint64_t getEnd(); bool isViewerOn(uint64_t time); - bool isConnected(); bool isTracked(); bool hasDataFor(uint64_t time); - bool hasData(); + std::string getStreamName(); + std::string getHost(); + std::string getSessId(); + std::string getCurrentProtocols(); + uint64_t newestDataPoint(); uint64_t getConnTime(uint64_t time); + uint64_t getConnTime(); uint64_t getLastSecond(uint64_t time); uint64_t getDown(uint64_t time); uint64_t getUp(); @@ -122,8 +98,6 @@ namespace Controller{ uint64_t getBpsUp(uint64_t start, uint64_t end); }; - extern std::map sessions; - extern std::map connToSession; extern tthread::mutex statsMutex; extern uint64_t statDropoff; @@ -155,6 +129,7 @@ namespace Controller{ void sessions_shutdown(const std::string &streamname, const std::string &protocol = ""); bool hasViewers(std::string streamName); void writeSessionCache(); /*LTS*/ + void killConnections(std::string sessId); #define PROMETHEUS_TEXT 0 #define PROMETHEUS_JSON 1 diff --git a/src/controller/controller_storage.cpp b/src/controller/controller_storage.cpp index 58e7b521..bdd52893 100644 --- a/src/controller/controller_storage.cpp +++ b/src/controller/controller_storage.cpp @@ -91,19 +91,6 @@ namespace Controller{ rlxAccs->setString("tags", tags, newEndPos); rlxAccs->setEndPos(newEndPos + 1); } - if (Triggers::shouldTrigger("USER_END", strm)){ - std::stringstream plgen; - plgen << sessId << "\n" - << strm << "\n" - << conn << "\n" - << host << "\n" - << duration << "\n" - << up << "\n" - << down << "\n" - << tags; - std::string payload = plgen.str(); - Triggers::doTrigger("USER_END", payload, strm); - } } void normalizeTrustedProxies(JSON::Value &tp){ @@ -450,7 +437,8 @@ namespace Controller{ systemBoot = globAccX.getInt("systemBoot"); } if(!globAccX.getFieldAccX("defaultStream") - || !globAccX.getFieldAccX("systemBoot")){ + || !globAccX.getFieldAccX("systemBoot") + || !globAccX.getFieldAccX("sessionMode")){ globAccX.setReload(); globCfg.master = true; globCfg.close(); @@ -461,12 +449,16 @@ namespace Controller{ if (!globAccX.isReady()){ globAccX.addField("defaultStream", RAX_128STRING); globAccX.addField("systemBoot", RAX_64UINT); + globAccX.addField("sessionMode", RAX_64UINT); + if (!Storage["config"]["sessionMode"]){ + Storage["config"]["sessionMode"] = SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID; + } globAccX.setRCount(1); globAccX.setEndPos(1); globAccX.setReady(); } globAccX.setString("defaultStream", Storage["config"]["defaultStream"].asStringRef()); - globAccX.setInt("systemBoot", systemBoot); + globAccX.setInt("sessionMode", Storage["config"]["sessionMode"].asInt()); globCfg.master = false; // leave the page after closing } } diff --git a/src/input/input.cpp b/src/input/input.cpp index a86fe3cd..0524f55b 100644 --- a/src/input/input.cpp +++ b/src/input/input.cpp @@ -794,7 +794,7 @@ namespace Mist{ void Input::streamMainLoop(){ uint64_t statTimer = 0; uint64_t startTime = Util::bootSecs(); - Comms::Statistics statComm; + Comms::Connections statComm; getNext(); if (thisPacket && !userSelect.count(thisIdx)){ userSelect[thisIdx].reload(streamName, thisIdx, COMM_STATUS_ACTIVE | COMM_STATUS_SOURCE | COMM_STATUS_DONOTTRACK); @@ -820,7 +820,7 @@ namespace Mist{ if (Util::bootSecs() - statTimer > 1){ // Connect to stats for INPUT detection - if (!statComm){statComm.reload();} + if (!statComm){statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID);} if (statComm){ if (!statComm){ config->is_active = false; @@ -829,7 +829,6 @@ namespace Mist{ } uint64_t now = Util::bootSecs(); statComm.setNow(now); - statComm.setCRC(getpid()); statComm.setStream(streamName); statComm.setConnector("INPUT:" + capa["name"].asStringRef()); statComm.setTime(now - startTime); @@ -842,7 +841,7 @@ namespace Mist{ } } - void Input::connStats(Comms::Statistics &statComm){ + void Input::connStats(Comms::Connections &statComm){ statComm.setUp(0); statComm.setDown(streamByteCount()); statComm.setHost(getConnectedBinHost()); @@ -853,7 +852,7 @@ namespace Mist{ uint64_t statTimer = 0; uint64_t startTime = Util::bootSecs(); size_t idx; - Comms::Statistics statComm; + Comms::Connections statComm; DTSC::Meta liveMeta(config->getString("streamname"), false); @@ -985,7 +984,7 @@ namespace Mist{ if (Util::bootSecs() - statTimer > 1){ // Connect to stats for INPUT detection - if (!statComm){statComm.reload();} + if (!statComm){statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID);} if (statComm){ if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ config->is_active = false; @@ -994,7 +993,6 @@ namespace Mist{ } uint64_t now = Util::bootSecs(); statComm.setNow(now); - statComm.setCRC(getpid()); statComm.setStream(streamName); statComm.setConnector("INPUT:" + capa["name"].asStringRef()); statComm.setTime(now - startTime); diff --git a/src/input/input.h b/src/input/input.h index 8d7e8891..ce7686c1 100644 --- a/src/input/input.h +++ b/src/input/input.h @@ -69,7 +69,7 @@ namespace Mist{ virtual void userOnActive(size_t id); virtual void userOnDisconnect(size_t id); virtual void userLeadOut(); - virtual void connStats(Comms::Statistics & statComm); + virtual void connStats(Comms::Connections & statComm); virtual void parseHeader(); bool bufferFrame(size_t track, uint32_t keyNum); diff --git a/src/input/input_rtsp.cpp b/src/input/input_rtsp.cpp index fef613bb..ebb812e1 100644 --- a/src/input/input_rtsp.cpp +++ b/src/input/input_rtsp.cpp @@ -196,7 +196,7 @@ namespace Mist{ } void InputRTSP::streamMainLoop(){ - Comms::Statistics statComm; + Comms::Connections statComm; uint64_t startTime = Util::epoch(); uint64_t lastPing = Util::bootSecs(); uint64_t lastSecs = 0; @@ -210,7 +210,7 @@ namespace Mist{ if (lastSecs != currSecs){ lastSecs = currSecs; // Connect to stats for INPUT detection - statComm.reload(); + statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID); if (statComm){ if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ config->is_active = false; @@ -219,7 +219,6 @@ namespace Mist{ } uint64_t now = Util::bootSecs(); statComm.setNow(now); - statComm.setCRC(getpid()); statComm.setStream(streamName); statComm.setConnector("INPUT:" + capa["name"].asStringRef()); statComm.setUp(tcpCon.dataUp()); diff --git a/src/input/input_sdp.cpp b/src/input/input_sdp.cpp index 62c02aef..3169a836 100644 --- a/src/input/input_sdp.cpp +++ b/src/input/input_sdp.cpp @@ -193,7 +193,7 @@ namespace Mist{ // Updates stats and quits if parsePacket returns false void InputSDP::streamMainLoop(){ - Comms::Statistics statComm; + Comms::Connections statComm; uint64_t startTime = Util::epoch(); uint64_t lastSecs = 0; // Get RTP packets from UDP socket and stop if this fails @@ -202,7 +202,7 @@ namespace Mist{ if (lastSecs != currSecs){ lastSecs = currSecs; // Connect to stats for INPUT detection - statComm.reload(); + statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID); if (statComm){ if (statComm.getStatus() == COMM_STATUS_REQDISCONNECT){ config->is_active = false; @@ -211,7 +211,6 @@ namespace Mist{ } uint64_t now = Util::bootSecs(); statComm.setNow(now); - statComm.setCRC(getpid()); statComm.setStream(streamName); statComm.setConnector("INPUT:" + capa["name"].asStringRef()); statComm.setDown(bytesRead); diff --git a/src/input/input_ts.cpp b/src/input/input_ts.cpp index cfe3f7cd..23311dc0 100644 --- a/src/input/input_ts.cpp +++ b/src/input/input_ts.cpp @@ -527,7 +527,7 @@ namespace Mist{ void inputTS::streamMainLoop(){ meta.removeTrack(tmpIdx); INFO_MSG("Removed temptrack %zu", tmpIdx); - Comms::Statistics statComm; + Comms::Connections statComm; uint64_t downCounter = 0; uint64_t startTime = Util::bootSecs(); uint64_t noDataSince = Util::bootSecs(); @@ -621,7 +621,7 @@ namespace Mist{ // Check for and spawn threads here. if (Util::bootSecs() - threadCheckTimer > 1){ // Connect to stats for INPUT detection - statComm.reload(); + statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID); if (statComm){ if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ config->is_active = false; @@ -630,7 +630,6 @@ namespace Mist{ } uint64_t now = Util::bootSecs(); statComm.setNow(now); - statComm.setCRC(getpid()); statComm.setStream(streamName); statComm.setConnector("INPUT:" + capa["name"].asStringRef()); statComm.setUp(0); diff --git a/src/input/input_tssrt.cpp b/src/input/input_tssrt.cpp index 4219ebbc..8fef6d7d 100644 --- a/src/input/input_tssrt.cpp +++ b/src/input/input_tssrt.cpp @@ -282,7 +282,7 @@ namespace Mist{ void inputTSSRT::setSingular(bool newSingular){singularFlag = newSingular;} - void inputTSSRT::connStats(Comms::Statistics &statComm){ + void inputTSSRT::connStats(Comms::Connections &statComm){ statComm.setUp(srtConn.dataUp()); statComm.setDown(srtConn.dataDown()); statComm.setHost(getConnectedBinHost()); diff --git a/src/input/input_tssrt.h b/src/input/input_tssrt.h index 4f337b48..143174cb 100644 --- a/src/input/input_tssrt.h +++ b/src/input/input_tssrt.h @@ -40,7 +40,7 @@ namespace Mist{ Socket::SRTConnection srtConn; bool singularFlag; - virtual void connStats(Comms::Statistics &statComm); + virtual void connStats(Comms::Connections &statComm); Util::ResizeablePointer rawBuffer; size_t rawIdx; diff --git a/src/output/output.cpp b/src/output/output.cpp index 1fa86eee..50d67549 100644 --- a/src/output/output.cpp +++ b/src/output/output.cpp @@ -7,7 +7,7 @@ #include #include -#include "output.h" +#include "output.h" #include #include #include @@ -92,7 +92,7 @@ namespace Mist{ firstTime = 0; firstPacketTime = 0xFFFFFFFFFFFFFFFFull; lastPacketTime = 0; - crc = getpid(); + sid = ""; parseData = false; wantRequest = true; sought = false; @@ -100,7 +100,7 @@ namespace Mist{ isBlocking = false; needsLookAhead = 0; extraKeepAway = 0; - lastStats = 0; + lastStats = 0xFFFFFFFFFFFFFFFFull; maxSkipAhead = 7500; uaDelay = 10; realTime = 1000; @@ -111,6 +111,7 @@ namespace Mist{ lastPushUpdate = 0; previousFile = ""; currentFile = ""; + sessionMode = 0xFFFFFFFFFFFFFFFFull; lastRecv = Util::bootSecs(); if (myConn){ @@ -211,95 +212,9 @@ namespace Mist{ onFail("Not allowed to play (CONN_PLAY)"); } } - doSync(true); /*LTS-END*/ } - /// If called with force set to true and a USER_NEW trigger enabled, forces a sync immediately. - /// Otherwise, does nothing unless the sync byte is set to 2, in which case it forces a sync as well. - /// May be called recursively because it calls stats() which calls this function. - /// If this happens, the extra calls to the function return instantly. - void Output::doSync(bool force){ - if (!statComm){return;} - if (recursingSync){return;} - recursingSync = true; - if (statComm.getSync() == 2 || force){ - if (getStatsName() == capa["name"].asStringRef() && Triggers::shouldTrigger("USER_NEW", streamName)){ - // sync byte 0 = no sync yet, wait for sync from controller... - char initialSync = 0; - // attempt to load sync status from session cache in shm - { - IPC::semaphore cacheLock(SEM_SESSCACHE, O_RDWR, ACCESSPERMS, 16); - if (cacheLock){cacheLock.wait();} - IPC::sharedPage shmSessions(SHM_SESSIONS, SHM_SESSIONS_SIZE, false, false); - if (shmSessions.mapped){ - char shmEmpty[SHM_SESSIONS_ITEM]; - memset(shmEmpty, 0, SHM_SESSIONS_ITEM); - std::string host; - Socket::hostBytesToStr(statComm.getHost().data(), 16, host); - uint32_t shmOffset = 0; - const std::string &cName = capa["name"].asStringRef(); - while (shmOffset + SHM_SESSIONS_ITEM < SHM_SESSIONS_SIZE){ - // compare crc - if (*(uint32_t*)(shmSessions.mapped + shmOffset) == crc){ - // compare stream name - if (strncmp(shmSessions.mapped + shmOffset + 4, streamName.c_str(), 100) == 0){ - // compare connector - if (strncmp(shmSessions.mapped + shmOffset + 104, cName.c_str(), 20) == 0){ - // compare host - if (strncmp(shmSessions.mapped + shmOffset + 124, host.c_str(), 40) == 0){ - initialSync = shmSessions.mapped[shmOffset + 164]; - HIGH_MSG("Instant-sync from session cache to %u", (unsigned int)initialSync); - break; - } - } - } - } - // stop if we reached the end - if (memcmp(shmSessions.mapped + shmOffset, shmEmpty, SHM_SESSIONS_ITEM) == 0){ - break; - } - shmOffset += SHM_SESSIONS_ITEM; - } - } - if (cacheLock){cacheLock.post();} - } - unsigned int i = 0; - statComm.setSync(initialSync); - // wait max 10 seconds for sync - while ((!statComm.getSync() || statComm.getSync() == 2) && i++ < 100){ - Util::wait(100); - stats(true); - } - HIGH_MSG("USER_NEW sync achieved: %u", statComm.getSync()); - // 1 = check requested (connection is new) - if (statComm.getSync() == 1){ - std::string payload = streamName + "\n" + getConnectedHost() + "\n" + - JSON::Value(crc).asString() + "\n" + capa["name"].asStringRef() + - "\n" + reqUrl + "\n" + statComm.getSessId(); - if (!Triggers::doTrigger("USER_NEW", payload, streamName)){ - onFail("Not allowed to play (USER_NEW)"); - statComm.setSync(100); // 100 = denied - }else{ - statComm.setSync(10); // 10 = accepted - } - } - // 100 = denied - if (statComm.getSync() == 100){onFail("Not allowed to play (USER_NEW cache)");} - if (statComm.getSync() == 0){ - onFail("Not allowed to play (USER_NEW init timeout)", true); - } - if (statComm.getSync() == 2){ - onFail("Not allowed to play (USER_NEW re-init timeout)", true); - } - // anything else = accepted - }else{ - statComm.setSync(10); // auto-accept if no trigger - } - } - recursingSync = false; - } - std::string Output::getConnectedHost(){return myConn.getHost();} std::string Output::getConnectedBinHost(){ @@ -433,10 +348,10 @@ namespace Mist{ //Connect to stats reporting, if not connected already if (!statComm){ - statComm.reload(); + statComm.reload(streamName, getConnectedHost(), sid, capa["name"].asStringRef(), reqUrl, sessionMode); stats(true); } - + //push inputs do not need to wait for stream to be ready for playback if (isPushing()){return;} @@ -986,7 +901,7 @@ namespace Mist{ INFO_MSG("Will split recording every %lld seconds", atoll(targetParams["split"].c_str())); targetParams["nxt-split"] = JSON::Value((int64_t)(seekPos + endRec)).asString(); } - // Duration to record in seconds. Overrides recstop. + // Duration to record in seconds. Oversides recstop. if (targetParams.count("duration")){ long long endRec = atoll(targetParams["duration"].c_str()) * 1000; targetParams["recstop"] = JSON::Value((int64_t)(seekPos + endRec)).asString(); @@ -1301,6 +1216,7 @@ namespace Mist{ /// request URL (if any) /// ~~~~~~~~~~~~~~~ int Output::run(){ + sessionMode = Util::getGlobalConfig("sessionMode").asInt(); /*LTS-START*/ // Connect to file target, if needed if (isFileTarget()){ @@ -1507,6 +1423,8 @@ namespace Mist{ /*LTS-END*/ disconnect(); + stats(true); + userSelect.clear(); myConn.close(); return 0; } @@ -1822,7 +1740,7 @@ namespace Mist{ // also cancel if it has been less than a second since the last update // unless force is set to true uint64_t now = Util::bootSecs(); - if (now == lastStats && !force){return;} + if (now <= lastStats && !force){return;} if (isRecording()){ if(lastPushUpdate == 0){ @@ -1861,13 +1779,17 @@ namespace Mist{ } } - if (!statComm){statComm.reload();} - if (!statComm){return;} + if (!statComm){statComm.reload(streamName, getConnectedHost(), sid, capa["name"].asStringRef(), reqUrl, sessionMode);} + if (!statComm){return;} + if (statComm.getExit()){ + onFail("Shutting down since this session is not allowed to view this stream"); + return; + } lastStats = now; - VERYHIGH_MSG("Writing stats: %s, %s, %u, %" PRIu64 ", %" PRIu64, getConnectedHost().c_str(), streamName.c_str(), - crc & 0xFFFFFFFFu, myConn.dataUp(), myConn.dataDown()); + VERYHIGH_MSG("Writing stats: %s, %s, %s, %" PRIu64 ", %" PRIu64, getConnectedHost().c_str(), streamName.c_str(), + sid.c_str(), myConn.dataUp(), myConn.dataDown()); /*LTS-START*/ if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ onFail("Shutting down on controller request"); @@ -1875,9 +1797,6 @@ namespace Mist{ } /*LTS-END*/ statComm.setNow(now); - statComm.setHost(getConnectedBinHost()); - statComm.setCRC(crc); - statComm.setStream(streamName); statComm.setConnector(getStatsName()); connStats(now, statComm); statComm.setLastSecond(thisPacket ? thisPacket.getTime() : 0); @@ -1887,7 +1806,7 @@ namespace Mist{ // Tag the session with the user agent if (newUA && ((now - myConn.connTime()) >= uaDelay || !myConn) && UA.size()){ std::string APIcall = - "{\"tag_sessid\":{\"" + statComm.getSessId() + "\":" + JSON::string_escape("UA:" + UA) + "}}"; + "{\"tag_sessid\":{\"" + statComm.sessionId + "\":" + JSON::string_escape("UA:" + UA) + "}}"; Socket::UDPConnection uSock; uSock.SetDestination(UDP_API_HOST, UDP_API_PORT); uSock.SendNow(APIcall); @@ -1895,8 +1814,6 @@ namespace Mist{ } /*LTS-END*/ - doSync(); - if (isPushing()){ for (std::map::iterator it = userSelect.begin(); it != userSelect.end(); it++){ if (it->second.getStatus() & COMM_STATUS_REQDISCONNECT){ @@ -1912,7 +1829,7 @@ namespace Mist{ } } - void Output::connStats(uint64_t now, Comms::Statistics &statComm){ + void Output::connStats(uint64_t now, Comms::Connections &statComm){ statComm.setUp(myConn.dataUp()); statComm.setDown(myConn.dataDown()); statComm.setTime(now - myConn.connTime()); diff --git a/src/output/output.h b/src/output/output.h index 84d17482..173b3840 100644 --- a/src/output/output.h +++ b/src/output/output.h @@ -86,7 +86,6 @@ namespace Mist{ std::string hostLookup(std::string ip); bool onList(std::string ip, std::string list); std::string getCountry(std::string ip); - void doSync(bool force = false); /*LTS-END*/ std::map currentPage; void loadPageForKey(size_t trackId, size_t keyNum); @@ -105,6 +104,7 @@ namespace Mist{ bool firstData; uint64_t lastPushUpdate; bool newUA; + protected: // these are to be messed with by child classes virtual bool inlineRestartCapable() const{ return false; @@ -122,15 +122,16 @@ namespace Mist{ virtual std::string getStatsName(); virtual bool hasSessionIDs(){return false;} - virtual void connStats(uint64_t now, Comms::Statistics &statComm); + virtual void connStats(uint64_t now, Comms::Connections &statComm); std::set getSupportedTracks(const std::string &type = "") const; inline virtual bool keepGoing(){return config->is_active && myConn;} - Comms::Statistics statComm; + Comms::Connections statComm; bool isBlocking; ///< If true, indicates that myConn is blocking. - uint32_t crc; ///< Checksum, if any, for usage in the stats. + std::string sid; ///< Random identifier used to split connections into sessions + uint64_t sessionMode; uint64_t nextKeyTime(); // stream delaying variables diff --git a/src/output/output_cmaf.cpp b/src/output/output_cmaf.cpp index 4582a8b6..c2fbc5bc 100644 --- a/src/output/output_cmaf.cpp +++ b/src/output/output_cmaf.cpp @@ -101,7 +101,7 @@ namespace Mist{ } } - void OutCMAF::connStats(uint64_t now, Comms::Statistics &statComm){ + void OutCMAF::connStats(uint64_t now, Comms::Connections &statComm){ // For non-push usage, call usual function. if (!isRecording()){ Output::connStats(now, statComm); diff --git a/src/output/output_cmaf.h b/src/output/output_cmaf.h index efc36511..390549cc 100644 --- a/src/output/output_cmaf.h +++ b/src/output/output_cmaf.h @@ -39,7 +39,7 @@ namespace Mist{ bool isReadyForPlay(); protected: - virtual void connStats(uint64_t now, Comms::Statistics &statComm); + virtual void connStats(uint64_t now, Comms::Connections &statComm); void onTrackEnd(size_t idx); bool hasSessionIDs(){return !config->getBool("mergesessions");} @@ -72,6 +72,7 @@ namespace Mist{ void startPushOut(); void pushNext(); + uint32_t crc; HTTP::URL pushUrl; std::map pushTracks; void setupTrackObject(size_t idx); diff --git a/src/output/output_http.cpp b/src/output/output_http.cpp index 252f069a..f55b974d 100644 --- a/src/output/output_http.cpp +++ b/src/output/output_http.cpp @@ -55,7 +55,6 @@ namespace Mist{ } void HTTPOutput::onFail(const std::string &msg, bool critical){ - INFO_MSG("Failing '%s': %s", H.url.c_str(), msg.c_str()); if (!webSock && !isRecording() && !responded){ H.Clean(); // make sure no parts of old requests are left in any buffers H.SetHeader("Server", APPIDENT); @@ -238,18 +237,6 @@ namespace Mist{ } /*LTS-END*/ if (H.hasHeader("User-Agent")){UA = H.GetHeader("User-Agent");} - if (hasSessionIDs()){ - if (H.GetVar("sessId").size()){ - std::string ua = H.GetVar("sessId"); - crc = checksum::crc32(0, ua.data(), ua.size()); - }else{ - std::string ua = JSON::Value(getpid()).asString(); - crc = checksum::crc32(0, ua.data(), ua.size()); - } - }else{ - std::string mixed_ua = UA + H.GetHeader("X-Playback-Session-Id"); - crc = checksum::crc32(0, mixed_ua.data(), mixed_ua.size()); - } if (H.GetVar("audio") != ""){targetParams["audio"] = H.GetVar("audio");} if (H.GetVar("video") != ""){targetParams["video"] = H.GetVar("video");} @@ -281,6 +268,21 @@ namespace Mist{ realTime = 0; } } + // Get session ID cookie or generate a random one if it wasn't set + if (!sid.size()){ + std::map storage; + const std::string koekjes = H.GetHeader("Cookie"); + HTTP::parseVars(koekjes, storage); + if (storage.count("sid")){ + // Get sid cookie, which is used to divide connections into sessions + sid = storage.at("sid"); + }else{ + // Else generate one + const std::string newSid = UA + JSON::Value(getpid()).asString(); + sid = JSON::Value(checksum::crc32(0, newSid.data(), newSid.size())).asString(); + H.SetHeader("sid", sid.c_str()); + } + } // Handle upgrade to websocket if the output supports it std::string upgradeHeader = H.GetHeader("Upgrade"); Util::stringToLower(upgradeHeader); diff --git a/src/output/output_tssrt.cpp b/src/output/output_tssrt.cpp index db07dc91..dc04b247 100644 --- a/src/output/output_tssrt.cpp +++ b/src/output/output_tssrt.cpp @@ -344,7 +344,7 @@ namespace Mist{ } } - void OutTSSRT::connStats(uint64_t now, Comms::Statistics &statComm){ + void OutTSSRT::connStats(uint64_t now, Comms::Connections &statComm){ if (!srtConn){return;} statComm.setUp(srtConn.dataUp()); statComm.setDown(srtConn.dataDown()); diff --git a/src/output/output_tssrt.h b/src/output/output_tssrt.h index 1423af8d..71c9b72f 100644 --- a/src/output/output_tssrt.h +++ b/src/output/output_tssrt.h @@ -15,7 +15,7 @@ namespace Mist{ bool isReadyForPlay(){return true;} virtual void requestHandler(); protected: - virtual void connStats(uint64_t now, Comms::Statistics &statComm); + virtual void connStats(uint64_t now, Comms::Connections &statComm); virtual std::string getConnectedHost(){return srtConn.remotehost;} virtual std::string getConnectedBinHost(){return srtConn.getBinHost();} diff --git a/src/output/output_webrtc.cpp b/src/output/output_webrtc.cpp index b4289b64..058cd737 100644 --- a/src/output/output_webrtc.cpp +++ b/src/output/output_webrtc.cpp @@ -1015,7 +1015,7 @@ namespace Mist{ } } - void OutWebRTC::connStats(uint64_t now, Comms::Statistics &statComm){ + void OutWebRTC::connStats(uint64_t now, Comms::Connections &statComm){ statComm.setUp(myConn.dataUp()); statComm.setDown(myConn.dataDown()); statComm.setPacketCount(totalPkts); diff --git a/src/output/output_webrtc.h b/src/output/output_webrtc.h index 9c3db580..b2b528c5 100644 --- a/src/output/output_webrtc.h +++ b/src/output/output_webrtc.h @@ -144,7 +144,7 @@ namespace Mist{ void onDTSCConverterHasInitData(const size_t trackID, const std::string &initData); void onRTPPacketizerHasRTPPacket(const char *data, size_t nbytes); void onRTPPacketizerHasRTCPPacket(const char *data, uint32_t nbytes); - virtual void connStats(uint64_t now, Comms::Statistics &statComm); + virtual void connStats(uint64_t now, Comms::Connections &statComm); private: uint64_t lastRecv; diff --git a/src/process/process_exec.cpp b/src/process/process_exec.cpp index 7b775848..34ac53fd 100644 --- a/src/process/process_exec.cpp +++ b/src/process/process_exec.cpp @@ -72,7 +72,7 @@ namespace Mist{ } bool needsLock(){return false;} bool isSingular(){return false;} - void connStats(Comms::Statistics &statComm){ + void connStats(Comms::Connections &statComm){ for (std::map::iterator it = userSelect.begin(); it != userSelect.end(); it++){ if (it->second){it->second.setStatus(COMM_STATUS_DONOTTRACK | it->second.getStatus());} } @@ -117,7 +117,7 @@ namespace Mist{ realTime = 0; OutEBML::sendHeader(); }; - void connStats(uint64_t now, Comms::Statistics &statComm){ + void connStats(uint64_t now, Comms::Connections &statComm){ for (std::map::iterator it = userSelect.begin(); it != userSelect.end(); it++){ if (it->second){it->second.setStatus(COMM_STATUS_DONOTTRACK | it->second.getStatus());} } diff --git a/src/session.cpp b/src/session.cpp new file mode 100644 index 00000000..3865e0ec --- /dev/null +++ b/src/session.cpp @@ -0,0 +1,367 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +// Stats of connections which have closed are added to these global counters +uint64_t globalNow = 0; +uint64_t globalTime = 0; +uint64_t globalDown = 0; +uint64_t globalUp = 0; +uint64_t globalPktcount = 0; +uint64_t globalPktloss = 0; +uint64_t globalPktretrans = 0; +// Counts the duration a connector has been active +std::map connectorCount; +std::map connectorLastActive; +// Set to True when a session gets invalidated, so that we know to run a new USER_NEW trigger +bool forceTrigger = false; +void handleSignal(int signum){ + if (signum == SIGUSR1){ + forceTrigger = true; + } +} + +void userOnActive(uint64_t &connections){ + ++connections; +} + +std::string getEnvWithDefault(const std::string variableName, const std::string defaultValue){ + const char* value = getenv(variableName.c_str()); + if (value){ + unsetenv(variableName.c_str()); + return value; + }else{ + return defaultValue; + } +} + +/// \brief Adds stats of closed connections to global counters +void userOnDisconnect(Comms::Connections & connections, size_t idx){ + std::string thisConnector = connections.getConnector(idx); + if (thisConnector != ""){ + connectorCount[thisConnector] += connections.getTime(idx); + } + globalTime += connections.getTime(idx); + globalDown += connections.getDown(idx); + globalUp += connections.getUp(idx); + globalPktcount += connections.getPacketCount(idx); + globalPktloss += connections.getPacketLostCount(idx); + globalPktretrans += connections.getPacketRetransmitCount(idx); +} + +int main(int argc, char **argv){ + Comms::Connections connections; + Comms::Sessions sessions; + uint64_t lastSeen = Util::bootSecs(); + uint64_t currentConnections = 0; + Util::redirectLogsIfNeeded(); + signal(SIGUSR1, handleSignal); + // Init config and parse arguments + Util::Config config = Util::Config("MistSession"); + JSON::Value option; + + option.null(); + option["arg_num"] = 1; + option["arg"] = "string"; + option["help"] = "Session identifier of the entire session"; + option["default"] = ""; + config.addOption("sessionid", option); + + option.null(); + option["long"] = "sessionmode"; + option["short"] = "m"; + option["arg"] = "integer"; + option["default"] = 0; + config.addOption("sessionmode", option); + + option.null(); + option["long"] = "streamname"; + option["short"] = "n"; + option["arg"] = "string"; + option["default"] = ""; + config.addOption("streamname", option); + + option.null(); + option["long"] = "ip"; + option["short"] = "i"; + option["arg"] = "string"; + option["default"] = ""; + config.addOption("ip", option); + + option.null(); + option["long"] = "sid"; + option["short"] = "s"; + option["arg"] = "string"; + option["default"] = ""; + config.addOption("sid", option); + + option.null(); + option["long"] = "protocol"; + option["short"] = "p"; + option["arg"] = "string"; + option["default"] = ""; + config.addOption("protocol", option); + + option.null(); + option["long"] = "requrl"; + option["short"] = "r"; + option["arg"] = "string"; + option["default"] = ""; + config.addOption("requrl", option); + + config.activate(); + if (!(config.parseArgs(argc, argv))){ + FAIL_MSG("Cannot start a new session due to invalid arguments"); + return 1; + } + + const uint64_t bootTime = Util::getMicros(); + // Get session ID, session mode and other variables used as payload for the USER_NEW and USER_END triggers + const std::string thisStreamName = config.getString("streamname"); + const std::string thisHost = config.getString("ip"); + const std::string thisSid = config.getString("sid"); + const std::string thisProtocol = config.getString("protocol"); + const std::string thisReqUrl = config.getString("requrl"); + const std::string thisSessionId = config.getString("sessionid"); + const uint64_t sessionMode = config.getInteger("sessionmode"); + + if (thisSessionId == "" || thisProtocol == "" || thisStreamName == ""){ + FAIL_MSG("Given the following incomplete arguments: SessionId: '%s', protocol: '%s', stream name: '%s'. Aborting opening a new session", + thisSessionId.c_str(), thisProtocol.c_str(), thisStreamName.c_str()); + return 1; + } + + MEDIUM_MSG("Starting a new session for sessionId '%s'", thisSessionId.c_str()); + if (sessionMode < 1 || sessionMode > 15) { + FAIL_MSG("Invalid session mode of value %lu. Should be larger than 0 and smaller than 16", sessionMode); + return 1; + } + + // Try to lock to ensure we are the only process initialising this session + IPC::semaphore sessionLock; + char semName[NAME_BUFFER_SIZE]; + snprintf(semName, NAME_BUFFER_SIZE, SEM_SESSION, thisSessionId.c_str()); + sessionLock.open(semName, O_CREAT | O_RDWR, ACCESSPERMS, 1); + // If the lock fails, the previous Session process must've failed in spectacular fashion + // It's the Controller's task to clean everything up. When the lock fails, this cleanup hasn't happened yet + if (!sessionLock.tryWaitOneSecond()){ + FAIL_MSG("Session '%s' already locked", thisSessionId.c_str()); + return 1; + } + + // Check if a page already exists for this session ID. If so, quit + { + IPC::sharedPage dataPage; + char userPageName[NAME_BUFFER_SIZE]; + snprintf(userPageName, NAME_BUFFER_SIZE, COMMS_SESSIONS, thisSessionId.c_str()); + dataPage.init(userPageName, 0, false, false); + if (dataPage){ + INFO_MSG("Session '%s' already has a running process", thisSessionId.c_str()); + sessionLock.post(); + return 0; + } + } + + // Claim a spot in shared memory for this session on the global statistics page + sessions.reload(); + if (!sessions){ + FAIL_MSG("Unable to register entry for session '%s' on the stats page", thisSessionId.c_str()); + sessionLock.post(); + return 1; + } + // Open the shared memory page containing statistics for each individual connection in this session + connections.reload(thisStreamName, thisHost, thisSid, thisProtocol, thisReqUrl, sessionMode, true, false); + // Initialise global session data + sessions.setHost(thisHost); + sessions.setSessId(thisSessionId); + sessions.setStream(thisStreamName); + sessionLock.post(); + + // Determine session type, since triggers only get run for viewer type sessions + uint64_t thisType = 0; + if (thisSessionId[0] == 'I'){ + INFO_MSG("Started new input session %s in %lu microseconds", thisSessionId.c_str(), Util::getMicros(bootTime)); + thisType = 1; + } + else if (thisSessionId[0] == 'O'){ + INFO_MSG("Started new output session %s in %lu microseconds", thisSessionId.c_str(), Util::getMicros(bootTime)); + thisType = 2; + } + else{ + INFO_MSG("Started new viewer session %s in %lu microseconds", thisSessionId.c_str(), Util::getMicros(bootTime)); + } + + // Do a USER_NEW trigger if it is defined for this stream + if (!thisType && Triggers::shouldTrigger("USER_NEW", thisStreamName)){ + std::string payload = thisStreamName + "\n" + thisHost + "\n" + + thisSid + "\n" + thisProtocol + + "\n" + thisReqUrl + "\n" + thisSessionId; + if (!Triggers::doTrigger("USER_NEW", payload, thisStreamName)){ + // Mark all connections of this session as finished, since this viewer is not allowed to view this stream + connections.setExit(); + connections.finishAll(); + } + } + + uint64_t lastSecond = 0; + uint64_t now = 0; + uint64_t time = 0; + uint64_t down = 0; + uint64_t up = 0; + uint64_t pktcount = 0; + uint64_t pktloss = 0; + uint64_t pktretrans = 0; + std::string connector = ""; + // Stay active until Mist exits or we no longer have an active connection + while (config.is_active && (currentConnections || Util::bootSecs() - lastSeen <= 10)){ + time = 0; + connector = ""; + down = 0; + up = 0; + pktcount = 0; + pktloss = 0; + pktretrans = 0; + currentConnections = 0; + + // Count active connections + COMM_LOOP(connections, userOnActive(currentConnections), userOnDisconnect(connections, id)); + // Loop through all connection entries to get a summary of statistics + for (uint64_t idx = 0; idx < connections.recordCount(); idx++){ + if (connections.getStatus(idx) == COMM_STATUS_INVALID || connections.getStatus(idx) & COMM_STATUS_DISCONNECT){continue;} + uint64_t thisLastSecond = connections.getLastSecond(idx); + std::string thisConnector = connections.getConnector(idx); + // Save info on the latest active connection separately + if (thisLastSecond > lastSecond){ + lastSecond = thisLastSecond; + now = connections.getNow(idx); + } + connectorLastActive[thisConnector] = thisLastSecond; + // Sum all other variables + time += connections.getTime(idx); + down += connections.getDown(idx); + up += connections.getUp(idx); + pktcount += connections.getPacketCount(idx); + pktloss += connections.getPacketLostCount(idx); + pktretrans += connections.getPacketRetransmitCount(idx); + } + + // Convert connector duration to string + std::stringstream connectorSummary; + bool addDelimiter = false; + connectorSummary << "{"; + for (std::map::iterator it = connectorLastActive.begin(); + it != connectorLastActive.end(); ++it){ + if (lastSecond - it->second < 10000){ + connectorSummary << (addDelimiter ? "," : "") << it->first; + addDelimiter = true; + } + } + connectorSummary << "}"; + + // Write summary to global statistics + sessions.setTime(time + globalTime); + sessions.setDown(down + globalDown); + sessions.setUp(up + globalUp); + sessions.setPacketCount(pktcount + globalPktcount); + sessions.setPacketLostCount(pktloss + globalPktloss); + sessions.setPacketRetransmitCount(pktretrans + globalPktretrans); + sessions.setLastSecond(lastSecond); + sessions.setConnector(connectorSummary.str()); + sessions.setNow(now); + + // Retrigger USER_NEW if a re-sync was requested + if (!thisType && forceTrigger){ + forceTrigger = false; + if (Triggers::shouldTrigger("USER_NEW", thisStreamName)){ + INFO_MSG("Triggering USER_NEW for stream %s", thisStreamName.c_str()); + std::string payload = thisStreamName + "\n" + thisHost + "\n" + + thisSid + "\n" + thisProtocol + + "\n" + thisReqUrl + "\n" + thisSessionId; + if (!Triggers::doTrigger("USER_NEW", payload, thisStreamName)){ + INFO_MSG("USER_NEW rejected stream %s", thisStreamName.c_str()); + connections.setExit(); + connections.finishAll(); + }else{ + INFO_MSG("USER_NEW accepted stream %s", thisStreamName.c_str()); + } + } + } + + // Invalidate connections if the session is marked as invalid + if(connections.getExit()){ + connections.finishAll(); + break; + } + // Remember latest activity so we know when this session ends + if (currentConnections){ + lastSeen = Util::bootSecs(); + } + Util::sleep(1000); + } + + // Trigger USER_END + if (!thisType && Triggers::shouldTrigger("USER_END", thisStreamName)){ + lastSecond = 0; + time = 0; + down = 0; + up = 0; + + // Get a final summary of this session + for (uint64_t idx = 0; idx < connections.recordCount(); idx++){ + if (connections.getStatus(idx) == COMM_STATUS_INVALID || connections.getStatus(idx) & COMM_STATUS_DISCONNECT){continue;} + uint64_t thisLastSecond = connections.getLastSecond(idx); + // Set last second to the latest entry + if (thisLastSecond > lastSecond){ + lastSecond = thisLastSecond; + } + // Count protocol durations across the entire session + std::string thisConnector = connections.getConnector(idx); + if (thisConnector != ""){ + connectorCount[thisConnector] += connections.getTime(idx); + } + // Sum all other variables + time += connections.getTime(idx); + down += connections.getDown(idx); + up += connections.getUp(idx); + } + + // Convert connector duration to string + std::stringstream connectorSummary; + bool addDelimiter = false; + connectorSummary << "{"; + for (std::map::iterator it = connectorCount.begin(); + it != connectorCount.end(); ++it){ + connectorSummary << (addDelimiter ? "," : "") << it->first << ":" << it->second; + addDelimiter = true; + } + connectorSummary << "}"; + + const uint64_t duration = lastSecond - (bootTime / 1000); + std::stringstream summary; + summary << thisSessionId << "\n" + << thisStreamName << "\n" + << connectorSummary.str() << "\n" + << thisHost << "\n" + << duration << "\n" + << up << "\n" + << down << "\n" + << sessions.getTags(); + Triggers::doTrigger("USER_END", summary.str(), thisStreamName); + } + + if (!thisType && connections.getExit()){ + WARN_MSG("Session %s has been invalidated since it is not allowed to view stream %s", thisSessionId.c_str(), thisStreamName.c_str()); + uint64_t sleepStart = Util::bootSecs(); + // Keep session invalidated for 10 minutes, or until the session stops + while (config.is_active && sleepStart - Util::bootSecs() < 600){ + Util::sleep(1000); + } + } + INFO_MSG("Shutting down session %s", thisSessionId.c_str()); + return 0; +} -- 2.25.1 From 8ac486b815cac711c434422a1b0955486c28bb68 Mon Sep 17 00:00:00 2001 From: Marco van Dijk Date: Wed, 16 Mar 2022 13:46:14 +0100 Subject: [PATCH 27/38] Completed new sessions system Co-authored-by: Thulinma --- lib/comms.cpp | 206 ++++++++---- lib/comms.h | 36 +- lib/defines.h | 2 +- lib/hls_support.cpp | 12 +- lib/http_parser.cpp | 6 +- lib/http_parser.h | 2 +- lib/socket.cpp | 10 + lib/socket.h | 1 + lib/websocket.cpp | 22 +- lib/websocket.h | 2 +- src/controller/controller.cpp | 18 + src/controller/controller_api.cpp | 11 +- src/controller/controller_statistics.cpp | 375 +++++++++++++-------- src/controller/controller_statistics.h | 31 +- src/controller/controller_storage.cpp | 28 +- src/controller/controller_storage.h | 1 + src/input/input.cpp | 21 +- src/input/input_rtsp.cpp | 3 +- src/input/input_sdp.cpp | 3 +- src/input/input_ts.cpp | 3 +- src/input/input_tsrist.cpp | 2 +- src/input/input_tsrist.h | 2 +- src/output/output.cpp | 40 ++- src/output/output.h | 3 +- src/output/output_cmaf.cpp | 27 +- src/output/output_hls.cpp | 28 +- src/output/output_http.cpp | 51 ++- src/output/output_http_internal.cpp | 179 +++++----- src/output/output_http_internal.h | 8 +- src/output/output_sdp.cpp | 12 + src/output/output_sdp.h | 2 + src/output/output_ts.cpp | 12 + src/output/output_ts.h | 2 + src/output/output_tsrist.cpp | 16 +- src/output/output_tsrist.h | 4 +- src/session.cpp | 410 +++++++++++++---------- 36 files changed, 981 insertions(+), 610 deletions(-) diff --git a/lib/comms.cpp b/lib/comms.cpp index 85f1e6d7..14faa980 100644 --- a/lib/comms.cpp +++ b/lib/comms.cpp @@ -3,6 +3,7 @@ #include "comms.h" #include "defines.h" #include "encode.h" +#include "stream.h" #include "procs.h" #include "timing.h" #include @@ -10,6 +11,34 @@ #include "config.h" namespace Comms{ + uint8_t sessionViewerMode = SESS_BUNDLE_DEFAULT_VIEWER; + uint8_t sessionInputMode = SESS_BUNDLE_DEFAULT_OTHER; + uint8_t sessionOutputMode = SESS_BUNDLE_DEFAULT_OTHER; + uint8_t sessionUnspecifiedMode = 0; + uint8_t sessionStreamInfoMode = SESS_DEFAULT_STREAM_INFO_MODE; + uint8_t tknMode = SESS_TKN_DEFAULT_MODE; + + /// \brief Refreshes the session configuration if the last update was more than 5 seconds ago + void sessionConfigCache(){ + static uint64_t lastUpdate = 0; + if (Util::bootSecs() > lastUpdate + 5){ + VERYHIGH_MSG("Updating session config"); + JSON::Value tmpVal = Util::getGlobalConfig("sessionViewerMode"); + if (!tmpVal.isNull()){ sessionViewerMode = tmpVal.asInt(); } + tmpVal = Util::getGlobalConfig("sessionInputMode"); + if (!tmpVal.isNull()){ sessionInputMode = tmpVal.asInt(); } + tmpVal = Util::getGlobalConfig("sessionOutputMode"); + if (!tmpVal.isNull()){ sessionOutputMode = tmpVal.asInt(); } + tmpVal = Util::getGlobalConfig("sessionUnspecifiedMode"); + if (!tmpVal.isNull()){ sessionUnspecifiedMode = tmpVal.asInt(); } + tmpVal = Util::getGlobalConfig("sessionStreamInfoMode"); + if (!tmpVal.isNull()){ sessionStreamInfoMode = tmpVal.asInt(); } + tmpVal = Util::getGlobalConfig("tknMode"); + if (!tmpVal.isNull()){ tknMode = tmpVal.asInt(); } + lastUpdate = Util::bootSecs(); + } + } + Comms::Comms(){ index = INVALID_RECORD_INDEX; currentSize = 0; @@ -17,7 +46,7 @@ namespace Comms{ } Comms::~Comms(){ - if (index != INVALID_RECORD_INDEX){ + if (index != INVALID_RECORD_INDEX && status){ setStatus(COMM_STATUS_DISCONNECT | getStatus()); } if (master){ @@ -123,6 +152,10 @@ namespace Comms{ return; } dataAccX = Util::RelAccX(dataPage.mapped); + if (dataAccX.isExit()){ + dataPage.close(); + return; + } fieldAccess(); if (index == INVALID_RECORD_INDEX || reIssue){ size_t reqCount = dataAccX.getRCount(); @@ -170,19 +203,30 @@ namespace Comms{ void Sessions::addFields(){ Connections::addFields(); + dataAccX.addField("tags", RAX_STRING, 512); dataAccX.addField("sessid", RAX_STRING, 80); } void Sessions::nullFields(){ Connections::nullFields(); setSessId(""); + setTags(""); } void Sessions::fieldAccess(){ Connections::fieldAccess(); + tags = dataAccX.getFieldAccX("tags"); sessId = dataAccX.getFieldAccX("sessid"); } + std::string Sessions::getTags() const{return tags.string(index);} + std::string Sessions::getTags(size_t idx) const{return (master ? tags.string(idx) : 0);} + void Sessions::setTags(std::string _sid){tags.set(_sid, index);} + void Sessions::setTags(std::string _sid, size_t idx){ + if (!master){return;} + tags.set(_sid, idx); + } + Users::Users() : Comms(){} Users::Users(const Users &rhs) : Comms(){ @@ -251,31 +295,60 @@ namespace Comms{ keyNum.set(_keyNum, idx); } + + + void Connections::reload(const std::string & sessId, bool _master, bool reIssue){ + // Open SEM_SESSION + if(!sem){ + char semName[NAME_BUFFER_SIZE]; + snprintf(semName, NAME_BUFFER_SIZE, SEM_SESSION, sessId.c_str()); + sem.open(semName, O_RDWR, ACCESSPERMS, 1); + if (!sem){return;} + } + char userPageName[NAME_BUFFER_SIZE]; + snprintf(userPageName, NAME_BUFFER_SIZE, COMMS_SESSIONS, sessId.c_str()); + Comms::reload(userPageName, COMMS_SESSIONS_INITSIZE, _master, reIssue); + } + /// \brief Claims a spot on the connections page for the input/output which calls this function /// Starts the MistSession binary for each session, which handles the statistics /// and the USER_NEW and USER_END triggers /// \param streamName: Name of the stream the input is providing or an output is making available to viewers /// \param ip: IP address of the viewer which wants to access streamName. For inputs this value can be set to any value - /// \param sid: Session ID given by the player or randomly generated + /// \param tkn: Session token given by the player or randomly generated /// \param protocol: Protocol currently in use for this connection - /// \param sessionMode: Determines how a viewer session is defined: - // If set to 0, all connections with the same viewer IP and stream name are bundled. - // If set to 1, all connections with the same viewer IP and player ID are bundled. - // If set to 2, all connections with the same player ID and stream name are bundled. - // If set to 3, all connections with the same viewer IP, player ID and stream name are bundled. /// \param _master: If True, we are reading from this page. If False, we are writing (to our entry) on this page /// \param reIssue: If True, claim a new entry on this page - void Connections::reload(std::string streamName, std::string ip, std::string sid, std::string protocol, std::string reqUrl, uint64_t sessionMode, bool _master, bool reIssue){ - if (sessionMode == 0xFFFFFFFFFFFFFFFFull){ - FAIL_MSG("The session mode was not initialised properly. Assuming default behaviour of bundling by viewer IP, stream name and player id"); - sessionMode = SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID; - } + void Connections::reload(const std::string & streamName, const std::string & ip, const std::string & tkn, const std::string & protocol, const std::string & reqUrl, bool _master, bool reIssue){ + initialTkn = tkn; + uint8_t sessMode = sessionViewerMode; // Generate a unique session ID for each viewer, input or output - sessionId = generateSession(streamName, ip, sid, protocol, sessionMode); if (protocol.size() >= 6 && protocol.substr(0, 6) == "INPUT:"){ - sessionId = "I" + sessionId; + sessMode = sessionInputMode; + sessionId = "I" + generateSession(streamName, ip, tkn, protocol, sessMode); }else if (protocol.size() >= 7 && protocol.substr(0, 7) == "OUTPUT:"){ - sessionId = "O" + sessionId; + sessMode = sessionOutputMode; + sessionId = "O" + generateSession(streamName, ip, tkn, protocol, sessMode); + }else{ + // If the session only contains the HTTP connector, check sessionStreamInfoMode + if (protocol.size() == 4 && protocol == "HTTP"){ + if (sessionStreamInfoMode == SESS_HTTP_AS_VIEWER){ + sessionId = generateSession(streamName, ip, tkn, protocol, sessMode); + }else if (sessionStreamInfoMode == SESS_HTTP_AS_OUTPUT){ + sessMode = sessionOutputMode; + sessionId = "O" + generateSession(streamName, ip, tkn, protocol, sessMode); + }else if (sessionStreamInfoMode == SESS_HTTP_DISABLED){ + return; + }else if (sessionStreamInfoMode == SESS_HTTP_AS_UNSPECIFIED){ + // Set sessMode to include all variables when determining the session ID + sessMode = sessionUnspecifiedMode; + sessionId = "U" + generateSession(streamName, ip, tkn, protocol, sessMode); + }else{ + sessionId = generateSession(streamName, ip, tkn, protocol, sessMode); + } + }else{ + sessionId = generateSession(streamName, ip, tkn, protocol, sessMode); + } } char userPageName[NAME_BUFFER_SIZE]; snprintf(userPageName, NAME_BUFFER_SIZE, COMMS_SESSIONS, sessionId.c_str()); @@ -283,36 +356,59 @@ namespace Comms{ if (!_master){ dataPage.init(userPageName, 0, false, false); if (!dataPage){ + std::string host; + Socket::hostBytesToStr(ip.data(), 16, host); pid_t thisPid; std::deque args; args.push_back(Util::getMyPath() + "MistSession"); args.push_back(sessionId); - args.push_back("--sessionmode"); - args.push_back(JSON::Value(sessionMode).asString()); - args.push_back("--streamname"); - args.push_back(streamName); - args.push_back("--ip"); - args.push_back(ip); - args.push_back("--sid"); - args.push_back(sid); - args.push_back("--protocol"); - args.push_back(protocol); - args.push_back("--requrl"); - args.push_back(reqUrl); + + // First bit defines whether to include stream name + if (sessMode & 0x08){ + args.push_back("--streamname"); + args.push_back(streamName); + }else{ + setenv("SESSION_STREAM", streamName.c_str(), 1); + } + // Second bit defines whether to include viewer ip + if (sessMode & 0x04){ + args.push_back("--ip"); + args.push_back(host); + }else{ + setenv("SESSION_IP", host.c_str(), 1); + } + // Third bit defines whether to include tkn + if (sessMode & 0x02){ + args.push_back("--tkn"); + args.push_back(tkn); + }else{ + setenv("SESSION_TKN", tkn.c_str(), 1); + } + // Fourth bit defines whether to include protocol + if (sessMode & 0x01){ + args.push_back("--protocol"); + args.push_back(protocol); + }else{ + setenv("SESSION_PROTOCOL", protocol.c_str(), 1); + } + setenv("SESSION_REQURL", reqUrl.c_str(), 1); int err = fileno(stderr); thisPid = Util::Procs::StartPiped(args, 0, 0, &err); Util::Procs::forget(thisPid); - HIGH_MSG("Spawned new session executeable (pid %u) for sessionId '%s', corresponding to host %s and stream %s", thisPid, sessionId.c_str(), ip.c_str(), streamName.c_str()); + unsetenv("SESSION_STREAM"); + unsetenv("SESSION_IP"); + unsetenv("SESSION_TKN"); + unsetenv("SESSION_PROTOCOL"); + unsetenv("SESSION_REQURL"); } } - // Open SEM_SESSION - if(!sem){ - char semName[NAME_BUFFER_SIZE]; - snprintf(semName, NAME_BUFFER_SIZE, SEM_SESSION, sessionId.c_str()); - sem.open(semName, O_RDWR, ACCESSPERMS, 1); + reload(sessionId, _master, reIssue); + if (index != INVALID_RECORD_INDEX){ + setConnector(protocol); + setHost(ip); + setStream(streamName); + VERYHIGH_MSG("Reloading connection. Claimed record %lu", index); } - Comms::reload(userPageName, COMMS_SESSIONS_INITSIZE, _master, reIssue); - VERYHIGH_MSG("Reloading connection. Claimed record %lu", index); } /// \brief Marks the data page as closed, so that we longer write any new data to is @@ -341,7 +437,6 @@ namespace Comms{ dataAccX.addField("host", RAX_RAW, 16); dataAccX.addField("stream", RAX_STRING, 100); dataAccX.addField("connector", RAX_STRING, 20); - dataAccX.addField("tags", RAX_STRING, 512); dataAccX.addField("pktcount", RAX_64UINT); dataAccX.addField("pktloss", RAX_64UINT); dataAccX.addField("pktretrans", RAX_64UINT); @@ -349,7 +444,6 @@ namespace Comms{ void Connections::nullFields(){ Comms::nullFields(); - setTags(""); setConnector(""); setStream(""); setHost(""); @@ -373,7 +467,6 @@ namespace Comms{ host = dataAccX.getFieldAccX("host"); stream = dataAccX.getFieldAccX("stream"); connector = dataAccX.getFieldAccX("connector"); - tags = dataAccX.getFieldAccX("tags"); pktcount = dataAccX.getFieldAccX("pktcount"); pktloss = dataAccX.getFieldAccX("pktloss"); pktretrans = dataAccX.getFieldAccX("pktretrans"); @@ -461,14 +554,6 @@ namespace Comms{ return false; } - std::string Connections::getTags() const{return tags.string(index);} - std::string Connections::getTags(size_t idx) const{return (master ? tags.string(idx) : 0);} - void Connections::setTags(std::string _sid){tags.set(_sid, index);} - void Connections::setTags(std::string _sid, size_t idx){ - if (!master){return;} - tags.set(_sid, idx); - } - uint64_t Connections::getPacketCount() const{return pktcount.uint(index);} uint64_t Connections::getPacketCount(size_t idx) const{ return (master ? pktcount.uint(idx) : 0); @@ -501,31 +586,32 @@ namespace Comms{ /// \brief Generates a session ID which is unique per viewer /// \return generated session ID as string - std::string Connections::generateSession(std::string streamName, std::string ip, std::string sid, std::string connector, uint64_t sessionMode){ + std::string Connections::generateSession(const std::string & streamName, const std::string & ip, const std::string & tkn, const std::string & connector, uint64_t sessionMode){ std::string concat; + std::string debugMsg = "Generating session id based on"; // First bit defines whether to include stream name - if (sessionMode > 7){ + if (sessionMode & 0x08){ concat += streamName; - sessionMode -= 8; + debugMsg += " stream name '" + streamName + "'"; } // Second bit defines whether to include viewer ip - if (sessionMode > 3){ + if (sessionMode & 0x04){ concat += ip; - sessionMode -= 4; + std::string ipHex; + Socket::hostBytesToStr(ip.c_str(), ip.size(), ipHex); + debugMsg += " IP '" + ipHex + "'"; } - // Third bit defines whether to include player ip - if (sessionMode > 1){ - concat += sid; - sessionMode -= 2; + // Third bit defines whether to include client-side session token + if (sessionMode & 0x02){ + concat += tkn; + debugMsg += " session token '" + tkn + "'"; } // Fourth bit defines whether to include protocol - if (sessionMode == 1){ + if (sessionMode & 0x01){ concat += connector; - sessionMode = 0; - } - if (sessionMode > 0){ - WARN_MSG("Could not resolve session mode of value %lu", sessionMode); + debugMsg += " protocol '" + connector + "'"; } + VERYHIGH_MSG("%s", debugMsg.c_str()); return Secure::sha256(concat.c_str(), concat.length()); } }// namespace Comms diff --git a/lib/comms.h b/lib/comms.h index 9a5c0ea9..ec36dcb0 100644 --- a/lib/comms.h +++ b/lib/comms.h @@ -9,13 +9,21 @@ #define COMM_STATUS_REQDISCONNECT 0x10 #define COMM_STATUS_ACTIVE 0x1 #define COMM_STATUS_INVALID 0x0 +#define SESS_BUNDLE_DEFAULT_VIEWER 14 +#define SESS_BUNDLE_DEFAULT_OTHER 15 +#define SESS_DEFAULT_STREAM_INFO_MODE 1 +#define SESS_HTTP_AS_VIEWER 1 +#define SESS_HTTP_AS_OUTPUT 2 +#define SESS_HTTP_DISABLED 3 +#define SESS_HTTP_AS_UNSPECIFIED 4 +#define SESS_TKN_DEFAULT_MODE 15 #define COMM_LOOP(comm, onActive, onDisconnect) \ {\ for (size_t id = 0; id < comm.recordCount(); id++){\ if (comm.getStatus(id) == COMM_STATUS_INVALID){continue;}\ - if (!Util::Procs::isRunning(comm.getPid(id))){\ + if (!(comm.getStatus(id) & COMM_STATUS_DISCONNECT) && comm.getPid(id) && !Util::Procs::isRunning(comm.getPid(id))){\ comm.setStatus(COMM_STATUS_DISCONNECT | comm.getStatus(id), id);\ }\ onActive;\ @@ -27,6 +35,14 @@ } namespace Comms{ + extern uint8_t sessionViewerMode; + extern uint8_t sessionInputMode; + extern uint8_t sessionOutputMode; + extern uint8_t sessionUnspecifiedMode; + extern uint8_t sessionStreamInfoMode; + extern uint8_t tknMode; + void sessionConfigCache(); + class Comms{ public: Comms(); @@ -66,11 +82,13 @@ namespace Comms{ class Connections : public Comms{ public: - void reload(std::string streamName, std::string ip, std::string sid, std::string protocol, std::string reqUrl, uint64_t sessionMode, bool _master = false, bool reIssue = false); + void reload(const std::string & streamName, const std::string & ip, const std::string & tkn, const std::string & protocol, const std::string & reqUrl, bool _master = false, bool reIssue = false); + void reload(const std::string & sessId, bool _master = false, bool reIssue = false); void unload(); operator bool() const{return dataPage.mapped && (master || index != INVALID_RECORD_INDEX);} - std::string generateSession(std::string streamName, std::string ip, std::string sid, std::string connector, uint64_t sessionMode); + std::string generateSession(const std::string & streamName, const std::string & ip, const std::string & tkn, const std::string & connector, uint64_t sessionMode); std::string sessionId; + std::string initialTkn; void setExit(); bool getExit(); @@ -79,6 +97,8 @@ namespace Comms{ virtual void nullFields(); virtual void fieldAccess(); + const std::string & getTkn() const{return initialTkn;} + uint64_t getNow() const; uint64_t getNow(size_t idx) const; void setNow(uint64_t _now); @@ -120,11 +140,6 @@ namespace Comms{ void setConnector(std::string _connector, size_t idx); bool hasConnector(size_t idx, std::string protocol); - std::string getTags() const; - std::string getTags(size_t idx) const; - void setTags(std::string _sid); - void setTags(std::string _sid, size_t idx); - uint64_t getPacketCount() const; uint64_t getPacketCount(size_t idx) const; void setPacketCount(uint64_t _count); @@ -197,5 +212,10 @@ namespace Comms{ virtual void addFields(); virtual void nullFields(); virtual void fieldAccess(); + + std::string getTags() const; + std::string getTags(size_t idx) const; + void setTags(std::string _sid); + void setTags(std::string _sid, size_t idx); }; }// namespace Comms diff --git a/lib/defines.h b/lib/defines.h index 203b75db..d08d1d04 100644 --- a/lib/defines.h +++ b/lib/defines.h @@ -231,7 +231,7 @@ static inline void show_stackframe(){} #define SEM_TRACKLIST "/MstTRKS%s" //%s stream name #define SEM_SESSION "MstSess%s" #define SEM_SESSCACHE "/MstSessCacheLock" -#define SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID 14 +#define SESS_TIMEOUT 600 // Session timeout in seconds #define SHM_CAPA "MstCapa" #define SHM_PROTO "MstProt" #define SHM_PROXY "MstProx" diff --git a/lib/hls_support.cpp b/lib/hls_support.cpp index c311a7ac..130e691d 100644 --- a/lib/hls_support.cpp +++ b/lib/hls_support.cpp @@ -274,7 +274,7 @@ namespace HLS{ if (trackData.mediaFormat == ".ts"){return;} result << "#EXT-X-MAP:URI=\"" << trackData.urlPrefix << "init" << trackData.mediaFormat; - if (trackData.sessionId.size()){result << "?sessId=" << trackData.sessionId;} + if (trackData.sessionId.size()){result << "?tkn=" << trackData.sessionId;} result << "\"\r\n"; } @@ -327,7 +327,7 @@ namespace HLS{ result << "?msn=" << fragData.currentFrag; result << "&mTrack=" << trackData.timingTrackId; result << "&dur=" << fragData.duration; - if (trackData.sessionId.size()){result << "&sessId=" << trackData.sessionId;} + if (trackData.sessionId.size()){result << "&tkn=" << trackData.sessionId;} result << "\r\n"; } @@ -341,7 +341,7 @@ namespace HLS{ result << "?msn=" << fragData.currentFrag; result << "&mTrack=" << trackData.timingTrackId; result << "&dur=" << duration; - if (trackData.sessionId.size()){result << "&sessId=" << trackData.sessionId;} + if (trackData.sessionId.size()){result << "&tkn=" << trackData.sessionId;} result << "\""; // NOTE: INDEPENDENT tags, specified ONLY for VIDEO tracks, indicate the first partial fragment @@ -448,7 +448,7 @@ namespace HLS{ result << "?msn=" << fragData.currentFrag - 1; result << "&mTrack=" << trackData.timingTrackId; result << "&dur=" << partDurationMaxMs; - if (trackData.sessionId.size()){result << "&sessId=" << trackData.sessionId;} + if (trackData.sessionId.size()){result << "&tkn=" << trackData.sessionId;} result << "\"\r\n"; } @@ -509,7 +509,7 @@ namespace HLS{ result << ",NAME=\"" << name << "\",URI=\"" << trackId << "/index.m3u8"; result << "?mTrack=" << masterData.mainTrack; result << "&iMsn=" << iFrag; - if (masterData.hasSessId){result << "&sessId=" << masterData.sessId;} + if (masterData.sessId.size()){result << "&tkn=" << masterData.sessId;} if (masterData.noLLHLS){result << "&llhls=0";} result << "\"\r\n"; } @@ -529,7 +529,7 @@ namespace HLS{ result << "/index.m3u8"; result << "?mTrack=" << masterData.mainTrack; result << "&iMsn=" << iFrag; - if (masterData.hasSessId){result << "&sessId=" << masterData.sessId;} + if (masterData.sessId.size()){result << "&tkn=" << masterData.sessId;} if (masterData.noLLHLS){result << "&llhls=0";} result << "\r\n"; } diff --git a/lib/http_parser.cpp b/lib/http_parser.cpp index c08e3d56..113fdb51 100644 --- a/lib/http_parser.cpp +++ b/lib/http_parser.cpp @@ -742,13 +742,13 @@ bool HTTP::Parser::parse(std::string &HTTPbuffer, Util::DataCallback &cb){ /// HTTP variable parser to std::map structure. /// Reads variables from data, decodes and stores them to storage. -void HTTP::parseVars(const std::string &data, std::map &storage){ +void HTTP::parseVars(const std::string &data, std::map &storage, const std::string & separator){ std::string varname; std::string varval; // position where a part starts (e.g. after &) size_t pos = 0; while (pos < data.length()){ - size_t nextpos = data.find('&', pos); + size_t nextpos = data.find(separator, pos); if (nextpos == std::string::npos){nextpos = data.length();} size_t eq_pos = data.find('=', pos); if (eq_pos < nextpos){ @@ -769,7 +769,7 @@ void HTTP::parseVars(const std::string &data, std::map break; } // erase & - pos = nextpos + 1; + pos = nextpos + separator.size(); } } diff --git a/lib/http_parser.h b/lib/http_parser.h index 9c843f20..a5528df3 100644 --- a/lib/http_parser.h +++ b/lib/http_parser.h @@ -14,7 +14,7 @@ namespace HTTP{ /// HTTP variable parser to std::map structure. /// Reads variables from data, decodes and stores them to storage. - void parseVars(const std::string &data, std::map &storage); + void parseVars(const std::string &data, std::map &storage, const std::string & separator = "&"); /// Simple class for reading and writing HTTP 1.0 and 1.1. class Parser : public Util::DataCallback{ diff --git a/lib/socket.cpp b/lib/socket.cpp index 6d63af00..47b2b6bd 100644 --- a/lib/socket.cpp +++ b/lib/socket.cpp @@ -166,6 +166,8 @@ bool Socket::isBinAddress(const std::string &binAddr, std::string addr){ /// Converts the given address with optional subnet to binary IPv6 form. /// Returns 16 bytes of address, followed by 1 byte of subnet bits, zero or more times. std::string Socket::getBinForms(std::string addr){ + // Check for empty address + if (!addr.size()){return std::string(17, (char)0);} // Check if we need to do prefix matching uint8_t prefixLen = 128; if (addr.find('/') != std::string::npos){ @@ -1796,6 +1798,14 @@ void Socket::UDPConnection::GetDestination(std::string &destIp, uint32_t &port){ FAIL_MSG("Could not get destination for UDP socket"); }// Socket::UDPConnection GetDestination +/// Gets the properties of the receiving end of this UDP socket. +/// This will be the receiving end for all SendNow calls. +std::string Socket::UDPConnection::getBinDestination(){ + std::string binList = getIPv6BinAddr(*(sockaddr_in6*)destAddr); + if (binList.size() < 16){ return std::string("\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000", 16); } + return binList.substr(0, 16); +}// Socket::UDPConnection GetDestination + /// Returns the port number of the receiving end of this socket. /// Returns 0 on error. uint32_t Socket::UDPConnection::getDestPort() const{ diff --git a/lib/socket.h b/lib/socket.h index a85fbf39..b369ac84 100644 --- a/lib/socket.h +++ b/lib/socket.h @@ -215,6 +215,7 @@ namespace Socket{ void setBlocking(bool blocking); void SetDestination(std::string hostname, uint32_t port); void GetDestination(std::string &hostname, uint32_t &port); + std::string getBinDestination(); const void * getDestAddr(){return destAddr;} size_t getDestAddrLen(){return destAddr_size;} std::string getBoundAddress(); diff --git a/lib/websocket.cpp b/lib/websocket.cpp index 05703b55..c82b4cba 100644 --- a/lib/websocket.cpp +++ b/lib/websocket.cpp @@ -73,31 +73,31 @@ namespace HTTP{ } /// Takes an incoming HTTP::Parser request for a Websocket, and turns it into one. - Websocket::Websocket(Socket::Connection &c, HTTP::Parser &h) : C(c){ + Websocket::Websocket(Socket::Connection &c, const HTTP::Parser &req, HTTP::Parser &resp) : C(c){ frameType = 0; maskOut = false; - std::string connHeader = h.GetHeader("Connection"); + std::string connHeader = req.GetHeader("Connection"); Util::stringToLower(connHeader); if (connHeader.find("upgrade") == std::string::npos){ FAIL_MSG("Could not negotiate websocket, connection header incorrect (%s).", connHeader.c_str()); C.close(); return; } - std::string upgradeHeader = h.GetHeader("Upgrade"); + std::string upgradeHeader = req.GetHeader("Upgrade"); Util::stringToLower(upgradeHeader); if (upgradeHeader != "websocket"){ FAIL_MSG("Could not negotiate websocket, upgrade header incorrect (%s).", upgradeHeader.c_str()); C.close(); return; } - if (h.GetHeader("Sec-WebSocket-Version") != "13"){ + if (req.GetHeader("Sec-WebSocket-Version") != "13"){ FAIL_MSG("Could not negotiate websocket, version incorrect (%s).", - h.GetHeader("Sec-WebSocket-Version").c_str()); + req.GetHeader("Sec-WebSocket-Version").c_str()); C.close(); return; } #ifdef SSL - std::string client_key = h.GetHeader("Sec-WebSocket-Key"); + std::string client_key = req.GetHeader("Sec-WebSocket-Key"); if (!client_key.size()){ FAIL_MSG("Could not negotiate websocket, missing key!"); C.close(); @@ -105,15 +105,13 @@ namespace HTTP{ } #endif - h.Clean(); - h.setCORSHeaders(); - h.SetHeader("Upgrade", "websocket"); - h.SetHeader("Connection", "Upgrade"); + resp.SetHeader("Upgrade", "websocket"); + resp.SetHeader("Connection", "Upgrade"); #ifdef SSL - h.SetHeader("Sec-WebSocket-Accept", calculateKeyAccept(client_key)); + resp.SetHeader("Sec-WebSocket-Accept", calculateKeyAccept(client_key)); #endif // H.SetHeader("Sec-WebSocket-Protocol", "json"); - h.SendResponse("101", "Websocket away!", C); + resp.SendResponse("101", "Websocket away!", C); } /// Loops calling readFrame until the connection is closed, sleeping in between reads if needed. diff --git a/lib/websocket.h b/lib/websocket.h index 819f18c9..07861a19 100644 --- a/lib/websocket.h +++ b/lib/websocket.h @@ -7,7 +7,7 @@ namespace HTTP{ class Websocket{ public: - Websocket(Socket::Connection &c, HTTP::Parser &h); + Websocket(Socket::Connection &c, const HTTP::Parser &req, HTTP::Parser &resp); Websocket(Socket::Connection &c, const HTTP::URL & url, std::map * headers = 0); Websocket(Socket::Connection &c, bool client); operator bool() const; diff --git a/src/controller/controller.cpp b/src/controller/controller.cpp index 9e17ca6b..972c7ed7 100644 --- a/src/controller/controller.cpp +++ b/src/controller/controller.cpp @@ -309,6 +309,24 @@ int main_loop(int argc, char **argv){ Controller::Storage["config"]["prometheus"] = Controller::conf.getString("prometheus"); Controller::Storage["config"]["accesslog"] = Controller::conf.getString("accesslog"); Controller::normalizeTrustedProxies(Controller::Storage["config"]["trustedproxy"]); + if (!Controller::Storage["config"]["sessionViewerMode"]){ + Controller::Storage["config"]["sessionViewerMode"] = SESS_BUNDLE_DEFAULT_VIEWER; + } + if (!Controller::Storage["config"]["sessionInputMode"]){ + Controller::Storage["config"]["sessionInputMode"] = SESS_BUNDLE_DEFAULT_OTHER; + } + if (!Controller::Storage["config"]["sessionOutputMode"]){ + Controller::Storage["config"]["sessionOutputMode"] = SESS_BUNDLE_DEFAULT_OTHER; + } + if (!Controller::Storage["config"]["sessionUnspecifiedMode"]){ + Controller::Storage["config"]["sessionUnspecifiedMode"] = 0; + } + if (!Controller::Storage["config"]["sessionStreamInfoMode"]){ + Controller::Storage["config"]["sessionStreamInfoMode"] = SESS_DEFAULT_STREAM_INFO_MODE; + } + if (!Controller::Storage["config"].isMember("tknMode")){ + Controller::Storage["config"]["tknMode"] = SESS_TKN_DEFAULT_MODE; + } Controller::prometheus = Controller::Storage["config"]["prometheus"].asStringRef(); Controller::accesslog = Controller::Storage["config"]["accesslog"].asStringRef(); Controller::writeConfig(); diff --git a/src/controller/controller_api.cpp b/src/controller/controller_api.cpp index 02491065..4e935aba 100644 --- a/src/controller/controller_api.cpp +++ b/src/controller/controller_api.cpp @@ -188,7 +188,9 @@ void Controller::handleWebSocket(HTTP::Parser &H, Socket::Connection &C){ std::string logs = H.GetVar("logs"); std::string accs = H.GetVar("accs"); bool doStreams = H.GetVar("streams").size(); - HTTP::Websocket W(C, H); + HTTP::Parser req = H; + H.Clean(); + HTTP::Websocket W(C, req, H); if (!W){return;} IPC::sharedPage shmLogs(SHM_STATE_LOGS, 1024 * 1024); @@ -594,7 +596,12 @@ void Controller::handleAPICommands(JSON::Value &Request, JSON::Value &Response){ out["prometheus"] = in["prometheus"]; Controller::prometheus = out["prometheus"].asStringRef(); } - if (in.isMember("sessionMode")){out["sessionMode"] = in["sessionMode"];} + if (in.isMember("sessionViewerMode")){out["sessionViewerMode"] = in["sessionViewerMode"];} + if (in.isMember("sessionInputMode")){out["sessionInputMode"] = in["sessionInputMode"];} + if (in.isMember("sessionOutputMode")){out["sessionOutputMode"] = in["sessionOutputMode"];} + if (in.isMember("sessionUnspecifiedMode")){out["sessionUnspecifiedMode"] = in["sessionUnspecifiedMode"];} + if (in.isMember("sessionStreamInfoMode")){out["sessionStreamInfoMode"] = in["sessionStreamInfoMode"];} + if (in.isMember("tknMode")){out["tknMode"] = in["tknMode"];} if (in.isMember("defaultStream")){out["defaultStream"] = in["defaultStream"];} if (in.isMember("location") && in["location"].isObject()){ out["location"]["lat"] = in["location"]["lat"].asDouble(); diff --git a/src/controller/controller_statistics.cpp b/src/controller/controller_statistics.cpp index 6e51f4ee..ffb89897 100644 --- a/src/controller/controller_statistics.cpp +++ b/src/controller/controller_statistics.cpp @@ -58,6 +58,11 @@ static uint64_t cpu_use = 0; char noBWCountMatches[1717]; uint64_t bwLimit = 128 * 1024 * 1024; // gigabit default limit +const char nullAddress[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; +static Controller::statLog emptyLogEntry = {0, 0, 0, 0, 0, 0 ,0 ,0, "", nullAddress, ""}; +bool notEmpty(const Controller::statLog & dta){ + return dta.time || dta.firstActive || dta.lastSecond || dta.down || dta.up || dta.streamName.size() || dta.connectors.size(); +} // For server-wide totals. Local to this file only. struct streamTotals{ @@ -66,9 +71,11 @@ struct streamTotals{ uint64_t inputs; uint64_t outputs; uint64_t viewers; + uint64_t unspecified; uint64_t currIns; uint64_t currOuts; uint64_t currViews; + uint64_t currUnspecified; uint8_t status; uint64_t viewSeconds; uint64_t packSent; @@ -84,6 +91,7 @@ static uint64_t servDownBytes = 0; static uint64_t servUpOtherBytes = 0; static uint64_t servDownOtherBytes = 0; static uint64_t servInputs = 0; +static uint64_t servUnspecified = 0; static uint64_t servOutputs = 0; static uint64_t servViewers = 0; static uint64_t servSeconds = 0; @@ -95,17 +103,19 @@ static uint64_t viewSecondsTotal = 0; // Mapping of streamName -> summary of stream-wide statistics static std::map streamStats; -// If sessId does not exist yet in streamStats, create and init an entry for it -static void createEmptyStatsIfNeeded(const std::string & sessId){ - if (streamStats.count(sessId)){return;} - streamTotals & sT = streamStats[sessId]; +// If streamName does not exist yet in streamStats, create and init an entry for it +static void createEmptyStatsIfNeeded(const std::string & streamName){ + if (streamStats.count(streamName)){return;} + streamTotals & sT = streamStats[streamName]; sT.upBytes = 0; sT.downBytes = 0; sT.inputs = 0; sT.outputs = 0; sT.viewers = 0; + sT.unspecified = 0; sT.currIns = 0; sT.currOuts = 0; + sT.currUnspecified = 0; sT.currViews = 0; sT.status = 0; sT.viewSeconds = 0; @@ -335,15 +345,23 @@ void Controller::SharedMemStats(void *config){ it->second.currViews = 0; it->second.currIns = 0; it->second.currOuts = 0; + it->second.currUnspecified = 0; } } // wipe old statistics and set session type counters if (sessions.size()){ std::list mustWipe; - uint64_t cutOffPoint = Util::bootSecs() - STAT_CUTOFF; + // Ensure cutOffPoint is either time of boot or 10 minutes ago, whichever is closer. + // Prevents wrapping around to high values close to system boot time. + uint64_t cutOffPoint = Util::bootSecs(); + if (cutOffPoint > STAT_CUTOFF){ + cutOffPoint -= STAT_CUTOFF; + }else{ + cutOffPoint = 0; + } for (std::map::iterator it = sessions.begin(); it != sessions.end(); it++){ // This part handles ending sessions, keeping them in cache for now - if (it->second.getEnd() < cutOffPoint && it->second.newestDataPoint() < cutOffPoint){ + if (it->second.getEnd() < cutOffPoint){ viewSecondsTotal += it->second.getConnTime(); mustWipe.push_back(it->first); // Don't count this session as a viewer @@ -353,19 +371,24 @@ void Controller::SharedMemStats(void *config){ switch (it->second.getSessType()){ case SESS_UNSET: break; case SESS_VIEWER: - if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){ - streamStats[it->first].currViews++; + if (it->second.hasDataFor(tOut)){ + streamStats[it->second.getStreamName()].currViews++; } servSeconds += it->second.getConnTime(); break; case SESS_INPUT: - if (it->second.hasDataFor(tIn) && it->second.isViewerOn(tIn)){ - streamStats[it->first].currIns++; + if (it->second.hasDataFor(tIn)){ + streamStats[it->second.getStreamName()].currIns++; } break; case SESS_OUTPUT: - if (it->second.hasDataFor(tOut) && it->second.isViewerOn(tOut)){ - streamStats[it->first].currOuts++; + if (it->second.hasDataFor(tOut)){ + streamStats[it->second.getStreamName()].currOuts++; + } + break; + case SESS_UNSPECIFIED: + if (it->second.hasDataFor(tOut)){ + streamStats[it->second.getStreamName()].currUnspecified++; } break; } @@ -406,6 +429,7 @@ void Controller::SharedMemStats(void *config){ strmStats->setInt("viewers", it->second.currViews, strmPos); strmStats->setInt("inputs", it->second.currIns, strmPos); strmStats->setInt("outputs", it->second.currOuts, strmPos); + strmStats->setInt("unspecified", it->second.currUnspecified, strmPos); ++strmPos; } } @@ -489,47 +513,46 @@ void Controller::killConnections(std::string sessId){ /// Updates the given active connection with new stats data. void Controller::statSession::update(uint64_t index, Comms::Sessions &statComm){ - if (host == ""){ - Socket::hostBytesToStr(statComm.getHost(index).data(), 16, host); - } - if (streamName == ""){ - streamName = statComm.getStream(index); - } - if (curConnector == ""){ - curConnector = statComm.getConnector(index); - } if (sessId == ""){ sessId = statComm.getSessId(index); } - // Export tags to session - if (tags.size()){ - std::stringstream tagStream; - for (std::set::iterator it = tags.begin(); it != tags.end(); ++it){ - tagStream << "[" << *it << "]"; + + if (sessionType == SESS_UNSET){ + if (sessId[0] == 'I'){ + sessionType = SESS_INPUT; + }else if (sessId[0] == 'O'){ + sessionType = SESS_OUTPUT; + }else if (sessId[0] == 'U'){ + sessionType = SESS_UNSPECIFIED; + }else{ + sessionType = SESS_VIEWER; } - statComm.setTags(tagStream.str(), index); } + uint64_t prevNow = curData.log.size() ? curData.log.rbegin()->first : 0; + // only parse last received data, if newer + if (prevNow > statComm.getNow(index)){return;}; long long prevDown = getDown(); long long prevUp = getUp(); uint64_t prevPktSent = getPktCount(); uint64_t prevPktLost = getPktLost(); uint64_t prevPktRetrans = getPktRetransmit(); + uint64_t prevFirstActive = getFirstActive(); + curData.update(statComm, index); - // store timestamp of first received data, if older - if (firstSec > statComm.getNow(index)){firstSec = statComm.getNow(index);} - uint64_t secIncr = 0; - // store timestamp of last received data, if newer - if (statComm.getNow(index) > lastSec){ - lastSec = statComm.getNow(index); - if (!tracked){ - tracked = true; - firstActive = firstSec; - }else{ - secIncr = (statComm.getNow(index) - lastSec); + const std::string& streamName = getStreamName(); + // Export tags to session + if (tags.size()){ + std::stringstream tagStream; + for (std::set::iterator it = tags.begin(); it != tags.end(); ++it){ + tagStream << "[" << *it << "]"; } - lastSec = statComm.getNow(index); + statComm.setTags(tagStream.str(), index); + } else { + statComm.setTags("", index); } + + uint64_t secIncr = prevFirstActive ? (statComm.getNow(index) - prevNow) : 0; long long currDown = getDown(); long long currUp = getUp(); uint64_t currPktSent = getPktCount(); @@ -537,7 +560,7 @@ void Controller::statSession::update(uint64_t index, Comms::Sessions &statComm){ uint64_t currPktRetrans = getPktRetransmit(); if (currUp - prevUp < 0 || currDown - prevDown < 0){ INFO_MSG("Negative data usage! %lldu/%lldd (u%lld->%lld) in %s over %s, #%" PRIu64, currUp - prevUp, - currDown - prevDown, prevUp, currUp, streamName.c_str(), curConnector.c_str(), index); + currDown - prevDown, prevUp, currUp, streamName.c_str(), curData.log.rbegin()->second.connectors.c_str(), index); }else{ if (!noBWCount){ size_t bwMatchOffset = 0; @@ -567,40 +590,38 @@ void Controller::statSession::update(uint64_t index, Comms::Sessions &statComm){ servPackRetrans += currPktRetrans - prevPktRetrans; } } - if (sessionType == SESS_UNSET){ - if (curConnector.size() >= 5 && curConnector.substr(0, 5) == "INPUT"){ - ++servInputs; - createEmptyStatsIfNeeded(streamName); - streamStats[streamName].inputs++; - streamStats[streamName].currIns++; - sessionType = SESS_INPUT; - }else if (curConnector.size() >= 6 && curConnector.substr(0, 6) == "OUTPUT"){ - ++servOutputs; - createEmptyStatsIfNeeded(streamName); - streamStats[streamName].outputs++; - streamStats[streamName].currOuts++; - sessionType = SESS_OUTPUT; - }else{ - ++servViewers; - createEmptyStatsIfNeeded(streamName); - streamStats[streamName].viewers++; - streamStats[streamName].currViews++; - sessionType = SESS_VIEWER; + if (!prevFirstActive && streamName.size()){ + createEmptyStatsIfNeeded(streamName); + switch(sessionType){ + case SESS_INPUT: + ++servInputs; + streamStats[streamName].inputs++; + break; + case SESS_OUTPUT: + ++servOutputs; + streamStats[streamName].outputs++; + break; + case SESS_VIEWER: + ++servViewers; + streamStats[streamName].viewers++; + break; + case SESS_UNSPECIFIED: + ++servUnspecified; + streamStats[streamName].unspecified++; + break; + case SESS_UNSET: + break; } } // Only count connections that are countable if (noBWCount != 2){ - if (!streamName.size() || streamName[0] == 0){ - if (streamStats.count(streamName)){streamStats.erase(streamName);} - }else{ - createEmptyStatsIfNeeded(streamName); - streamStats[streamName].upBytes += currUp - prevUp; - streamStats[streamName].downBytes += currDown - prevDown; - streamStats[streamName].packSent += currPktSent - prevPktSent; - streamStats[streamName].packLoss += currPktLost - prevPktLost; - streamStats[streamName].packRetrans += currPktRetrans - prevPktRetrans; - if (sessionType == SESS_VIEWER){streamStats[streamName].viewSeconds += secIncr;} - } + createEmptyStatsIfNeeded(streamName); + streamStats[streamName].upBytes += currUp - prevUp; + streamStats[streamName].downBytes += currDown - prevDown; + streamStats[streamName].packSent += currPktSent - prevPktSent; + streamStats[streamName].packLoss += currPktLost - prevPktLost; + streamStats[streamName].packRetrans += currPktRetrans - prevPktRetrans; + if (sessionType == SESS_VIEWER){streamStats[streamName].viewSeconds += secIncr;} } } @@ -608,21 +629,10 @@ Controller::sessType Controller::statSession::getSessType(){ return sessionType; } -Controller::statSession::~statSession(){ - if (!tracked){return;} - switch (sessionType){ - case SESS_INPUT: - if (streamStats.count(streamName) && streamStats[streamName].currIns){streamStats[streamName].currIns--;} - break; - case SESS_OUTPUT: - if (streamStats.count(streamName) && streamStats[streamName].currOuts){streamStats[streamName].currOuts--;} - break; - case SESS_VIEWER: - if (streamStats.count(streamName) && streamStats[streamName].currViews){streamStats[streamName].currViews--;} - break; - default: break; - } - uint64_t duration = lastSec - firstActive; +/// Ends the currently active session by inserting a null datapoint one second after the last datapoint +void Controller::statSession::finish(){ + if (!getFirstActive()){return;} + uint64_t duration = getEnd() - getFirstActive(); if (duration < 1){duration = 1;} std::stringstream tagStream; if (tags.size()){ @@ -630,6 +640,9 @@ Controller::statSession::~statSession(){ tagStream << "[" << *it << "]"; } } + const std::string& streamName = getStreamName(); + const std::string& curConnector = getConnectors(); + const std::string& host = getStrHost(); Controller::logAccess(sessId, streamName, curConnector, host, duration, getUp(), getDown(), tagStream.str()); if (Controller::accesslog.size()){ @@ -668,74 +681,99 @@ Controller::statSession::~statSession(){ } } } - tracked = false; - firstActive = 0; - firstSec = 0xFFFFFFFFFFFFFFFFull; - lastSec = 0; - sessionType = SESS_UNSET; + tags.clear(); + // Insert null datapoint + curData.log[curData.log.rbegin()->first + 1] = emptyLogEntry; } /// Constructs an empty session Controller::statSession::statSession(){ - firstActive = 0; - tracked = false; - firstSec = 0xFFFFFFFFFFFFFFFFull; - lastSec = 0; sessionType = SESS_UNSET; noBWCount = 0; - streamName = ""; - host = ""; - curConnector = ""; sessId = ""; } /// Returns the first measured timestamp in this session. uint64_t Controller::statSession::getStart(){ - return firstSec; + if (!curData.log.size()){return 0;} + return curData.log.begin()->first; } /// Returns the last measured timestamp in this session. uint64_t Controller::statSession::getEnd(){ - return lastSec; + if (!curData.log.size()){return 0;} + return curData.log.rbegin()->first; } /// Returns true if there is data for this session at timestamp t. bool Controller::statSession::hasDataFor(uint64_t t){ - if (lastSec < t){return false;} - if (firstSec > t){return false;} if (curData.hasDataFor(t)){return true;} return false; } -/// Returns true if this session should count as a viewer on the given timestamp. -bool Controller::statSession::isViewerOn(uint64_t t){ - return getUp(t) + getDown(t); +const std::string& Controller::statSession::getSessId(){ + return sessId; +} + +uint64_t Controller::statSession::getFirstActive(){ + if (curData.log.size()){ + return curData.log.rbegin()->second.firstActive; + } + return 0; +} + +const std::string& Controller::statSession::getStreamName(uint64_t t){ + if (curData.hasDataFor(t)){ + return curData.getDataFor(t).streamName; + } + return emptyLogEntry.streamName; } -std::string Controller::statSession::getStreamName(){ - return streamName; +const std::string& Controller::statSession::getStreamName(){ + if (curData.log.size()){ + return curData.log.rbegin()->second.streamName; + } + return emptyLogEntry.streamName; } -std::string Controller::statSession::getHost(){ +std::string Controller::statSession::getStrHost(uint64_t t){ + std::string host; + Socket::hostBytesToStr(getHost(t).data(), 16, host); return host; } -std::string Controller::statSession::getSessId(){ - return sessId; +std::string Controller::statSession::getStrHost(){ + std::string host; + Socket::hostBytesToStr(getHost().data(), 16, host); + return host; +} + +const std::string& Controller::statSession::getHost(uint64_t t){ + if (curData.hasDataFor(t)){ + return curData.getDataFor(t).host; + } + return emptyLogEntry.host; } -std::string Controller::statSession::getCurrentProtocols(){ - return curConnector; +const std::string& Controller::statSession::getHost(){ + if (curData.log.size()){ + return curData.log.rbegin()->second.host; + } + return emptyLogEntry.host; } -/// Returns true if this session should be considered connected -uint64_t Controller::statSession::newestDataPoint(){ - return lastSec; +const std::string& Controller::statSession::getConnectors(uint64_t t){ + if (curData.hasDataFor(t)){ + return curData.getDataFor(t).connectors; + } + return emptyLogEntry.connectors; } -/// Returns true if this session has started (tracked == true) but not yet ended (log entry written) -bool Controller::statSession::isTracked(){ - return tracked; +const std::string& Controller::statSession::getConnectors(){ + if (curData.log.size()){ + return curData.log.rbegin()->second.connectors; + } + return emptyLogEntry.connectors; } /// Returns the cumulative connected time for this session at timestamp t. @@ -842,7 +880,7 @@ uint64_t Controller::statSession::getPktRetransmit(){ /// Returns the cumulative downloaded bytes per second for this session at timestamp t. uint64_t Controller::statSession::getBpsDown(uint64_t t){ uint64_t aTime = t - 5; - if (aTime < firstSec){aTime = firstSec;} + if (aTime < curData.log.begin()->first){aTime = curData.log.begin()->first;} if (t <= aTime){return 0;} uint64_t valA = getDown(aTime); uint64_t valB = getDown(t); @@ -852,7 +890,7 @@ uint64_t Controller::statSession::getBpsDown(uint64_t t){ /// Returns the cumulative uploaded bytes per second for this session at timestamp t. uint64_t Controller::statSession::getBpsUp(uint64_t t){ uint64_t aTime = t - 5; - if (aTime < firstSec){aTime = firstSec;} + if (aTime < curData.log.begin()->first){aTime = curData.log.begin()->first;} if (t <= aTime){return 0;} uint64_t valA = getUp(aTime); uint64_t valB = getUp(t); @@ -867,17 +905,8 @@ bool Controller::statStorage::hasDataFor(unsigned long long t){ /// Returns a reference to the most current data available at timestamp t. Controller::statLog &Controller::statStorage::getDataFor(unsigned long long t){ - static statLog empty; if (!log.size()){ - empty.time = 0; - empty.lastSecond = 0; - empty.down = 0; - empty.up = 0; - empty.pktCount = 0; - empty.pktLost = 0; - empty.pktRetransmit = 0; - empty.connectors = ""; - return empty; + return emptyLogEntry; } std::map::iterator it = log.upper_bound(t); if (it != log.begin()){it--;} @@ -889,6 +918,11 @@ Controller::statLog &Controller::statStorage::getDataFor(unsigned long long t){ void Controller::statStorage::update(Comms::Sessions &statComm, size_t index){ statLog tmp; tmp.time = statComm.getTime(index); + if (!log.size() || !log.rbegin()->second.firstActive){ + tmp.firstActive = statComm.getNow(index); + } else{ + tmp.firstActive = log.rbegin()->second.firstActive; + } tmp.lastSecond = statComm.getLastSecond(index); tmp.down = statComm.getDown(index); tmp.up = statComm.getUp(index); @@ -896,9 +930,19 @@ void Controller::statStorage::update(Comms::Sessions &statComm, size_t index){ tmp.pktLost = statComm.getPacketLostCount(index); tmp.pktRetransmit = statComm.getPacketRetransmitCount(index); tmp.connectors = statComm.getConnector(index); + tmp.streamName = statComm.getStream(index); + tmp.host = statComm.getHost(index); log[statComm.getNow(index)] = tmp; // wipe data older than STAT_CUTOFF seconds - while (log.size() && log.begin()->first < Util::bootSecs() - STAT_CUTOFF){log.erase(log.begin());} + // Ensure cutOffPoint is either time of boot or 10 minutes ago, whichever is closer. + // Prevents wrapping around to high values close to system boot time. + uint64_t cutOffPoint = Util::bootSecs(); + if (cutOffPoint > STAT_CUTOFF){ + cutOffPoint -= STAT_CUTOFF; + }else{ + cutOffPoint = 0; + } + while (log.size() && log.begin()->first < cutOffPoint){log.erase(log.begin());} } void Controller::statLeadIn(){ @@ -915,6 +959,7 @@ void Controller::statOnActive(size_t id){ void Controller::statOnDisconnect(size_t id){ // Check to see if cleanup is required (when a Session binary fails) const std::string thisSessionId = statComm.getSessId(id); + sessions[thisSessionId].finish(); // Try to lock to see if the session crashed during boot IPC::semaphore sessionLock; char semName[NAME_BUFFER_SIZE]; @@ -932,7 +977,7 @@ void Controller::statOnDisconnect(size_t id){ if(dataPage){ // Session likely crashed while it was running dataPage.init(userPageName, 1, true); - FAIL_MSG("Session '%s' got canceled unexpectedly. Hoovering up the left overs...", thisSessionId.c_str()); + FAIL_MSG("Session '%s' got cancelled unexpectedly. Cleaning up the leftovers...", thisSessionId.c_str()); } // Finally remove the session lock which was created on bootup of the session sessionLock.unlink(); @@ -999,13 +1044,23 @@ void Controller::fillClients(JSON::Value &req, JSON::Value &rep){ if (req.isMember("time")){reqTime = req["time"].asInt();} // to make sure no nasty timing business takes place, we store the case "now" as a bool. bool now = (reqTime == 0); - //if greater than current bootsecs, assume unix time and subtract epoch from it - if (reqTime > (int64_t)epoch - STAT_CUTOFF){reqTime -= (epoch-bSecs);} + //if in the last 600 seconds of unix time (or higher), assume unix time and subtract epoch from it + if (reqTime > (int64_t)epoch - STAT_CUTOFF){reqTime -= Controller::systemBoot/1000;} // add the current time, if negative or zero. if (reqTime < 0){reqTime += bSecs;} - if (reqTime == 0){reqTime = bSecs - STAT_CUTOFF;} + if (reqTime == 0){ + // Ensure cutOffPoint is either time of boot or 10 minutes ago, whichever is closer. + // Prevents wrapping around to high values close to system boot time. + uint64_t cutOffPoint = bSecs; + if (cutOffPoint > STAT_CUTOFF){ + cutOffPoint -= STAT_CUTOFF; + }else{ + cutOffPoint = 0; + } + reqTime = cutOffPoint; + } // at this point, we have the absolute timestamp in bootsecs. - rep["time"] = reqTime + (epoch-bSecs); // fill the absolute timestamp + rep["time"] = reqTime + (Controller::systemBoot/1000); // fill the absolute timestamp unsigned int fields = 0; // next, figure out the fields wanted @@ -1062,13 +1117,14 @@ void Controller::fillClients(JSON::Value &req, JSON::Value &rep){ if (now && reqTime - it->second.getEnd() < 5){time = it->second.getEnd();} // data present and wanted? insert it! if ((it->second.getEnd() >= time && it->second.getStart() <= time) && - (!streams.size() || streams.count(it->second.getStreamName())) && - (!protos.size() || protos.count(it->second.getCurrentProtocols()))){ - if (it->second.hasDataFor(time)){ + (!streams.size() || streams.count(it->second.getStreamName(time))) && + (!protos.size() || protos.count(it->second.getConnectors(time)))){ + const statLog & dta = it->second.curData.getDataFor(time); + if (notEmpty(dta)){ JSON::Value d; - if (fields & STAT_CLI_HOST){d.append(it->second.getHost());} - if (fields & STAT_CLI_STREAM){d.append(it->second.getStreamName());} - if (fields & STAT_CLI_PROTO){d.append(it->second.getCurrentProtocols());} + if (fields & STAT_CLI_HOST){d.append(it->second.getStrHost(time));} + if (fields & STAT_CLI_STREAM){d.append(it->second.getStreamName(time));} + if (fields & STAT_CLI_PROTO){d.append(it->second.getConnectors(time));} if (fields & STAT_CLI_CONNTIME){d.append(it->second.getConnTime(time));} if (fields & STAT_CLI_POSITION){d.append(it->second.getLastSecond(time));} if (fields & STAT_CLI_DOWN){d.append(it->second.getDown(time));} @@ -1252,6 +1308,8 @@ void Controller::fillActive(JSON::Value &req, JSON::Value &rep){ F = it->second.currIns; }else if (j->asStringRef() == "outputs"){ F = it->second.currOuts; + }else if (j->asStringRef() == "unspecified"){ + F = it->second.currUnspecified; }else if (j->asStringRef() == "views"){ F = it->second.viewers; }else if (j->asStringRef() == "viewseconds"){ @@ -1323,6 +1381,7 @@ public: clients = 0; inputs = 0; outputs = 0; + unspecified = 0; downbps = 0; upbps = 0; pktCount = 0; @@ -1334,6 +1393,7 @@ public: case Controller::SESS_VIEWER: clients++; break; case Controller::SESS_INPUT: inputs++; break; case Controller::SESS_OUTPUT: outputs++; break; + case Controller::SESS_UNSPECIFIED: unspecified++; break; default: break; } downbps += down; @@ -1345,6 +1405,7 @@ public: uint64_t clients; uint64_t inputs; uint64_t outputs; + uint64_t unspecified; uint64_t downbps; uint64_t upbps; uint64_t pktCount; @@ -1363,11 +1424,21 @@ void Controller::fillTotals(JSON::Value &req, JSON::Value &rep){ if (req.isMember("start")){reqStart = req["start"].asInt();} if (req.isMember("end")){reqEnd = req["end"].asInt();} //if the reqStart or reqEnd is greater than current bootsecs, assume unix time and subtract epoch from it - if (reqStart > (int64_t)epoch - STAT_CUTOFF){reqStart -= (epoch-bSecs);} - if (reqEnd > (int64_t)epoch - STAT_CUTOFF){reqEnd -= (epoch-bSecs);} + if (reqStart > (int64_t)epoch - STAT_CUTOFF){reqStart -= Controller::systemBoot/1000;} + if (reqEnd > (int64_t)epoch - STAT_CUTOFF){reqEnd -= Controller::systemBoot/1000;} // add the current time, if negative or zero. if (reqStart < 0){reqStart += bSecs;} - if (reqStart == 0){reqStart = bSecs - STAT_CUTOFF;} + if (reqStart == 0){ + // Ensure cutOffPoint is either time of boot or 10 minutes ago, whichever is closer. + // Prevents wrapping around to high values close to system boot time. + uint64_t cutOffPoint = bSecs; + if (cutOffPoint > STAT_CUTOFF){ + cutOffPoint -= STAT_CUTOFF; + }else{ + cutOffPoint = 0; + } + reqStart = cutOffPoint; + } if (reqEnd <= 0){reqEnd += bSecs;} // at this point, reqStart and reqEnd are the absolute timestamp in bootsecs. if (reqEnd < reqStart){reqEnd = reqStart;} @@ -1417,7 +1488,7 @@ void Controller::fillTotals(JSON::Value &req, JSON::Value &rep){ if ((it->second.getEnd() >= (unsigned long long)reqStart || it->second.getStart() <= (unsigned long long)reqEnd) && (!streams.size() || streams.count(it->second.getStreamName())) && - (!protos.size() || protos.count(it->second.getCurrentProtocols()))){ + (!protos.size() || protos.count(it->second.getConnectors()))){ for (unsigned long long i = reqStart; i <= reqEnd; ++i){ if (it->second.hasDataFor(i)){ totalsCount[i].add(it->second.getBpsDown(i), it->second.getBpsUp(i), it->second.getSessType(), it->second.getPktCount(), it->second.getPktLost(), it->second.getPktRetransmit()); @@ -1436,8 +1507,8 @@ void Controller::fillTotals(JSON::Value &req, JSON::Value &rep){ return; } // yay! We have data! - rep["start"] = totalsCount.begin()->first + (epoch-bSecs); - rep["end"] = totalsCount.rbegin()->first + (epoch-bSecs); + rep["start"] = totalsCount.begin()->first + (Controller::systemBoot/1000); + rep["end"] = totalsCount.rbegin()->first + (Controller::systemBoot/1000); rep["data"].null(); rep["interval"].null(); uint64_t prevT = 0; @@ -1506,12 +1577,15 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int uint32_t totViewers = 0; uint32_t totInputs = 0; uint32_t totOutputs = 0; + uint32_t totUnspecified = 0; for (uint64_t idx = 0; idx < statComm.recordCount(); idx++){ if (statComm.getStatus(idx) == COMM_STATUS_INVALID || statComm.getStatus(idx) & COMM_STATUS_DISCONNECT){continue;} const std::string thisSessId = statComm.getSessId(idx); // Count active viewers, inputs, outputs and protocols if (thisSessId[0] == 'I'){ totInputs++; + }else if (thisSessId[0] == 'U'){ + totUnspecified++; }else if (thisSessId[0] == 'O'){ totOutputs++; outputs[statComm.getConnector(idx)]++; @@ -1602,6 +1676,7 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int response << "# TYPE mist_sessions_count counter\n"; response << "mist_sessions_count{sessType=\"viewers\"}" << servViewers << "\n"; response << "mist_sessions_count{sessType=\"incoming\"}" << servInputs << "\n"; + response << "mist_sessions_count{sessType=\"unspecified\"}" << servUnspecified << "\n"; response << "mist_sessions_count{sessType=\"outgoing\"}" << servOutputs << "\n\n"; response << "# HELP mist_bw_total Count of bytes handled since server start, by direction.\n"; @@ -1637,6 +1712,7 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int response << "mist_sessions_total{sessType=\"viewers\"}" << totViewers << "\n"; response << "mist_sessions_total{sessType=\"incoming\"}" << totInputs << "\n"; response << "mist_sessions_total{sessType=\"outgoing\"}" << totOutputs << "\n"; + response << "mist_sessions_total{sessType=\"unspecified\"}" << totUnspecified << "\n"; response << "mist_sessions_total{sessType=\"cached\"}" << sessions.size() << "\n"; response << "\n# HELP mist_viewcount Count of unique viewer sessions since stream start, per " @@ -1656,6 +1732,8 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int << it->second.currIns << "\n"; response << "mist_sessions{stream=\"" << it->first << "\",sessType=\"outgoing\"}" << it->second.currOuts << "\n"; + response << "mist_sessions{stream=\"" << it->first << "\",sessType=\"unspecified\"}" + << it->second.currUnspecified << "\n"; response << "mist_viewcount{stream=\"" << it->first << "\"}" << it->second.viewers << "\n"; response << "mist_viewseconds{stream=\"" << it->first << "\"} " << it->second.viewSeconds << "\n"; response << "mist_bw{stream=\"" << it->first << "\",direction=\"up\"}" << it->second.upBytes << "\n"; @@ -1691,9 +1769,11 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int resp["curr"].append(totViewers); resp["curr"].append(totInputs); resp["curr"].append(totOutputs); + resp["curr"].append(totUnspecified); resp["tot"].append(servViewers); resp["tot"].append(servInputs); resp["tot"].append(servOutputs); + resp["tot"].append(servUnspecified); resp["st"].append(bw_up_total); resp["st"].append(bw_down_total); resp["bw"].append(servUpBytes); @@ -1735,6 +1815,7 @@ void Controller::handlePrometheus(HTTP::Parser &H, Socket::Connection &conn, int resp["streams"][it->first]["curr"].append(it->second.currViews); resp["streams"][it->first]["curr"].append(it->second.currIns); resp["streams"][it->first]["curr"].append(it->second.currOuts); + resp["streams"][it->first]["curr"].append(it->second.currUnspecified); resp["streams"][it->first]["pkts"].append(it->second.packSent); resp["streams"][it->first]["pkts"].append(it->second.packLoss); resp["streams"][it->first]["pkts"].append(it->second.packRetrans); diff --git a/src/controller/controller_statistics.h b/src/controller/controller_statistics.h index f798f811..69605942 100644 --- a/src/controller/controller_statistics.h +++ b/src/controller/controller_statistics.h @@ -28,16 +28,19 @@ namespace Controller{ struct statLog{ uint64_t time; + uint64_t firstActive; uint64_t lastSecond; uint64_t down; uint64_t up; uint64_t pktCount; uint64_t pktLost; uint64_t pktRetransmit; + std::string streamName; + std::string host; std::string connectors; }; - enum sessType{SESS_UNSET = 0, SESS_INPUT, SESS_OUTPUT, SESS_VIEWER}; + enum sessType{SESS_UNSET = 0, SESS_INPUT, SESS_OUTPUT, SESS_VIEWER, SESS_UNSPECIFIED}; class statStorage{ public: @@ -51,34 +54,30 @@ namespace Controller{ /// Allows for moving of connections to another session. class statSession{ private: - uint64_t firstActive; - uint64_t firstSec; - uint64_t lastSec; sessType sessionType; - bool tracked; uint8_t noBWCount; ///< Set to 2 when not to count for external bandwidth - std::string streamName; - std::string host; - std::string curConnector; std::string sessId; public: statSession(); - ~statSession(); + void finish(); statStorage curData; std::set tags; sessType getSessType(); void update(uint64_t index, Comms::Sessions &data); uint64_t getStart(); uint64_t getEnd(); - bool isViewerOn(uint64_t time); - bool isTracked(); bool hasDataFor(uint64_t time); - std::string getStreamName(); - std::string getHost(); - std::string getSessId(); - std::string getCurrentProtocols(); - uint64_t newestDataPoint(); + const std::string& getSessId(); + const std::string& getStreamName(uint64_t t); + const std::string& getStreamName(); + std::string getStrHost(uint64_t t); + std::string getStrHost(); + const std::string& getHost(uint64_t t); + const std::string& getHost(); + const std::string& getConnectors(uint64_t t); + const std::string& getConnectors(); + uint64_t getFirstActive(); uint64_t getConnTime(uint64_t time); uint64_t getConnTime(); uint64_t getLastSecond(uint64_t time); diff --git a/src/controller/controller_storage.cpp b/src/controller/controller_storage.cpp index bdd52893..532ee4fe 100644 --- a/src/controller/controller_storage.cpp +++ b/src/controller/controller_storage.cpp @@ -196,6 +196,7 @@ namespace Controller{ rlxStrm->addField("viewers", RAX_64UINT); rlxStrm->addField("inputs", RAX_64UINT); rlxStrm->addField("outputs", RAX_64UINT); + rlxStrm->addField("unspecified", RAX_64UINT); rlxStrm->setReady(); } rlxStrm->setRCount((1024 * 1024 - rlxStrm->getOffset()) / rlxStrm->getRSize()); @@ -433,12 +434,17 @@ namespace Controller{ // if fields missing, recreate the page if (globAccX.isReady()){ - if(globAccX.getFieldAccX("systemBoot")){ + if(globAccX.getFieldAccX("systemBoot") && globAccX.getInt("systemBoot")){ systemBoot = globAccX.getInt("systemBoot"); } if(!globAccX.getFieldAccX("defaultStream") || !globAccX.getFieldAccX("systemBoot") - || !globAccX.getFieldAccX("sessionMode")){ + || !globAccX.getFieldAccX("sessionViewerMode") + || !globAccX.getFieldAccX("sessionInputMode") + || !globAccX.getFieldAccX("sessionOutputMode") + || !globAccX.getFieldAccX("sessionUnspecifiedMode") + || !globAccX.getFieldAccX("sessionStreamInfoMode") + || !globAccX.getFieldAccX("tknMode")){ globAccX.setReload(); globCfg.master = true; globCfg.close(); @@ -449,16 +455,24 @@ namespace Controller{ if (!globAccX.isReady()){ globAccX.addField("defaultStream", RAX_128STRING); globAccX.addField("systemBoot", RAX_64UINT); - globAccX.addField("sessionMode", RAX_64UINT); - if (!Storage["config"]["sessionMode"]){ - Storage["config"]["sessionMode"] = SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID; - } + globAccX.addField("sessionViewerMode", RAX_64UINT); + globAccX.addField("sessionInputMode", RAX_64UINT); + globAccX.addField("sessionOutputMode", RAX_64UINT); + globAccX.addField("sessionUnspecifiedMode", RAX_64UINT); + globAccX.addField("sessionStreamInfoMode", RAX_64UINT); + globAccX.addField("tknMode", RAX_64UINT); globAccX.setRCount(1); globAccX.setEndPos(1); globAccX.setReady(); } globAccX.setString("defaultStream", Storage["config"]["defaultStream"].asStringRef()); - globAccX.setInt("sessionMode", Storage["config"]["sessionMode"].asInt()); + globAccX.setInt("sessionViewerMode", Storage["config"]["sessionViewerMode"].asInt()); + globAccX.setInt("sessionInputMode", Storage["config"]["sessionInputMode"].asInt()); + globAccX.setInt("sessionOutputMode", Storage["config"]["sessionOutputMode"].asInt()); + globAccX.setInt("sessionUnspecifiedMode", Storage["config"]["sessionUnspecifiedMode"].asInt()); + globAccX.setInt("sessionStreamInfoMode", Storage["config"]["sessionStreamInfoMode"].asInt()); + globAccX.setInt("tknMode", Storage["config"]["tknMode"].asInt()); + globAccX.setInt("systemBoot", systemBoot); globCfg.master = false; // leave the page after closing } } diff --git a/src/controller/controller_storage.h b/src/controller/controller_storage.h index 6339cfd0..974d989b 100644 --- a/src/controller/controller_storage.h +++ b/src/controller/controller_storage.h @@ -16,6 +16,7 @@ namespace Controller{ extern bool isTerminal; ///< True if connected to a terminal and not a log file. extern bool isColorized; ///< True if we colorize the output extern uint64_t logCounter; ///< Count of logged messages since boot + extern uint64_t systemBoot; ///< Unix time in milliseconds of system boot Util::RelAccX *logAccessor(); Util::RelAccX *accesslogAccessor(); diff --git a/src/input/input.cpp b/src/input/input.cpp index 0524f55b..782dc4ef 100644 --- a/src/input/input.cpp +++ b/src/input/input.cpp @@ -502,6 +502,7 @@ namespace Mist{ } int Input::run(){ + Comms::sessionConfigCache(); if (streamStatus){streamStatus.mapped[0] = STRMSTAT_BOOT;} checkHeaderTimes(config->getString("input")); if (needHeader()){ @@ -623,6 +624,8 @@ namespace Mist{ /// ~~~~~~~~~~~~~~~ void Input::serve(){ users.reload(streamName, true); + Comms::Connections statComm; + uint64_t startTime = Util::bootSecs(); if (!M){ // Initialize meta page @@ -636,6 +639,7 @@ namespace Mist{ meta.setSource(config->getString("input")); bool internalOnly = (config->getString("input").find("INTERNAL_ONLY") != std::string::npos); + bool isBuffer = (capa["name"].asStringRef() == "Buffer"); /*LTS-START*/ if (Triggers::shouldTrigger("STREAM_READY", config->getString("streamname"))){ @@ -666,6 +670,18 @@ namespace Mist{ }else{ if (connectedUsers && M.getValidTracks().size()){activityCounter = Util::bootSecs();} } + // Connect to stats for INPUT detection + if (!internalOnly && !isBuffer){ + if (!statComm){statComm.reload(streamName, getConnectedBinHost(), JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "");} + if (statComm){ + uint64_t now = Util::bootSecs(); + statComm.setNow(now); + statComm.setStream(streamName); + statComm.setTime(now - startTime); + statComm.setLastSecond(0); + connStats(statComm); + } + } // if not shutting down, wait 1 second before looping if (config->is_active){Util::wait(INPUT_USER_INTERVAL);} } @@ -820,7 +836,7 @@ namespace Mist{ if (Util::bootSecs() - statTimer > 1){ // Connect to stats for INPUT detection - if (!statComm){statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID);} + if (!statComm){statComm.reload(streamName, getConnectedBinHost(), JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "");} if (statComm){ if (!statComm){ config->is_active = false; @@ -830,7 +846,6 @@ namespace Mist{ uint64_t now = Util::bootSecs(); statComm.setNow(now); statComm.setStream(streamName); - statComm.setConnector("INPUT:" + capa["name"].asStringRef()); statComm.setTime(now - startTime); statComm.setLastSecond(0); connStats(statComm); @@ -984,7 +999,7 @@ namespace Mist{ if (Util::bootSecs() - statTimer > 1){ // Connect to stats for INPUT detection - if (!statComm){statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID);} + if (!statComm){statComm.reload(streamName, getConnectedBinHost(), JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "");} if (statComm){ if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ config->is_active = false; diff --git a/src/input/input_rtsp.cpp b/src/input/input_rtsp.cpp index ebb812e1..25be6e23 100644 --- a/src/input/input_rtsp.cpp +++ b/src/input/input_rtsp.cpp @@ -210,7 +210,7 @@ namespace Mist{ if (lastSecs != currSecs){ lastSecs = currSecs; // Connect to stats for INPUT detection - statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID); + statComm.reload(streamName, getConnectedBinHost(), JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), ""); if (statComm){ if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ config->is_active = false; @@ -225,7 +225,6 @@ namespace Mist{ statComm.setDown(tcpCon.dataDown()); statComm.setTime(now - startTime); statComm.setLastSecond(0); - statComm.setHost(getConnectedBinHost()); } } } diff --git a/src/input/input_sdp.cpp b/src/input/input_sdp.cpp index 3169a836..0b8ddeb5 100644 --- a/src/input/input_sdp.cpp +++ b/src/input/input_sdp.cpp @@ -202,7 +202,7 @@ namespace Mist{ if (lastSecs != currSecs){ lastSecs = currSecs; // Connect to stats for INPUT detection - statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID); + statComm.reload(streamName, getConnectedBinHost(), JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), ""); if (statComm){ if (statComm.getStatus() == COMM_STATUS_REQDISCONNECT){ config->is_active = false; @@ -217,7 +217,6 @@ namespace Mist{ statComm.setUp(bytesUp); statComm.setTime(now - startTime); statComm.setLastSecond(0); - statComm.setHost(getConnectedBinHost()); } } // If the error flag is raised or we are lacking data, try to recover diff --git a/src/input/input_ts.cpp b/src/input/input_ts.cpp index 23311dc0..e405e3b6 100644 --- a/src/input/input_ts.cpp +++ b/src/input/input_ts.cpp @@ -621,7 +621,7 @@ namespace Mist{ // Check for and spawn threads here. if (Util::bootSecs() - threadCheckTimer > 1){ // Connect to stats for INPUT detection - statComm.reload(streamName, "", JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), "", SESS_BUNDLE_STREAMNAME_HOSTNAME_SESSIONID); + statComm.reload(streamName, getConnectedBinHost(), JSON::Value(getpid()).asString(), "INPUT:" + capa["name"].asStringRef(), ""); if (statComm){ if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ config->is_active = false; @@ -636,7 +636,6 @@ namespace Mist{ statComm.setDown(downCounter + tcpCon.dataDown()); statComm.setTime(now - startTime); statComm.setLastSecond(0); - statComm.setHost(getConnectedBinHost()); } std::set activeTracks = liveStream.getActiveTracks(); diff --git a/src/input/input_tsrist.cpp b/src/input/input_tsrist.cpp index 5b95aa9f..df658da1 100644 --- a/src/input/input_tsrist.cpp +++ b/src/input/input_tsrist.cpp @@ -284,7 +284,7 @@ namespace Mist{ } - void inputTSRIST::connStats(Comms::Statistics &statComm){ + void inputTSRIST::connStats(Comms::Connections &statComm){ statComm.setUp(0); statComm.setDown(downBytes); statComm.setHost(getConnectedBinHost()); diff --git a/src/input/input_tsrist.h b/src/input/input_tsrist.h index 731f9b04..a75f6601 100644 --- a/src/input/input_tsrist.h +++ b/src/input/input_tsrist.h @@ -30,7 +30,7 @@ namespace Mist{ int64_t timeStampOffset; uint64_t lastTimeStamp; - virtual void connStats(Comms::Statistics &statComm); + virtual void connStats(Comms::Connections &statComm); struct rist_ctx *receiver_ctx; diff --git a/src/output/output.cpp b/src/output/output.cpp index 50d67549..d525b8ea 100644 --- a/src/output/output.cpp +++ b/src/output/output.cpp @@ -92,7 +92,7 @@ namespace Mist{ firstTime = 0; firstPacketTime = 0xFFFFFFFFFFFFFFFFull; lastPacketTime = 0; - sid = ""; + tkn = ""; parseData = false; wantRequest = true; sought = false; @@ -111,7 +111,6 @@ namespace Mist{ lastPushUpdate = 0; previousFile = ""; currentFile = ""; - sessionMode = 0xFFFFFFFFFFFFFFFFull; lastRecv = Util::bootSecs(); if (myConn){ @@ -230,7 +229,7 @@ namespace Mist{ bool Output::isReadyForPlay(){ // If a protocol does not support any codecs, we assume you know what you're doing if (!capa.isMember("codecs")){return true;} - if (!isInitialized){initialize();} + if (!isInitialized){return false;} meta.reloadReplacedPagesIfNeeded(); if (getSupportedTracks().size()){ size_t minTracks = 2; @@ -277,6 +276,7 @@ namespace Mist{ /// Assumes streamName class member has been set already. /// Will start input if not currently active, calls onFail() if this does not succeed. void Output::reconnect(){ + Comms::sessionConfigCache(); thisPacket.null(); if (config->hasOption("noinput") && config->getBool("noinput")){ Util::sanitizeName(streamName); @@ -347,11 +347,10 @@ namespace Mist{ isInitialized = true; //Connect to stats reporting, if not connected already - if (!statComm){ - statComm.reload(streamName, getConnectedHost(), sid, capa["name"].asStringRef(), reqUrl, sessionMode); - stats(true); - } - + stats(true); + //Abort if the stats code shut us down just now + if (!isInitialized){return;} + //push inputs do not need to wait for stream to be ready for playback if (isPushing()){return;} @@ -1216,7 +1215,7 @@ namespace Mist{ /// request URL (if any) /// ~~~~~~~~~~~~~~~ int Output::run(){ - sessionMode = Util::getGlobalConfig("sessionMode").asInt(); + Comms::sessionConfigCache(); /*LTS-START*/ // Connect to file target, if needed if (isFileTarget()){ @@ -1257,6 +1256,7 @@ namespace Mist{ /*LTS-END*/ DONTEVEN_MSG("MistOut client handler started"); while (keepGoing() && (wantRequest || parseData)){ + Comms::sessionConfigCache(); if (wantRequest){requestHandler();} if (parseData){ if (!isInitialized){ @@ -1779,27 +1779,35 @@ namespace Mist{ } } - if (!statComm){statComm.reload(streamName, getConnectedHost(), sid, capa["name"].asStringRef(), reqUrl, sessionMode);} - if (!statComm){return;} - if (statComm.getExit()){ + // Disable stats for HTTP internal output + if (Comms::sessionStreamInfoMode == SESS_HTTP_DISABLED && capa["name"].asStringRef() == "HTTP"){return;} + + // Set the token to the pid for outputs which do not generate it in the requestHandler + if (!tkn.size()){ tkn = JSON::Value(getpid()).asString(); } + + if (!statComm){ + statComm.reload(streamName, getConnectedBinHost(), tkn, getStatsName(), reqUrl); + } + if (!statComm || statComm.getExit()){ onFail("Shutting down since this session is not allowed to view this stream"); + statComm.unload(); return; - } + } lastStats = now; VERYHIGH_MSG("Writing stats: %s, %s, %s, %" PRIu64 ", %" PRIu64, getConnectedHost().c_str(), streamName.c_str(), - sid.c_str(), myConn.dataUp(), myConn.dataDown()); + tkn.c_str(), myConn.dataUp(), myConn.dataDown()); /*LTS-START*/ if (statComm.getStatus() & COMM_STATUS_REQDISCONNECT){ onFail("Shutting down on controller request"); + statComm.unload(); return; } /*LTS-END*/ statComm.setNow(now); - statComm.setConnector(getStatsName()); connStats(now, statComm); - statComm.setLastSecond(thisPacket ? thisPacket.getTime() : 0); + statComm.setLastSecond(thisPacket ? thisPacket.getTime()/1000 : 0); statComm.setPid(getpid()); /*LTS-START*/ diff --git a/src/output/output.h b/src/output/output.h index 173b3840..441c06ab 100644 --- a/src/output/output.h +++ b/src/output/output.h @@ -130,8 +130,7 @@ namespace Mist{ Comms::Connections statComm; bool isBlocking; ///< If true, indicates that myConn is blocking. - std::string sid; ///< Random identifier used to split connections into sessions - uint64_t sessionMode; + std::string tkn; ///< Random identifier used to split connections into sessions uint64_t nextKeyTime(); // stream delaying variables diff --git a/src/output/output_cmaf.cpp b/src/output/output_cmaf.cpp index c2fbc5bc..eb7c87da 100644 --- a/src/output/output_cmaf.cpp +++ b/src/output/output_cmaf.cpp @@ -222,24 +222,17 @@ namespace Mist{ void OutCMAF::sendHlsMasterManifest(){ selectDefaultTracks(); - std::string sessId = ""; - if (hasSessionIDs()){ - std::string ua = UA + JSON::Value(getpid()).asString(); - crc = checksum::crc32(0, ua.data(), ua.size()); - sessId = JSON::Value(crc).asString(); - } - // check for forced "no low latency" parameter bool noLLHLS = H.GetVar("llhls").size() ? H.GetVar("llhls") == "0" : false; // Populate the struct that will help generate the master playlist const HLS::MasterData masterData ={ - hasSessionIDs(), + false,//hasSessionIDs, unused noLLHLS, hlsMediaFormat == ".ts", getMainSelectedTrack(), H.GetHeader("User-Agent"), - sessId, + (Comms::tknMode & 0x04)?tkn:"", systemBoot, bootMsOffset, }; @@ -261,11 +254,8 @@ namespace Mist{ // Chunkpath & Session ID logic std::string urlPrefix = ""; - std::string sessId = ""; if (config->getString("chunkpath").size()){ urlPrefix = HTTP::URL(config->getString("chunkpath")).link("./" + H.url).link("./").getUrl(); - }else{ - sessId = H.GetVar("sessId"); } // check for forced "no low latency" parameter @@ -279,7 +269,7 @@ namespace Mist{ noLLHLS, hlsMediaFormat, M.getEncryption(requestTid), - sessId, + (Comms::tknMode & 0x04)?tkn:"", timingTid, requestTid, M.biggestFragment(timingTid) / 1000, @@ -346,6 +336,16 @@ namespace Mist{ std::string url = H.url.substr(H.url.find('/', 6) + 1); HTTP::URL req(reqUrl); + + if (tkn.size()){ + if (Comms::tknMode & 0x08){ + const std::string koekjes = H.GetHeader("Cookie"); + std::stringstream cookieHeader; + cookieHeader << "tkn=" << tkn << "; Max-Age=" << SESS_TIMEOUT; + H.SetHeader("Set-Cookie", cookieHeader.str()); + } + } + // Send a dash manifest for any URL with .mpd in the path if (req.getExt() == "mpd"){ sendDashManifest(); @@ -438,6 +438,7 @@ namespace Mist{ H.SendResponse("400", "Bad Request: Could not parse the url", myConn); return; } + std::string headerData = CMAF::keyHeader(M, idx, startTime, targetTime, fragmentIndex, false, false); diff --git a/src/output/output_hls.cpp b/src/output/output_hls.cpp index ad810aa7..112e7ee2 100644 --- a/src/output/output_hls.cpp +++ b/src/output/output_hls.cpp @@ -11,7 +11,7 @@ const std::string hlsMediaFormat = ".ts"; namespace Mist{ bool OutHLS::isReadyForPlay(){ - if (!isInitialized){initialize();} + if (!isInitialized){return false;} meta.reloadReplacedPagesIfNeeded(); if (!M.getValidTracks().size()){return false;} uint32_t mainTrack = M.mainTrack(); @@ -110,25 +110,17 @@ namespace Mist{ ///\return The master playlist file for (LL)HLS. void OutHLS::sendHlsMasterManifest(){ selectDefaultTracks(); - - std::string sessId = ""; - if (hasSessionIDs()){ - std::string ua = UA + JSON::Value(getpid()).asString(); - crc = checksum::crc32(0, ua.data(), ua.size()); - sessId = JSON::Value(crc).asString(); - } - // check for forced "no low latency" parameter bool noLLHLS = H.GetVar("llhls").size() ? H.GetVar("llhls") == "0" : false; // Populate the struct that will help generate the master playlist const HLS::MasterData masterData ={ - hasSessionIDs(), + false,//hasSessionIDs, unused noLLHLS, hlsMediaFormat == ".ts", getMainSelectedTrack(), H.GetHeader("User-Agent"), - sessId, + (Comms::tknMode & 0x04)?tkn:"", systemBoot, bootMsOffset, }; @@ -150,11 +142,8 @@ namespace Mist{ // Chunkpath & Session ID logic std::string urlPrefix = ""; - std::string sessId = ""; if (config->getString("chunkpath").size()){ urlPrefix = HTTP::URL(config->getString("chunkpath")).link("./" + H.url).link("./").getUrl(); - }else{ - sessId = H.GetVar("sessId"); } // check for forced "no low latency" parameter @@ -168,7 +157,7 @@ namespace Mist{ noLLHLS, hlsMediaFormat, M.getEncryption(requestTid), - sessId, + (Comms::tknMode & 0x04)?tkn:"", timingTid, requestTid, M.biggestFragment(timingTid) / 1000, @@ -226,6 +215,15 @@ namespace Mist{ bootMsOffset = 0; if (M.getLive()){bootMsOffset = M.getBootMsOffset();} + if (tkn.size()){ + if (Comms::tknMode & 0x08){ + const std::string koekjes = H.GetHeader("Cookie"); + std::stringstream cookieHeader; + cookieHeader << "tkn=" << tkn << "; Max-Age=" << SESS_TIMEOUT; + H.SetHeader("Set-Cookie", cookieHeader.str()); + } + } + if (H.url == "/crossdomain.xml"){ H.SetHeader("Content-Type", "text/xml"); H.SetHeader("Server", APPIDENT); diff --git a/src/output/output_http.cpp b/src/output/output_http.cpp index f55b974d..ed8fa33b 100644 --- a/src/output/output_http.cpp +++ b/src/output/output_http.cpp @@ -217,7 +217,9 @@ namespace Mist{ myConn.close(); return; } - if (handler != capa["name"].asStringRef() || H.GetVar("stream") != streamName){ + + //Check if we need to change binary and/or reconnect + if (handler != capa["name"].asStringRef() || H.GetVar("stream") != streamName || (statComm && (statComm.getHost() != getConnectedBinHost() || statComm.getTkn() != tkn))){ MEDIUM_MSG("Switching from %s (%s) to %s (%s)", capa["name"].asStringRef().c_str(), streamName.c_str(), handler.c_str(), H.GetVar("stream").c_str()); streamName = H.GetVar("stream"); @@ -268,21 +270,32 @@ namespace Mist{ realTime = 0; } } - // Get session ID cookie or generate a random one if it wasn't set - if (!sid.size()){ + // Read the session token + if (Comms::tknMode & 0x01){ + // Get session token from the request url + if (H.GetVar("tkn") != ""){ + tkn = H.GetVar("tkn"); + } else if (H.GetVar("sid") != ""){ + tkn = H.GetVar("sid"); + } else if (H.GetVar("sessId") != ""){ + tkn = H.GetVar("sessId"); + } + } + if ((Comms::tknMode & 0x02) && !tkn.size()){ + // Get session token from the request cookie std::map storage; const std::string koekjes = H.GetHeader("Cookie"); - HTTP::parseVars(koekjes, storage); - if (storage.count("sid")){ - // Get sid cookie, which is used to divide connections into sessions - sid = storage.at("sid"); - }else{ - // Else generate one - const std::string newSid = UA + JSON::Value(getpid()).asString(); - sid = JSON::Value(checksum::crc32(0, newSid.data(), newSid.size())).asString(); - H.SetHeader("sid", sid.c_str()); + HTTP::parseVars(koekjes, storage, "; "); + if (storage.count("tkn")){ + tkn = storage.at("tkn"); } } + // Generate a session token if it is being sent as a cookie or url parameter and we couldn't read one + if (!tkn.size() && Comms::tknMode > 3){ + const std::string newTkn = UA + JSON::Value(getpid()).asString(); + tkn = JSON::Value(checksum::crc32(0, newTkn.data(), newTkn.size())).asString(); + HIGH_MSG("Generated tkn '%s'", tkn.c_str()); + } // Handle upgrade to websocket if the output supports it std::string upgradeHeader = H.GetHeader("Upgrade"); Util::stringToLower(upgradeHeader); @@ -290,7 +303,9 @@ namespace Mist{ INFO_MSG("Switching to Websocket mode"); setBlocking(false); preWebsocketConnect(); - webSock = new HTTP::Websocket(myConn, H); + HTTP::Parser req = H; + H.Clean(); + webSock = new HTTP::Websocket(myConn, req, H); if (!(*webSock)){ delete webSock; webSock = 0; @@ -333,6 +348,14 @@ namespace Mist{ void HTTPOutput::respondHTTP(const HTTP::Parser & req, bool headersOnly){ //We generally want the CORS headers to be set for all responses H.setCORSHeaders(); + H.SetHeader("Server", APPIDENT); + if (tkn.size()){ + if (Comms::tknMode & 0x08){ + std::stringstream cookieHeader; + cookieHeader << "tkn=" << tkn << "; Max-Age=" << SESS_TIMEOUT; + H.SetHeader("Set-Cookie", cookieHeader.str()); + } + } //Set attachment header to force download, if applicable if (req.GetVar("dl").size()){ //If we want to download, and the string contains a dot, use as-is. @@ -395,6 +418,8 @@ namespace Mist{ ///\brief Handles requests by starting a corresponding output process. ///\param connector The type of connector to be invoked. void HTTPOutput::reConnector(std::string &connector){ + // Clear tkn in order to deal with reverse proxies + tkn = ""; // taken from CheckProtocols (controller_connectors.cpp) char *argarr[32]; for (int i = 0; i < 32; i++){argarr[i] = 0;} diff --git a/src/output/output_http_internal.cpp b/src/output/output_http_internal.cpp index 97ca455f..be955196 100644 --- a/src/output/output_http_internal.cpp +++ b/src/output/output_http_internal.cpp @@ -76,11 +76,11 @@ namespace Mist{ std::string method = H.method; // send logo icon if (H.url.length() > 4 && H.url.substr(H.url.length() - 4, 4) == ".ico"){ - sendIcon(); + sendIcon(false); return; } if (H.url.length() > 6 && H.url.substr(H.url.length() - 5, 5) == ".html"){ - HTMLResponse(); + HTMLResponse(H, false); return; } if (H.url.size() >= 3 && H.url.substr(H.url.size() - 3) == ".js"){ @@ -337,9 +337,9 @@ namespace Mist{ } } - void OutHTTP::HTMLResponse(){ - std::string method = H.method; - HTTP::URL fullURL(H.GetHeader("Host")); + void OutHTTP::HTMLResponse(const HTTP::Parser & req, bool headersOnly){ + HTTPOutput::respondHTTP(req, headersOnly); + HTTP::URL fullURL(req.GetHeader("Host")); if (!fullURL.protocol.size()){fullURL.protocol = getProtocolForPort(fullURL.getPort());} if (config->getString("pubaddr") != ""){ HTTP::URL altURL(config->getString("pubaddr")); @@ -349,24 +349,22 @@ namespace Mist{ fullURL.path = altURL.path; } if (mistPath.size()){fullURL = mistPath;} - std::string uAgent = H.GetHeader("User-Agent"); + std::string uAgent = req.GetHeader("User-Agent"); std::string forceType = ""; if (H.GetVar("forcetype").size()){ - forceType = ",forceType:\"" + H.GetVar("forcetype") + "\""; + forceType = ",forceType:\"" + req.GetVar("forcetype") + "\""; } std::string devSkin = ""; - if (H.GetVar("dev").size()){devSkin = ",skin:\"dev\"";} - H.SetVar("stream", ""); - H.SetVar("dev", ""); + if (req.GetVar("dev").size()){devSkin = ",skin:\"dev\"";} devSkin += ",urlappend:\"" + H.allVars() + "\""; H.SetVar("stream", streamName); std::string seekTo = ""; - if (H.GetVar("t").size()){ + if (req.GetVar("t").size()){ uint64_t autoSeekTime = 0; - std::string sTime = H.GetVar("t"); + std::string sTime = req.GetVar("t"); unsigned long long h = 0, m = 0, s = 0; autoSeekTime = JSON::Value(sTime).asInt(); if (sscanf(sTime.c_str(), "%llum%llus", &m, &s) == 2){autoSeekTime = m * 60 + s;} @@ -385,13 +383,10 @@ namespace Mist{ streamName + "\").addEventListener(\"initialized\",f);"; } } - - H.Clean(); + H.SetHeader("Content-Type", "text/html"); H.SetHeader("X-UA-Compatible", "IE=edge"); - H.SetHeader("Server", APPIDENT); - H.setCORSHeaders(); - if (method == "OPTIONS" || method == "HEAD"){ + if (headersOnly){ H.SendResponse("200", "OK", myConn); responded = true; H.Clean(); @@ -427,6 +422,7 @@ namespace Mist{ } H.SendResponse("200", "OK", myConn); responded = true; + H.Clean(); } JSON::Value OutHTTP::getStatusJSON(std::string &reqHost, const std::string &useragent){ @@ -634,23 +630,31 @@ namespace Mist{ // loop over the added sources, add them to json_resp["sources"] for (std::set::iterator it = sources.begin(); it != sources.end(); it++){ - if ((*it)["simul_tracks"].asInt() > 0){json_resp["source"].append(*it);} + if ((*it)["simul_tracks"].asInt() > 0){ + if (Comms::tknMode & 0x04){ + JSON::Value tmp; + tmp = (*it); + tmp["url"] = tmp["url"].asStringRef() + "?tkn=" + tkn; + tmp["relurl"] = tmp["relurl"].asStringRef() + "?tkn=" + tkn; + json_resp["source"].append(tmp); + }else{ + json_resp["source"].append(*it); + } + } } return json_resp; } - void OutHTTP::onHTTP(){ + void OutHTTP::respondHTTP(const HTTP::Parser & req, bool headersOnly){ origStreamName = streamName; - std::string method = H.method; - if (H.GetHeader("X-Mst-Path").size()){mistPath = H.GetHeader("X-Mst-Path");} + if (req.GetHeader("X-Mst-Path").size()){mistPath = req.GetHeader("X-Mst-Path");} // Handle certbot validations - if (H.url.substr(0, 28) == "/.well-known/acme-challenge/"){ + if (req.url.substr(0, 28) == "/.well-known/acme-challenge/"){ std::string cbToken = H.url.substr(28); jsonForEach(config->getOption("certbot", true), it){ if (it->asStringRef().substr(0, cbToken.size() + 1) == cbToken + ":"){ - H.Clean(); H.SetHeader("Content-Type", "text/plain"); H.SetHeader("Server", APPIDENT); H.setCORSHeaders(); @@ -661,9 +665,7 @@ namespace Mist{ return; } } - H.Clean(); H.SetHeader("Content-Type", "text/plain"); - H.SetHeader("Server", APPIDENT); H.setCORSHeaders(); H.SetBody("No matching validation found for token '" + cbToken + "'"); H.SendResponse("404", "Not found", myConn); @@ -672,12 +674,11 @@ namespace Mist{ return; } - if (H.url == "/crossdomain.xml"){ - H.Clean(); + if (req.url == "/crossdomain.xml"){ H.SetHeader("Content-Type", "text/xml"); H.SetHeader("Server", APPIDENT); H.setCORSHeaders(); - if (method == "OPTIONS" || method == "HEAD"){ + if (headersOnly){ H.SendResponse("200", "OK", myConn); responded = true; H.Clean(); @@ -693,12 +694,11 @@ namespace Mist{ return; }// crossdomain.xml - if (H.url == "/clientaccesspolicy.xml"){ - H.Clean(); + if (req.url == "/clientaccesspolicy.xml"){ H.SetHeader("Content-Type", "text/xml"); H.SetHeader("Server", APPIDENT); H.setCORSHeaders(); - if (method == "OPTIONS" || method == "HEAD"){ + if (headersOnly){ H.SendResponse("200", "OK", myConn); responded = true; H.Clean(); @@ -716,8 +716,7 @@ namespace Mist{ return; }// clientaccesspolicy.xml - if (H.url == "/flashplayer.swf"){ - H.Clean(); + if (req.url == "/flashplayer.swf"){ H.SetHeader("Content-Type", "application/x-shockwave-flash"); H.SetHeader("Server", APPIDENT); H.SetBody((const char *)FlashMediaPlayback_101_swf, FlashMediaPlayback_101_swf_len); @@ -725,8 +724,7 @@ namespace Mist{ responded = true; return; } - if (H.url == "/oldflashplayer.swf"){ - H.Clean(); + if (req.url == "/oldflashplayer.swf"){ H.SetHeader("Content-Type", "application/x-shockwave-flash"); H.SetHeader("Server", APPIDENT); H.SetBody((const char *)FlashMediaPlayback_swf, FlashMediaPlayback_swf_len); @@ -735,20 +733,21 @@ namespace Mist{ return; } // send logo icon - if (H.url.length() > 4 && H.url.substr(H.url.length() - 4, 4) == ".ico"){ - sendIcon(); + if (req.url.length() > 4 && req.url.substr(req.url.length() - 4, 4) == ".ico"){ + sendIcon(headersOnly); return; } // send generic HTML page - if (H.url.length() > 6 && H.url.substr(H.url.length() - 5, 5) == ".html"){ - HTMLResponse(); + if (req.url.length() > 6 && req.url.substr(req.url.length() - 5, 5) == ".html"){ + HTMLResponse(req, headersOnly); return; } // send smil MBR index - if (H.url.length() > 6 && H.url.substr(H.url.length() - 5, 5) == ".smil"){ - std::string reqHost = HTTP::URL(H.GetHeader("Host")).host; + if (req.url.length() > 6 && req.url.substr(req.url.length() - 5, 5) == ".smil"){ + HTTPOutput::respondHTTP(req, headersOnly); + std::string reqHost = HTTP::URL(req.GetHeader("Host")).host; std::string port, url_rel; std::string trackSources; // this string contains all track sources for MBR smil { @@ -782,11 +781,8 @@ namespace Mist{ } } - H.Clean(); H.SetHeader("Content-Type", "application/smil"); - H.SetHeader("Server", APPIDENT); - H.setCORSHeaders(); - if (method == "OPTIONS" || method == "HEAD"){ + if (headersOnly){ H.SendResponse("200", "OK", myConn); responded = true; H.Clean(); @@ -800,24 +796,22 @@ namespace Mist{ return; } - if ((H.url.length() > 9 && H.url.substr(0, 6) == "/info_" && H.url.substr(H.url.length() - 3, 3) == ".js") || - (H.url.length() > 9 && H.url.substr(0, 6) == "/json_" && H.url.substr(H.url.length() - 3, 3) == ".js")){ - if (websocketHandler()){return;} - std::string reqHost = HTTP::URL(H.GetHeader("Host")).host; - std::string useragent = H.GetVar("ua"); - if (!useragent.size()){useragent = H.GetHeader("User-Agent");} + if ((req.url.length() > 9 && req.url.substr(0, 6) == "/info_" && req.url.substr(req.url.length() - 3, 3) == ".js") || + (req.url.length() > 9 && req.url.substr(0, 6) == "/json_" && req.url.substr(req.url.length() - 3, 3) == ".js")){ + HTTPOutput::respondHTTP(req, headersOnly); + if (websocketHandler(req, headersOnly)){return;} + std::string reqHost = HTTP::URL(req.GetHeader("Host")).host; + std::string useragent = req.GetVar("ua"); + if (!useragent.size()){useragent = req.GetHeader("User-Agent");} std::string response; - std::string rURL = H.url; - if (method != "OPTIONS" && method != "HEAD"){initialize();} - H.Clean(); - H.SetHeader("Server", APPIDENT); - H.setCORSHeaders(); + std::string rURL = req.url; + if (headersOnly){initialize();} if (rURL.substr(0, 6) != "/json_"){ H.SetHeader("Content-Type", "application/javascript"); }else{ H.SetHeader("Content-Type", "application/json"); } - if (method == "OPTIONS" || method == "HEAD"){ + if (headersOnly){ H.SendResponse("200", "OK", myConn); responded = true; H.Clean(); @@ -837,9 +831,9 @@ namespace Mist{ return; }// embed code generator - if ((H.url == "/player.js") || ((H.url.substr(0, 7) == "/embed_") && (H.url.length() > 10) && - (H.url.substr(H.url.length() - 3, 3) == ".js"))){ - HTTP::URL fullURL(H.GetHeader("Host")); + if ((req.url == "/player.js") || ((req.url.substr(0, 7) == "/embed_") && (req.url.length() > 10) && + (req.url.substr(H.url.length() - 3, 3) == ".js"))){ + HTTP::URL fullURL(req.GetHeader("Host")); if (!fullURL.protocol.size()){fullURL.protocol = getProtocolForPort(fullURL.getPort());} if (config->getString("pubaddr") != ""){ HTTP::URL altURL(config->getString("pubaddr")); @@ -850,12 +844,17 @@ namespace Mist{ } if (mistPath.size()){fullURL = mistPath;} std::string response; - std::string rURL = H.url; - H.Clean(); + std::string rURL = req.url; + + if ((rURL.substr(0, 7) == "/embed_") && (rURL.length() > 10) && + (rURL.substr(rURL.length() - 3, 3) == ".js")){ + HTTPOutput::respondHTTP(req, headersOnly); + } + H.SetHeader("Server", APPIDENT); H.setCORSHeaders(); H.SetHeader("Content-Type", "application/javascript; charset=utf-8"); - if (method == "OPTIONS" || method == "HEAD"){ + if (headersOnly){ H.SendResponse("200", "OK", myConn); responded = true; H.Clean(); @@ -933,14 +932,13 @@ namespace Mist{ return; } - if (H.url.substr(0, 7) == "/skins/"){ + if (req.url.substr(0, 7) == "/skins/"){ std::string response; - std::string url = H.url; - H.Clean(); + std::string url = req.url; H.SetHeader("Server", APPIDENT); H.setCORSHeaders(); H.SetHeader("Content-Type", "text/css"); - if (method == "OPTIONS" || method == "HEAD"){ + if (headersOnly){ H.SendResponse("200", "OK", myConn); responded = true; H.Clean(); @@ -970,13 +968,12 @@ namespace Mist{ H.Clean(); return; } - if (H.url == "/videojs.js"){ + if (req.url == "/videojs.js"){ std::string response; - H.Clean(); H.SetHeader("Server", APPIDENT); H.setCORSHeaders(); H.SetHeader("Content-Type", "application/javascript"); - if (method == "OPTIONS" || method == "HEAD"){ + if (headersOnly){ H.SendResponse("200", "OK", myConn); responded = true; H.Clean(); @@ -992,13 +989,12 @@ namespace Mist{ H.Clean(); return; } - if (H.url == "/dashjs.js"){ + if (req.url == "/dashjs.js"){ std::string response; - H.Clean(); H.SetHeader("Server", APPIDENT); H.setCORSHeaders(); H.SetHeader("Content-Type", "application/javascript"); - if (method == "OPTIONS" || method == "HEAD"){ + if (headersOnly){ H.SendResponse("200", "OK", myConn); responded = true; H.Clean(); @@ -1016,13 +1012,12 @@ namespace Mist{ H.Clean(); return; } - if (H.url == "/webrtc.js"){ + if (req.url == "/webrtc.js"){ std::string response; - H.Clean(); H.SetHeader("Server", APPIDENT); H.setCORSHeaders(); H.SetHeader("Content-Type", "application/javascript"); - if (method == "OPTIONS" || method == "HEAD"){ + if (headersOnly){ H.SendResponse("200", "OK", myConn); responded = true; H.Clean(); @@ -1038,13 +1033,12 @@ namespace Mist{ H.Clean(); return; } - if (H.url == "/flv.js"){ + if (req.url == "/flv.js"){ std::string response; - H.Clean(); H.SetHeader("Server", "MistServer/" PACKAGE_VERSION); H.setCORSHeaders(); H.SetHeader("Content-Type", "application/javascript"); - if (method == "OPTIONS" || method == "HEAD"){ + if (headersOnly){ H.SendResponse("200", "OK", myConn); H.Clean(); return; @@ -1058,13 +1052,12 @@ namespace Mist{ H.Clean(); return; } - if (H.url == "/hlsjs.js"){ + if (req.url == "/hlsjs.js"){ std::string response; - H.Clean(); H.SetHeader("Server", "MistServer/" PACKAGE_VERSION); H.setCORSHeaders(); H.SetHeader("Content-Type", "application/javascript"); - if (method == "OPTIONS" || method == "HEAD"){ + if (headersOnly){ H.SendResponse("200", "OK", myConn); H.Clean(); return; @@ -1084,7 +1077,7 @@ namespace Mist{ H.SetHeader("Server", "MistServer/" PACKAGE_VERSION); H.setCORSHeaders(); H.SetHeader("Content-Type", "application/javascript"); - if (method == "OPTIONS" || method == "HEAD"){ + if (headersOnly){ H.SendResponse("200", "OK", myConn); H.Clean(); return; @@ -1100,15 +1093,13 @@ namespace Mist{ } } - void OutHTTP::sendIcon(){ - std::string method = H.method; - H.Clean(); + void OutHTTP::sendIcon(bool headersOnly){ #include "../icon.h" H.SetHeader("Content-Type", "image/x-icon"); H.SetHeader("Server", APPIDENT); H.SetHeader("Content-Length", icon_len); H.setCORSHeaders(); - if (method == "OPTIONS" || method == "HEAD"){ + if (headersOnly){ H.SendResponse("200", "OK", myConn); responded = true; H.Clean(); @@ -1120,16 +1111,16 @@ namespace Mist{ H.Clean(); } - bool OutHTTP::websocketHandler(){ + bool OutHTTP::websocketHandler(const HTTP::Parser & req, bool headersOnly){ stayConnected = true; - std::string reqHost = HTTP::URL(H.GetHeader("Host")).host; - if (H.GetHeader("X-Mst-Path").size()){mistPath = H.GetHeader("X-Mst-Path");} - std::string useragent = H.GetVar("ua"); - if (!useragent.size()){useragent = H.GetHeader("User-Agent");} - std::string upgradeHeader = H.GetHeader("Upgrade"); + std::string reqHost = HTTP::URL(req.GetHeader("Host")).host; + if (req.GetHeader("X-Mst-Path").size()){mistPath = req.GetHeader("X-Mst-Path");} + std::string useragent = req.GetVar("ua"); + if (!useragent.size()){useragent = req.GetHeader("User-Agent");} + std::string upgradeHeader = req.GetHeader("Upgrade"); Util::stringToLower(upgradeHeader); if (upgradeHeader != "websocket"){return false;} - HTTP::Websocket ws(myConn, H); + HTTP::Websocket ws(myConn, req, H); if (!ws){return false;} setBlocking(false); // start the stream, if needed diff --git a/src/output/output_http_internal.h b/src/output/output_http_internal.h index 774eb186..8e610145 100644 --- a/src/output/output_http_internal.h +++ b/src/output/output_http_internal.h @@ -10,10 +10,10 @@ namespace Mist{ virtual void onFail(const std::string &msg, bool critical = false); /// preHTTP is disabled in the internal HTTP output, since most don't need the stream alive to work virtual void preHTTP(){}; - void HTMLResponse(); - void onHTTP(); - void sendIcon(); - bool websocketHandler(); + void HTMLResponse(const HTTP::Parser & req, bool headersOnly); + void respondHTTP(const HTTP::Parser & req, bool headersOnly); + void sendIcon(bool headersOnly); + bool websocketHandler(const HTTP::Parser & req, bool headersOnly); JSON::Value getStatusJSON(std::string &reqHost, const std::string &useragent = ""); bool stayConnected; virtual bool onFinish(){return stayConnected;} diff --git a/src/output/output_sdp.cpp b/src/output/output_sdp.cpp index 6baf9067..50d491e8 100644 --- a/src/output/output_sdp.cpp +++ b/src/output/output_sdp.cpp @@ -141,6 +141,18 @@ namespace Mist{ } } + std::string OutSDP::getConnectedHost(){ + if (!sdpState.tracks.size()) { return Output::getConnectedHost(); } + std::string hostname; + uint32_t port; + sdpState.tracks[0].data.GetDestination(hostname, port); + return hostname; + } + std::string OutSDP::getConnectedBinHost(){ + if (!sdpState.tracks.size()) { return Output::getConnectedBinHost(); } + return sdpState.tracks[0].data.getBinDestination(); + } + void OutSDP::sendNext(){ char *dataPointer = 0; size_t dataLen = 0; diff --git a/src/output/output_sdp.h b/src/output/output_sdp.h index 2d4b3368..6c466753 100644 --- a/src/output/output_sdp.h +++ b/src/output/output_sdp.h @@ -16,6 +16,8 @@ namespace Mist{ void sendNext(); void sendHeader(); bool onFinish(); + std::string getConnectedHost(); + std::string getConnectedBinHost(); private: void initTracks(uint32_t & port, std::string targetIP); diff --git a/src/output/output_ts.cpp b/src/output/output_ts.cpp index 561bf9aa..bf46663b 100644 --- a/src/output/output_ts.cpp +++ b/src/output/output_ts.cpp @@ -239,6 +239,18 @@ namespace Mist{ } } + std::string OutTS::getConnectedHost(){ + if (!pushOut) { return Output::getConnectedHost(); } + std::string hostname; + uint32_t port; + pushSock.GetDestination(hostname, port); + return hostname; + } + std::string OutTS::getConnectedBinHost(){ + if (!pushOut) { return Output::getConnectedBinHost(); } + return pushSock.getBinDestination(); + } + bool OutTS::listenMode(){return !(config->getString("target").size());} void OutTS::onRequest(){ diff --git a/src/output/output_ts.h b/src/output/output_ts.h index 32aa6958..0c38cc70 100644 --- a/src/output/output_ts.h +++ b/src/output/output_ts.h @@ -12,6 +12,8 @@ namespace Mist{ virtual void initialSeek(); bool isReadyForPlay(); void onRequest(); + std::string getConnectedHost(); + std::string getConnectedBinHost(); private: size_t udpSize; diff --git a/src/output/output_tsrist.cpp b/src/output/output_tsrist.cpp index c4e03f80..10b29673 100644 --- a/src/output/output_tsrist.cpp +++ b/src/output/output_tsrist.cpp @@ -178,6 +178,20 @@ namespace Mist{ } OutTSRIST::~OutTSRIST(){} + + std::string OutTSRIST::getConnectedHost(){ + if (!pushOut) { return Output::getConnectedHost(); } + return target.host; + } + std::string OutTSRIST::getConnectedBinHost(){ + if (!pushOut) { return Output::getConnectedBinHost(); } + std::string binHost = Socket::getBinForms(target.host); + if (binHost.size() > 16){ binHost = binHost.substr(0, 16); } + if (binHost.size() < 16){ + binHost = std::string("\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\001", 16); + } + return binHost; + } void OutTSRIST::init(Util::Config *cfg){ Output::init(cfg); @@ -319,7 +333,7 @@ namespace Mist{ } } - void OutTSRIST::connStats(uint64_t now, Comms::Statistics &statComm){ + void OutTSRIST::connStats(uint64_t now, Comms::Connections &statComm){ if (!myConn){return;} statComm.setUp(upBytes); statComm.setDown(0); diff --git a/src/output/output_tsrist.h b/src/output/output_tsrist.h index c9a5db39..80ee2376 100644 --- a/src/output/output_tsrist.h +++ b/src/output/output_tsrist.h @@ -16,9 +16,11 @@ namespace Mist{ bool isReadyForPlay(){return true;} virtual void requestHandler(); static void listener(Util::Config &conf, int (*callback)(Socket::Connection &S)); + std::string getConnectedHost(); + std::string getConnectedBinHost(); protected: - virtual void connStats(uint64_t now, Comms::Statistics &statComm); + virtual void connStats(uint64_t now, Comms::Connections &statComm); //virtual std::string getConnectedHost(){ // return srtConn.remotehost; //} diff --git a/src/session.cpp b/src/session.cpp index 3865e0ec..d2331c53 100644 --- a/src/session.cpp +++ b/src/session.cpp @@ -7,17 +7,31 @@ #include #include #include -// Stats of connections which have closed are added to these global counters -uint64_t globalNow = 0; + +// Global counters +uint64_t now = Util::bootSecs(); +uint64_t currentConnections = 0; +uint64_t lastSecond = 0; uint64_t globalTime = 0; uint64_t globalDown = 0; uint64_t globalUp = 0; uint64_t globalPktcount = 0; uint64_t globalPktloss = 0; uint64_t globalPktretrans = 0; +// Stores last values of each connection +std::map connTime; +std::map connDown; +std::map connUp; +std::map connPktcount; +std::map connPktloss; +std::map connPktretrans; // Counts the duration a connector has been active std::map connectorCount; std::map connectorLastActive; +std::map hostCount; +std::map hostLastActive; +std::map streamCount; +std::map streamLastActive; // Set to True when a session gets invalidated, so that we know to run a new USER_NEW trigger bool forceTrigger = false; void handleSignal(int signum){ @@ -26,96 +40,141 @@ void handleSignal(int signum){ } } -void userOnActive(uint64_t &connections){ - ++connections; -} +const char nullAddress[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; -std::string getEnvWithDefault(const std::string variableName, const std::string defaultValue){ - const char* value = getenv(variableName.c_str()); - if (value){ - unsetenv(variableName.c_str()); - return value; - }else{ - return defaultValue; - } +void userOnActive(Comms::Connections &connections, size_t idx){ + uint64_t lastUpdate = connections.getNow(idx); + if (lastUpdate < now - 10){return;} + ++currentConnections; + std::string thisConnector = connections.getConnector(idx); + std::string thisStreamName = connections.getStream(idx); + const std::string& thisHost = connections.getHost(idx); + + if (connections.getLastSecond(idx) > lastSecond){lastSecond = connections.getLastSecond(idx);} + // Save info on the latest active stream, protocol and host separately + if (thisConnector.size() && thisConnector != "HTTP"){ + connectorCount[thisConnector]++; + if (connectorLastActive[thisConnector] < lastUpdate){connectorLastActive[thisConnector] = lastUpdate;} + } + if (thisStreamName.size()){ + streamCount[thisStreamName]++; + if (streamLastActive[thisStreamName] < lastUpdate){streamLastActive[thisStreamName] = lastUpdate;} + } + if (memcmp(thisHost.data(), nullAddress, 16)){ + hostCount[thisHost]++; + if (!hostLastActive.count(thisHost) || hostLastActive[thisHost] < lastUpdate){hostLastActive[thisHost] = lastUpdate;} + } + // Sanity checks + if (connections.getDown(idx) < connDown[idx]){ + WARN_MSG("Connection downloaded bytes should be a counter, but has decreased in value"); + connDown[idx] = connections.getDown(idx); + } + if (connections.getUp(idx) < connUp[idx]){ + WARN_MSG("Connection uploaded bytes should be a counter, but has decreased in value"); + connUp[idx] = connections.getUp(idx); + } + if (connections.getPacketCount(idx) < connPktcount[idx]){ + WARN_MSG("Connection packet count should be a counter, but has decreased in value"); + connPktcount[idx] = connections.getPacketCount(idx); + } + if (connections.getPacketLostCount(idx) < connPktloss[idx]){ + WARN_MSG("Connection packet loss count should be a counter, but has decreased in value"); + connPktloss[idx] = connections.getPacketLostCount(idx); + } + if (connections.getPacketRetransmitCount(idx) < connPktretrans[idx]){ + WARN_MSG("Connection packets retransmitted should be a counter, but has decreased in value"); + connPktretrans[idx] = connections.getPacketRetransmitCount(idx); + } + // Add increase in stats to global stats + globalDown += connections.getDown(idx) - connDown[idx]; + globalUp += connections.getUp(idx) - connUp[idx]; + globalPktcount += connections.getPacketCount(idx) - connPktcount[idx]; + globalPktloss += connections.getPacketLostCount(idx) - connPktloss[idx]; + globalPktretrans += connections.getPacketRetransmitCount(idx) - connPktretrans[idx]; + // Set last values of this connection + connTime[idx]++; + connDown[idx] = connections.getDown(idx); + connUp[idx] = connections.getUp(idx); + connPktcount[idx] = connections.getPacketCount(idx); + connPktloss[idx] = connections.getPacketLostCount(idx); + connPktretrans[idx] = connections.getPacketRetransmitCount(idx); } -/// \brief Adds stats of closed connections to global counters +/// \brief Remove mappings of inactive connections void userOnDisconnect(Comms::Connections & connections, size_t idx){ - std::string thisConnector = connections.getConnector(idx); - if (thisConnector != ""){ - connectorCount[thisConnector] += connections.getTime(idx); - } - globalTime += connections.getTime(idx); - globalDown += connections.getDown(idx); - globalUp += connections.getUp(idx); - globalPktcount += connections.getPacketCount(idx); - globalPktloss += connections.getPacketLostCount(idx); - globalPktretrans += connections.getPacketRetransmitCount(idx); + connTime.erase(idx); + connDown.erase(idx); + connUp.erase(idx); + connPktcount.erase(idx); + connPktloss.erase(idx); + connPktretrans.erase(idx); } int main(int argc, char **argv){ Comms::Connections connections; Comms::Sessions sessions; uint64_t lastSeen = Util::bootSecs(); - uint64_t currentConnections = 0; Util::redirectLogsIfNeeded(); signal(SIGUSR1, handleSignal); // Init config and parse arguments Util::Config config = Util::Config("MistSession"); JSON::Value option; + char * tmpStr = 0; option.null(); option["arg_num"] = 1; option["arg"] = "string"; option["help"] = "Session identifier of the entire session"; - option["default"] = ""; config.addOption("sessionid", option); - option.null(); - option["long"] = "sessionmode"; - option["short"] = "m"; - option["arg"] = "integer"; - option["default"] = 0; - config.addOption("sessionmode", option); - option.null(); option["long"] = "streamname"; - option["short"] = "n"; + option["short"] = "s"; option["arg"] = "string"; - option["default"] = ""; + option["help"] = "Stream name initial value. May also be passed as SESSION_STREAM"; + tmpStr = getenv("SESSION_STREAM"); + option["default"] = tmpStr?tmpStr:""; config.addOption("streamname", option); option.null(); option["long"] = "ip"; option["short"] = "i"; option["arg"] = "string"; - option["default"] = ""; + option["help"] = "IP address initial value. May also be passed as SESSION_IP"; + tmpStr = getenv("SESSION_IP"); + option["default"] = tmpStr?tmpStr:""; config.addOption("ip", option); option.null(); - option["long"] = "sid"; - option["short"] = "s"; + option["long"] = "tkn"; + option["short"] = "t"; option["arg"] = "string"; - option["default"] = ""; - config.addOption("sid", option); + option["help"] = "Client-side session ID initial value. May also be passed as SESSION_TKN"; + tmpStr = getenv("SESSION_TKN"); + option["default"] = tmpStr?tmpStr:""; + config.addOption("tkn", option); option.null(); option["long"] = "protocol"; option["short"] = "p"; option["arg"] = "string"; - option["default"] = ""; + option["help"] = "Protocol initial value. May also be passed as SESSION_PROTOCOL"; + tmpStr = getenv("SESSION_PROTOCOL"); + option["default"] = tmpStr?tmpStr:""; config.addOption("protocol", option); option.null(); option["long"] = "requrl"; option["short"] = "r"; option["arg"] = "string"; - option["default"] = ""; + option["help"] = "Request URL initial value. May also be passed as SESSION_REQURL"; + tmpStr = getenv("SESSION_REQURL"); + option["default"] = tmpStr?tmpStr:""; config.addOption("requrl", option); config.activate(); if (!(config.parseArgs(argc, argv))){ + config.printHelp(std::cout); FAIL_MSG("Cannot start a new session due to invalid arguments"); return 1; } @@ -123,24 +182,17 @@ int main(int argc, char **argv){ const uint64_t bootTime = Util::getMicros(); // Get session ID, session mode and other variables used as payload for the USER_NEW and USER_END triggers const std::string thisStreamName = config.getString("streamname"); - const std::string thisHost = config.getString("ip"); - const std::string thisSid = config.getString("sid"); + const std::string thisToken = config.getString("tkn"); const std::string thisProtocol = config.getString("protocol"); const std::string thisReqUrl = config.getString("requrl"); const std::string thisSessionId = config.getString("sessionid"); - const uint64_t sessionMode = config.getInteger("sessionmode"); - - if (thisSessionId == "" || thisProtocol == "" || thisStreamName == ""){ - FAIL_MSG("Given the following incomplete arguments: SessionId: '%s', protocol: '%s', stream name: '%s'. Aborting opening a new session", - thisSessionId.c_str(), thisProtocol.c_str(), thisStreamName.c_str()); - return 1; - } + std::string thisHost = Socket::getBinForms(config.getString("ip")); + if (thisHost.size() > 16){thisHost = thisHost.substr(0, 16);} - MEDIUM_MSG("Starting a new session for sessionId '%s'", thisSessionId.c_str()); - if (sessionMode < 1 || sessionMode > 15) { - FAIL_MSG("Invalid session mode of value %lu. Should be larger than 0 and smaller than 16", sessionMode); - return 1; - } + std::string ipHex; + Socket::hostBytesToStr(thisHost.c_str(), thisHost.size(), ipHex); + VERYHIGH_MSG("Starting a new session. Passed variables are stream name '%s', session token '%s', protocol '%s', requested URL '%s', IP '%s' and session id '%s'", + thisStreamName.c_str(), thisToken.c_str(), thisProtocol.c_str(), thisReqUrl.c_str(), ipHex.c_str(), thisSessionId.c_str()); // Try to lock to ensure we are the only process initialising this session IPC::semaphore sessionLock; @@ -174,194 +226,200 @@ int main(int argc, char **argv){ sessionLock.post(); return 1; } - // Open the shared memory page containing statistics for each individual connection in this session - connections.reload(thisStreamName, thisHost, thisSid, thisProtocol, thisReqUrl, sessionMode, true, false); + // Initialise global session data sessions.setHost(thisHost); sessions.setSessId(thisSessionId); sessions.setStream(thisStreamName); - sessionLock.post(); + if (thisProtocol.size() && thisProtocol != "HTTP"){connectorLastActive[thisProtocol] = now;} + if (thisStreamName.size()){streamLastActive[thisStreamName] = now;} + if (memcmp(thisHost.data(), nullAddress, 16)){hostLastActive[thisHost] = now;} + + // Open the shared memory page containing statistics for each individual connection in this session + connections.reload(thisSessionId, true); // Determine session type, since triggers only get run for viewer type sessions uint64_t thisType = 0; if (thisSessionId[0] == 'I'){ - INFO_MSG("Started new input session %s in %lu microseconds", thisSessionId.c_str(), Util::getMicros(bootTime)); thisType = 1; - } - else if (thisSessionId[0] == 'O'){ - INFO_MSG("Started new output session %s in %lu microseconds", thisSessionId.c_str(), Util::getMicros(bootTime)); + } else if (thisSessionId[0] == 'O'){ thisType = 2; - } - else{ - INFO_MSG("Started new viewer session %s in %lu microseconds", thisSessionId.c_str(), Util::getMicros(bootTime)); + } else if (thisSessionId[0] == 'U'){ + thisType = 3; } // Do a USER_NEW trigger if it is defined for this stream if (!thisType && Triggers::shouldTrigger("USER_NEW", thisStreamName)){ - std::string payload = thisStreamName + "\n" + thisHost + "\n" + - thisSid + "\n" + thisProtocol + + std::string payload = thisStreamName + "\n" + config.getString("ip") + "\n" + + thisToken + "\n" + thisProtocol + "\n" + thisReqUrl + "\n" + thisSessionId; if (!Triggers::doTrigger("USER_NEW", payload, thisStreamName)){ // Mark all connections of this session as finished, since this viewer is not allowed to view this stream + Util::logExitReason("Session rejected by USER_NEW"); connections.setExit(); connections.finishAll(); } } - uint64_t lastSecond = 0; - uint64_t now = 0; - uint64_t time = 0; - uint64_t down = 0; - uint64_t up = 0; - uint64_t pktcount = 0; - uint64_t pktloss = 0; - uint64_t pktretrans = 0; - std::string connector = ""; + //start allowing viewers + sessionLock.post(); + + INFO_MSG("Started new session %s in %.3f ms", thisSessionId.c_str(), (double)Util::getMicros(bootTime)/1000.0); + // Stay active until Mist exits or we no longer have an active connection - while (config.is_active && (currentConnections || Util::bootSecs() - lastSeen <= 10)){ - time = 0; - connector = ""; - down = 0; - up = 0; - pktcount = 0; - pktloss = 0; - pktretrans = 0; + while (config.is_active && (currentConnections || now - lastSeen <= STATS_DELAY) && !connections.getExit()){ currentConnections = 0; + lastSecond = 0; + now = Util::bootSecs(); - // Count active connections - COMM_LOOP(connections, userOnActive(currentConnections), userOnDisconnect(connections, id)); // Loop through all connection entries to get a summary of statistics - for (uint64_t idx = 0; idx < connections.recordCount(); idx++){ - if (connections.getStatus(idx) == COMM_STATUS_INVALID || connections.getStatus(idx) & COMM_STATUS_DISCONNECT){continue;} - uint64_t thisLastSecond = connections.getLastSecond(idx); - std::string thisConnector = connections.getConnector(idx); - // Save info on the latest active connection separately - if (thisLastSecond > lastSecond){ - lastSecond = thisLastSecond; - now = connections.getNow(idx); - } - connectorLastActive[thisConnector] = thisLastSecond; - // Sum all other variables - time += connections.getTime(idx); - down += connections.getDown(idx); - up += connections.getUp(idx); - pktcount += connections.getPacketCount(idx); - pktloss += connections.getPacketLostCount(idx); - pktretrans += connections.getPacketRetransmitCount(idx); + COMM_LOOP(connections, userOnActive(connections, id), userOnDisconnect(connections, id)); + if (currentConnections){ + globalTime++; + lastSeen = now; } - // Convert connector duration to string - std::stringstream connectorSummary; - bool addDelimiter = false; - connectorSummary << "{"; - for (std::map::iterator it = connectorLastActive.begin(); - it != connectorLastActive.end(); ++it){ - if (lastSecond - it->second < 10000){ - connectorSummary << (addDelimiter ? "," : "") << it->first; - addDelimiter = true; - } - } - connectorSummary << "}"; - - // Write summary to global statistics - sessions.setTime(time + globalTime); - sessions.setDown(down + globalDown); - sessions.setUp(up + globalUp); - sessions.setPacketCount(pktcount + globalPktcount); - sessions.setPacketLostCount(pktloss + globalPktloss); - sessions.setPacketRetransmitCount(pktretrans + globalPktretrans); + + sessions.setTime(globalTime); + sessions.setDown(globalDown); + sessions.setUp(globalUp); + sessions.setPacketCount(globalPktcount); + sessions.setPacketLostCount(globalPktloss); + sessions.setPacketRetransmitCount(globalPktretrans); sessions.setLastSecond(lastSecond); - sessions.setConnector(connectorSummary.str()); sessions.setNow(now); + if (currentConnections){ + { + // Convert active protocols to string + std::stringstream connectorSummary; + for (std::map::iterator it = connectorLastActive.begin(); + it != connectorLastActive.end(); ++it){ + if (now - it->second < STATS_DELAY){ + connectorSummary << (connectorSummary.str().size() ? "," : "") << it->first; + } + } + sessions.setConnector(connectorSummary.str()); + } + + { + // Set active host to last active or 0 if there were various hosts active recently + std::string thisHost; + for (std::map::iterator it = hostLastActive.begin(); + it != hostLastActive.end(); ++it){ + if (now - it->second < STATS_DELAY){ + if (!thisHost.size()){ + thisHost = it->first; + }else if (thisHost != it->first){ + thisHost = nullAddress; + break; + } + } + } + if (!thisHost.size()){ + thisHost = nullAddress; + } + sessions.setHost(thisHost); + } + + { + // Set active stream name to last active or "" if there were multiple streams active recently + std::string thisStream = ""; + for (std::map::iterator it = streamLastActive.begin(); + it != streamLastActive.end(); ++it){ + if (now - it->second < STATS_DELAY){ + if (!thisStream.size()){ + thisStream = it->first; + }else if (thisStream != it->first){ + thisStream = ""; + break; + } + } + } + sessions.setStream(thisStream); + } + } + // Retrigger USER_NEW if a re-sync was requested if (!thisType && forceTrigger){ forceTrigger = false; + std::string host; + Socket::hostBytesToStr(thisHost.data(), 16, host); if (Triggers::shouldTrigger("USER_NEW", thisStreamName)){ INFO_MSG("Triggering USER_NEW for stream %s", thisStreamName.c_str()); - std::string payload = thisStreamName + "\n" + thisHost + "\n" + - thisSid + "\n" + thisProtocol + + std::string payload = thisStreamName + "\n" + host + "\n" + + thisToken + "\n" + thisProtocol + "\n" + thisReqUrl + "\n" + thisSessionId; if (!Triggers::doTrigger("USER_NEW", payload, thisStreamName)){ INFO_MSG("USER_NEW rejected stream %s", thisStreamName.c_str()); + Util::logExitReason("Session rejected by USER_NEW"); connections.setExit(); connections.finishAll(); + break; }else{ INFO_MSG("USER_NEW accepted stream %s", thisStreamName.c_str()); } } } - // Invalidate connections if the session is marked as invalid - if(connections.getExit()){ - connections.finishAll(); - break; - } // Remember latest activity so we know when this session ends if (currentConnections){ - lastSeen = Util::bootSecs(); } - Util::sleep(1000); + Util::wait(1000); + } + if (Util::bootSecs() - lastSeen > STATS_DELAY){ + Util::logExitReason("Session inactive for %d seconds", STATS_DELAY); } // Trigger USER_END if (!thisType && Triggers::shouldTrigger("USER_END", thisStreamName)){ - lastSecond = 0; - time = 0; - down = 0; - up = 0; - - // Get a final summary of this session - for (uint64_t idx = 0; idx < connections.recordCount(); idx++){ - if (connections.getStatus(idx) == COMM_STATUS_INVALID || connections.getStatus(idx) & COMM_STATUS_DISCONNECT){continue;} - uint64_t thisLastSecond = connections.getLastSecond(idx); - // Set last second to the latest entry - if (thisLastSecond > lastSecond){ - lastSecond = thisLastSecond; - } - // Count protocol durations across the entire session - std::string thisConnector = connections.getConnector(idx); - if (thisConnector != ""){ - connectorCount[thisConnector] += connections.getTime(idx); - } - // Sum all other variables - time += connections.getTime(idx); - down += connections.getDown(idx); - up += connections.getUp(idx); - } - // Convert connector duration to string + // Convert connector, host and stream into lists and counts std::stringstream connectorSummary; - bool addDelimiter = false; - connectorSummary << "{"; - for (std::map::iterator it = connectorCount.begin(); - it != connectorCount.end(); ++it){ - connectorSummary << (addDelimiter ? "," : "") << it->first << ":" << it->second; - addDelimiter = true; + std::stringstream connectorTimes; + for (std::map::iterator it = connectorCount.begin(); it != connectorCount.end(); ++it){ + connectorSummary << (connectorSummary.str().size() ? "," : "") << it->first; + connectorTimes << (connectorTimes.str().size() ? "," : "") << it->second; + } + std::stringstream hostSummary; + std::stringstream hostTimes; + for (std::map::iterator it = hostCount.begin(); it != hostCount.end(); ++it){ + std::string host; + Socket::hostBytesToStr(it->first.data(), 16, host); + hostSummary << (hostSummary.str().size() ? "," : "") << host; + hostTimes << (hostTimes.str().size() ? "," : "") << it->second; + } + std::stringstream streamSummary; + std::stringstream streamTimes; + for (std::map::iterator it = streamCount.begin(); it != streamCount.end(); ++it){ + streamSummary << (streamSummary.str().size() ? "," : "") << it->first; + streamTimes << (streamTimes.str().size() ? "," : "") << it->second; } - connectorSummary << "}"; - const uint64_t duration = lastSecond - (bootTime / 1000); std::stringstream summary; - summary << thisSessionId << "\n" - << thisStreamName << "\n" + summary << thisToken << "\n" + << streamSummary.str() << "\n" << connectorSummary.str() << "\n" - << thisHost << "\n" - << duration << "\n" - << up << "\n" - << down << "\n" - << sessions.getTags(); + << hostSummary.str() << "\n" + << globalTime << "\n" + << globalUp << "\n" + << globalDown << "\n" + << sessions.getTags() << "\n" + << hostTimes.str() << "\n" + << connectorTimes.str() << "\n" + << streamTimes.str() << "\n" + << thisSessionId; Triggers::doTrigger("USER_END", summary.str(), thisStreamName); } if (!thisType && connections.getExit()){ - WARN_MSG("Session %s has been invalidated since it is not allowed to view stream %s", thisSessionId.c_str(), thisStreamName.c_str()); uint64_t sleepStart = Util::bootSecs(); // Keep session invalidated for 10 minutes, or until the session stops - while (config.is_active && sleepStart - Util::bootSecs() < 600){ + while (config.is_active && Util::bootSecs() - sleepStart < SESS_TIMEOUT){ Util::sleep(1000); + if (forceTrigger){break;} } } - INFO_MSG("Shutting down session %s", thisSessionId.c_str()); + INFO_MSG("Shutting down session %s: %s", thisSessionId.c_str(), Util::exitReason); return 0; } -- 2.25.1 From 1a4a526a112d0bdeaa0f269a133fe55c39a0d129 Mon Sep 17 00:00:00 2001 From: Cat Date: Wed, 5 Oct 2022 03:10:04 +0200 Subject: [PATCH 28/38] LSP: Moved some settings to new "General" tab, added bitmask inputtype, removed LTSonly code --- lsp/main.css | 30 +++ lsp/minified.js | 396 ++++++++++++++++---------------- lsp/mist.js | 582 +++++++++++++++++++++++++++++------------------- 3 files changed, 590 insertions(+), 418 deletions(-) diff --git a/lsp/main.css b/lsp/main.css index 6c72e6a7..9289d211 100644 --- a/lsp/main.css +++ b/lsp/main.css @@ -279,6 +279,21 @@ main > button { width: 1.5em; flex-grow: 0; } +.input_container .field_container .field.bitmask { + display: flex; + flex-flow: row wrap; + justify-content: space-between; + padding: 0; + margin-bottom: 0.5em; +} +.input_container .field_container .field.bitmask > label { + display: flex; + flex-flow: row nowrap; + align-items: center; + margin-right: -0.25em; +} +.input_container .field_container .field.bitmask > label > input { margin: 0; } +.input_container .field_container .field.bitmask > label > span { margin: 0 0.25em; } textarea { font-size: 0.8em; } @@ -349,6 +364,20 @@ select option[value=""] { display: block; z-index: 0; } +.input_container label.active { position: relative; } +.input_container label.active:after { + content: ""; + display: block; + position: absolute; + z-index: -1; + top: -0.25em; + left: -0.25em; + width: 55.5em; + bottom: -0.25em; + background: linear-gradient(to right,#9ec2da,transparent 55em); + opacity: 0.3; + filter: blur(0.1em); +} .input_container label.active .ih_balloon, .input_container > .help_container .ih_balloon { display: block; @@ -577,6 +606,7 @@ table.valigntop th { } .input_container .field_container .field.inputlist > * { width: 100%; + box-sizing: border-box; } .input_container .field_container .field.checkcontainer { font-size: 0.8em; diff --git a/lsp/minified.js b/lsp/minified.js index 52a6f6bc..9db52d63 100644 --- a/lsp/minified.js +++ b/lsp/minified.js @@ -1,11 +1,11 @@ -var MD5=function(a){function b(a,b){var c,d,g,i,e;g=a&2147483648;i=b&2147483648;c=a&1073741824;d=b&1073741824;e=(a&1073741823)+(b&1073741823);return c&d?e^2147483648^g^i:c|d?e&1073741824?e^3221225472^g^i:e^1073741824^g^i:e^g^i}function c(a,c,d,g,i,e,f){a=b(a,b(b(c&d|~c&g,i),f));return b(a<>>32-e,c)}function d(a,c,d,g,i,e,f){a=b(a,b(b(c&g|d&~g,i),f));return b(a<>>32-e,c)}function e(a,c,d,g,e,i,f){a=b(a,b(b(c^d^g,e),f));return b(a<>>32-i,c)}function l(a,c,d,g,i,e,f){a=b(a,b(b(d^(c|~g), -i),f));return b(a<>>32-e,c)}function m(a){var b="",c="",d;for(d=0;3>=d;d++)c=a>>>8*d&255,c="0"+c.toString(16),b+=c.substr(c.length-2,2);return b}var f=[],t,o,k,w,h,g,i,j,f=a.replace(/\r\n/g,"\n"),a="";for(t=0;to?a+=String.fromCharCode(o):(127o?a+=String.fromCharCode(o>>6|192):(a+=String.fromCharCode(o>>12|224),a+=String.fromCharCode(o>>6&63|128)),a+=String.fromCharCode(o&63|128));f=a;a=f.length;t=a+8;o=16*((t-t%64)/64+1);k=Array(o-1);for(h=w=0;h>>29;f=k;h=1732584193;g=4023233417;i=2562383102;j=271733878;for(a=0;ac?1*d:a .menu"),main:$("main"),header:$("header"),connection:{status:$("#connection"),user_and_host:$("#user_and_host"),msg:$("#message")}};UI.buildMenu();UI.stored.getOpts();try{if("mistLogin"in sessionStorage){var a=JSON.parse(sessionStorage.mistLogin);mist.user.name=a.name;mist.user.password=a.password;mist.user.host=a.host}}catch(b){}location.hash&&(a=decodeURIComponent(location.hash).substring(1).split("@")[0].split("&"),mist.user.name=a[0],a[1]&&(mist.user.host= +var MD5=function(a){function b(a,b){var c,d,g,f,e;g=a&2147483648;f=b&2147483648;c=a&1073741824;d=b&1073741824;e=(a&1073741823)+(b&1073741823);return c&d?e^2147483648^g^f:c|d?e&1073741824?e^3221225472^g^f:e^1073741824^g^f:e^g^f}function c(a,c,d,g,f,e,h){a=b(a,b(b(c&d|~c&g,f),h));return b(a<>>32-e,c)}function d(a,c,d,g,e,f,h){a=b(a,b(b(c&g|d&~g,e),h));return b(a<>>32-f,c)}function e(a,c,d,g,f,e,h){a=b(a,b(b(c^d^g,f),h));return b(a<>>32-e,c)}function l(a,c,d,g,e,f,h){a=b(a,b(b(d^(c|~g), +e),h));return b(a<>>32-f,c)}function n(a){var b="",c="",d;for(d=0;3>=d;d++)c=a>>>8*d&255,c="0"+c.toString(16),b+=c.substr(c.length-2,2);return b}var h=[],t,m,k,w,i,g,f,j,h=a.replace(/\r\n/g,"\n"),a="";for(t=0;tm?a+=String.fromCharCode(m):(127m?a+=String.fromCharCode(m>>6|192):(a+=String.fromCharCode(m>>12|224),a+=String.fromCharCode(m>>6&63|128)),a+=String.fromCharCode(m&63|128));h=a;a=h.length;t=a+8;m=16*((t-t%64)/64+1);k=Array(m-1);for(i=w=0;i>>29;h=k;i=1732584193;g=4023233417;f=2562383102;j=271733878;for(a=0;ac?1*d:a .menu"),main:$("main"),header:$("header"),connection:{status:$("#connection"),user_and_host:$("#user_and_host"),msg:$("#message")}};UI.buildMenu();UI.stored.getOpts();try{if("mistLogin"in sessionStorage){var a=JSON.parse(sessionStorage.mistLogin);mist.user.name=a.name;mist.user.password=a.password;mist.user.host=a.host}}catch(b){}location.hash&&(a=decodeURIComponent(location.hash).substring(1).split("@")[0].split("&"),mist.user.name=a[0],a[1]&&(mist.user.host= a[1]));mist.send(function(){$(window).trigger("hashchange")},{},{timeout:5,hide:!0});var c=0;$("body > div.filler").on("scroll",function(){var a=$(this).scrollLeft();a!=c&&UI.elements.header.css("margin-right",-1*a+"px");c=a})});$(window).on("hashchange",function(){var a=decodeURIComponent(location.hash).substring(1).split("@");a[1]||(a[1]="");a=a[1].split("&");""==a[0]&&(a[0]="Overview");UI.showTab(a[0],a[1])}); var MistVideoObject={},otherhost={host:!1,https:!1},UI={debug:!1,elements:{},stored:{getOpts:function(){var a=localStorage.stored;a&&(a=JSON.parse(a));$.extend(!0,this.vars,a);return this.vars},saveOpt:function(a,b){this.vars[a]=b;localStorage.stored=JSON.stringify(this.vars);return this.vars},vars:{helpme:!0}},interval:{clear:function(){"undefined"!=typeof this.opts&&(clearInterval(this.opts.id),delete this.opts)},set:function(a,b){this.opts&&log("[interval]","Set called on interval, but an interval is already active."); this.opts={delay:b,callback:a};this.opts.id=setInterval(a,b)}},returnTab:["Overview"],countrylist:{AF:"Afghanistan",AX:"Åland Islands",AL:"Albania",DZ:"Algeria",AS:"American Samoa",AD:"Andorra",AO:"Angola",AI:"Anguilla",AQ:"Antarctica",AG:"Antigua and Barbuda",AR:"Argentina",AM:"Armenia",AW:"Aruba",AU:"Australia",AT:"Austria",AZ:"Azerbaijan",BS:"Bahamas",BH:"Bahrain",BD:"Bangladesh",BB:"Barbados",BY:"Belarus",BE:"Belgium",BZ:"Belize",BJ:"Benin",BM:"Bermuda",BT:"Bhutan",BO:"Bolivia, Plurinational State of", @@ -20,61 +20,61 @@ TV:"Tuvalu",UG:"Uganda",UA:"Ukraine",AE:"United Arab Emirates",GB:"United Kingdo clearTimeout(this.hiding);delete this.hiding;var c=$(document).height()-$tooltip.outerHeight(),d=$(document).width()-$tooltip.outerWidth();$tooltip.css("left",Math.min(a.pageX+10,d-10));$tooltip.css("top",Math.min(a.pageY+25,c-10));$tooltip.show().addClass("show")},hide:function(){$tooltip=this.element;$tooltip.removeClass("show");this.hiding=setTimeout(function(){$tooltip.hide()},500)},element:$("
").attr("id","tooltip")},humanMime:function(a){var b=!1;switch(a){case "html5/application/vnd.apple.mpegurl":b= "HLS (TS)";break;case "html5/application/vnd.apple.mpegurl;version=7":b="HLS (CMAF)";break;case "html5/video/webm":b="WebM";break;case "html5/video/mp4":b="MP4";break;case "dash/video/mp4":b="DASH";break;case "flash/11":b="HDS";break;case "flash/10":b="RTMP";break;case "flash/7":b="Progressive";break;case "html5/audio/mp3":b="MP3";break;case "html5/audio/wav":b="WAV";break;case "html5/video/mp2t":case "html5/video/mpeg":b="TS";break;case "html5/application/vnd.ms-sstr+xml":case "html5/application/vnd.ms-ss":b= "Smooth Streaming";break;case "html5/text/vtt":b="VTT Subtitles";break;case "html5/text/plain":b="SRT Subtitles";break;case "html5/text/javascript":b="JSON Subtitles";break;case "rtsp":b="RTSP";break;case "webrtc":b="WebRTC"}return b},popup:{element:null,show:function(a){this.element=$("
").attr("id","popup").append($("