curl: Fix librtmp variant (#40713)

* rtmpdump: New package

* curl: Fix librtmp variant

Add the previously missing dependency required for rtmp support.

The variant has been broken since its addition in PR #25166.

Fixes one of the two issues reported in #26887.

7 files changed, 394 insertions, 0 deletions

diff --git a/var/spack/repos/builtin/packages/curl/package.py b/var/spack/repos/builtin/packages/curl/package.py
index 46894046df..362b559ab6 100644
--- a/var/spack/repos/builtin/packages/curl/package.py
+++ b/var/spack/repos/builtin/packages/curl/package.py
@@ -305,6 +305,7 @@ class Curl(NMakePackage, AutotoolsPackage):
     depends_on("libssh2", when="+libssh2")
     depends_on("libssh", when="+libssh")
     depends_on("krb5", when="+gssapi")
+    depends_on("rtmpdump", when="+librtmp")
 
     # https://github.com/curl/curl/pull/9054
     patch("easy-lock-sched-header.patch", when="@7.84.0")
diff --git a/var/spack/repos/builtin/packages/rtmpdump/missing-include.patch b/var/spack/repos/builtin/packages/rtmpdump/missing-include.patch
new file mode 100644
index 0000000000..4325ed0738
--- /dev/null
+++ b/var/spack/repos/builtin/packages/rtmpdump/missing-include.patch
@@ -0,0 +1,23 @@
+https://bugs.gentoo.org/828082
+--- a/librtmp/rtmp.c
++++ b/librtmp/rtmp.c
+@@ -28,6 +28,7 @@
+ #include <string.h>
+ #include <assert.h>
+ #include <time.h>
++#include <limits.h>
+ 
+ #include "rtmp_sys.h"
+ #include "log.h"
+diff --git a/librtmp/hashswf.c b/librtmp/hashswf.c
+index 32b2eed..e3669e3 100644
+--- a/librtmp/hashswf.c
++++ b/librtmp/hashswf.c
+@@ -25,6 +25,7 @@
+ #include <string.h>
+ #include <ctype.h>
+ #include <time.h>
++#include <limits.h>
+ 
+ #include "rtmp_sys.h"
+ #include "log.h"
diff --git a/var/spack/repos/builtin/packages/rtmpdump/package.py b/var/spack/repos/builtin/packages/rtmpdump/package.py
new file mode 100644
index 0000000000..a868e6e3d0
--- /dev/null
+++ b/var/spack/repos/builtin/packages/rtmpdump/package.py
@@ -0,0 +1,38 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class Rtmpdump(MakefilePackage):
+    """rtmpdump is a toolkit for RTMP streams."""
+
+    homepage = "https://rtmpdump.mplayerhq.hu/"
+    git = "https://git.ffmpeg.org/rtmpdump.git"
+
+    maintainers("tobbez")
+
+    license("GPL-2.0-or-later")
+
+    version("2021-02-19", commit="f1b83c10d8beb43fcc70a6e88cf4325499f25857")
+
+    variant("tls", default="openssl", description="TLS backend", values=("gnutls", "openssl"))
+
+    depends_on("openssl@:3", when="tls=openssl")
+    depends_on("gnutls", when="tls=gnutls")
+    depends_on("zlib-api")
+
+    patch("missing-include.patch")
+    patch("rtmpdump-fix-chunk-size.patch")
+    patch("rtmpdump-openssl-1.1-v2.patch")
+    patch("rtmpdump-swf_vertification_type_2.patch")
+    patch("rtmpdump-swf_vertification_type_2_part_2.patch")
+
+    @property
+    def build_targets(self):
+        return [f"CRYPTO={self.spec.variants['tls'].value.upper()}"]
+
+    def install(self, spec, prefix):
+        make("install", f"prefix={prefix}", "sbindir=$(bindir)")
diff --git a/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-fix-chunk-size.patch b/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-fix-chunk-size.patch
new file mode 100644
index 0000000000..1c6cfdc626
--- /dev/null
+++ b/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-fix-chunk-size.patch
@@ -0,0 +1,48 @@
+https://git.alpinelinux.org/aports/commit/main/rtmpdump/fix-chunk-size.patch?id=bf39fb1177ee77eee6c214a7393cc0054958ce08
+https://git.alpinelinux.org/aports/commit/main/rtmpdump/fix-chunk-size.patch?id=69bc162319b12e9b6c6d3ea345dbf7c218753594
+diff --git a/librtmp/rtmp.c b/librtmp/rtmp.c
+index a2863b0..ac1b3be 100644
+--- a/librtmp/rtmp.c
++++ b/librtmp/rtmp.c
+@@ -2077,6 +2077,29 @@ RTMP_SendClientBW(RTMP *r)
+ }
+ 
+ static int
++SendClientChunkSize(RTMP *r, int chunkSize)
++{
++  RTMPPacket packet;
++  char pbuf[256], *pend = pbuf + sizeof(pbuf);
++  int ret;
++
++  packet.m_nChannel = 0x02;	/* control channel (invoke) */
++  packet.m_headerType = RTMP_PACKET_SIZE_LARGE;
++  packet.m_packetType = RTMP_PACKET_TYPE_CHUNK_SIZE;
++  packet.m_nTimeStamp = 0;
++  packet.m_nInfoField2 = 0;
++  packet.m_hasAbsTimestamp = 0;
++  packet.m_body = pbuf + RTMP_MAX_HEADER_SIZE;
++
++  packet.m_nBodySize = 4;
++
++  AMF_EncodeInt32(packet.m_body, pend, chunkSize);
++  ret = RTMP_SendPacket(r, &packet, FALSE);
++  r->m_outChunkSize = chunkSize;
++  return ret;
++}
++
++static int
+ SendBytesReceived(RTMP *r)
+ {
+   RTMPPacket packet;
+@@ -3349,6 +3372,11 @@ HandleChangeChunkSize(RTMP *r, const RTMPPacket *packet)
+       r->m_inChunkSize = AMF_DecodeInt32(packet->m_body);
+       RTMP_Log(RTMP_LOGDEBUG, "%s, received: chunk size change to %d", __FUNCTION__,
+ 	  r->m_inChunkSize);
++      if (r->Link.protocol & RTMP_FEATURE_WRITE)
++        {
++          RTMP_Log(RTMP_LOGDEBUG, "%s, updating outChunkSize too", __FUNCTION__);
++          SendClientChunkSize(r, r->m_inChunkSize);
++        }
+     }
+ }
+ 
diff --git a/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-openssl-1.1-v2.patch b/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-openssl-1.1-v2.patch
new file mode 100644
index 0000000000..146243bd11
--- /dev/null
+++ b/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-openssl-1.1-v2.patch
@@ -0,0 +1,248 @@
+https://raw.githubusercontent.com/xbmc/inputstream.rtmp/master/depends/common/librtmp/0003-openssl-1.1.patch
+See also https://github.com/xbmc/inputstream.rtmp/pull/46
+--- a/librtmp/dh.h
++++ b/librtmp/dh.h
+@@ -253,20 +253,42 @@
+   if (!dh)
+     goto failed;
+ 
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   MP_new(dh->g);
+ 
+   if (!dh->g)
+     goto failed;
++#else
++  BIGNUM *g = NULL;
++  MP_new(g);
++  if (!g)
++    goto failed;
++#endif
+ 
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   MP_gethex(dh->p, P1024, res);	/* prime P1024, see dhgroups.h */
++#else
++  BIGNUM* p = NULL;
++  DH_get0_pqg(dh, (BIGNUM const**)&p, NULL, NULL);
++  MP_gethex(p, P1024, res); /* prime P1024, see dhgroups.h */
++#endif
+   if (!res)
+     {
+       goto failed;
+     }
+ 
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   MP_set_w(dh->g, 2);	/* base 2 */
++#else
++  MP_set_w(g, 2);   /* base 2 */
++  DH_set0_pqg(dh, p, NULL, g);
++#endif
+ 
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   dh->length = nKeyBits;
++#else
++  DH_set_length(dh, nKeyBits);
++#endif
+   return dh;
+ 
+ failed:
+@@ -293,12 +315,24 @@
+       MP_gethex(q1, Q1024, res);
+       assert(res);
+ 
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+       res = isValidPublicKey(dh->pub_key, dh->p, q1);
++#else
++      BIGNUM const* pub_key = NULL;
++      BIGNUM const* p = NULL;
++      DH_get0_key(dh, &pub_key, NULL);
++      DH_get0_pqg(dh, &p, NULL, NULL);
++      res = isValidPublicKey((BIGNUM*)pub_key, (BIGNUM*)p, q1);
++#endif
+       if (!res)
+ 	{
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+ 	  MP_free(dh->pub_key);
+ 	  MP_free(dh->priv_key);
+ 	  dh->pub_key = dh->priv_key = 0;
++#else
++          DH_free(dh);
++#endif
+ 	}
+ 
+       MP_free(q1);
+@@ -314,15 +348,29 @@
+ DHGetPublicKey(MDH *dh, uint8_t *pubkey, size_t nPubkeyLen)
+ {
+   int len;
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   if (!dh || !dh->pub_key)
++#else
++  BIGNUM const* pub_key = NULL;
++  DH_get0_key(dh, &pub_key, NULL);
++  if (!dh || !pub_key)
++#endif
+     return 0;
+ 
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   len = MP_bytes(dh->pub_key);
++#else
++  len = MP_bytes(pub_key);
++#endif
+   if (len <= 0 || len > (int) nPubkeyLen)
+     return 0;
+ 
+   memset(pubkey, 0, nPubkeyLen);
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   MP_setbin(dh->pub_key, pubkey + (nPubkeyLen - len), len);
++#else
++  MP_setbin(pub_key, pubkey + (nPubkeyLen - len), len);
++#endif
+   return 1;
+ }
+ 
+@@ -364,7 +412,13 @@
+   MP_gethex(q1, Q1024, len);
+   assert(len);
+ 
++#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L
+   if (isValidPublicKey(pubkeyBn, dh->p, q1))
++#else
++  BIGNUM const* p = NULL;
++  DH_get0_pqg(dh, &p, NULL, NULL);
++  if (isValidPublicKey(pubkeyBn, (BIGNUM*)p, q1))
++#endif
+     res = MDH_compute_key(secret, nPubkeyLen, pubkeyBn, dh);
+   else
+     res = -1;
+--- a/librtmp/handshake.h
++++ b/librtmp/handshake.h
+@@ -31,9 +31,9 @@
+ #define SHA256_DIGEST_LENGTH	32
+ #endif
+ #define HMAC_CTX	sha2_context
+-#define HMAC_setup(ctx, key, len)	sha2_hmac_starts(&ctx, (unsigned char *)key, len, 0)
+-#define HMAC_crunch(ctx, buf, len)	sha2_hmac_update(&ctx, buf, len)
+-#define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; sha2_hmac_finish(&ctx, dig)
++#define HMAC_setup(ctx, key, len)	sha2_hmac_starts(ctx, (unsigned char *)key, len, 0)
++#define HMAC_crunch(ctx, buf, len)	sha2_hmac_update(ctx, buf, len)
++#define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; sha2_hmac_finish(ctx, dig)
+ 
+ typedef arc4_context *	RC4_handle;
+ #define RC4_alloc(h)	*h = malloc(sizeof(arc4_context))
+@@ -50,9 +50,9 @@
+ #endif
+ #undef HMAC_CTX
+ #define HMAC_CTX	struct hmac_sha256_ctx
+-#define HMAC_setup(ctx, key, len)	hmac_sha256_set_key(&ctx, len, key)
+-#define HMAC_crunch(ctx, buf, len)	hmac_sha256_update(&ctx, len, buf)
+-#define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig)
++#define HMAC_setup(ctx, key, len)	hmac_sha256_set_key(ctx, len, key)
++#define HMAC_crunch(ctx, buf, len)	hmac_sha256_update(ctx, len, buf)
++#define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(ctx, SHA256_DIGEST_LENGTH, dig)
+ #define HMAC_close(ctx)
+ 
+ typedef struct arcfour_ctx*	RC4_handle;
+@@ -64,14 +64,23 @@
+ 
+ #else	/* USE_OPENSSL */
+ #include <openssl/sha.h>
++#include <openssl/ossl_typ.h>
+ #include <openssl/hmac.h>
+ #include <openssl/rc4.h>
+ #if OPENSSL_VERSION_NUMBER < 0x0090800 || !defined(SHA256_DIGEST_LENGTH)
+ #error Your OpenSSL is too old, need 0.9.8 or newer with SHA256
+ #endif
+-#define HMAC_setup(ctx, key, len)	HMAC_CTX_init(&ctx); HMAC_Init_ex(&ctx, key, len, EVP_sha256(), 0)
+-#define HMAC_crunch(ctx, buf, len)	HMAC_Update(&ctx, buf, len)
+-#define HMAC_finish(ctx, dig, dlen)	HMAC_Final(&ctx, dig, &dlen); HMAC_CTX_cleanup(&ctx)
++#if OPENSSL_VERSION_NUMBER < 0x10100000L
++#define HMAC_setup(ctx, key, len)	HMAC_CTX_init(ctx); HMAC_Init_ex(ctx, key, len, EVP_sha256(), 0)
++#else
++#define HMAC_setup(ctx, key, len)	ctx = HMAC_CTX_new(); HMAC_CTX_reset(ctx); HMAC_Init_ex(ctx, key, len, EVP_sha256(), 0)
++#endif
++#define HMAC_crunch(ctx, buf, len)	HMAC_Update(ctx, buf, len)
++#if OPENSSL_VERSION_NUMBER < 0x10100000L
++#define HMAC_finish(ctx, dig, dlen)	HMAC_Final(ctx, dig, &dlen); HMAC_CTX_cleanup(ctx)
++#else
++#define HMAC_finish(ctx, dig, dlen)     HMAC_Final(ctx, dig, &dlen); HMAC_CTX_free(ctx)
++#endif
+ 
+ typedef RC4_KEY *	RC4_handle;
+ #define RC4_alloc(h)	*h = malloc(sizeof(RC4_KEY))
+@@ -117,7 +126,7 @@
+ {
+   uint8_t digest[SHA256_DIGEST_LENGTH];
+   unsigned int digestLen = 0;
+-  HMAC_CTX ctx;
++  HMAC_CTX* ctx = NULL;
+ 
+   RC4_alloc(rc4keyIn);
+   RC4_alloc(rc4keyOut);
+@@ -266,7 +275,7 @@
+ 	   size_t keylen, uint8_t *digest)
+ {
+   unsigned int digestLen;
+-  HMAC_CTX ctx;
++  HMAC_CTX* ctx = NULL;
+ 
+   HMAC_setup(ctx, key, keylen);
+   HMAC_crunch(ctx, message, messageLen);
+--- a/librtmp/hashswf.c
++++ b/librtmp/hashswf.c
+@@ -37,9 +37,9 @@
+ #define SHA256_DIGEST_LENGTH	32
+ #endif
+ #define HMAC_CTX	sha2_context
+-#define HMAC_setup(ctx, key, len)	sha2_hmac_starts(&ctx, (unsigned char *)key, len, 0)
+-#define HMAC_crunch(ctx, buf, len)	sha2_hmac_update(&ctx, buf, len)
+-#define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; sha2_hmac_finish(&ctx, dig)
++#define HMAC_setup(ctx, key, len)	sha2_hmac_starts(ctx, (unsigned char *)key, len, 0)
++#define HMAC_crunch(ctx, buf, len)	sha2_hmac_update(ctx, buf, len)
++#define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; sha2_hmac_finish(ctx, dig)
+ #define HMAC_close(ctx)
+ #elif defined(USE_GNUTLS)
+ #include <nettle/hmac.h>
+@@ -48,19 +48,27 @@
+ #endif
+ #undef HMAC_CTX
+ #define HMAC_CTX	struct hmac_sha256_ctx
+-#define HMAC_setup(ctx, key, len)	hmac_sha256_set_key(&ctx, len, key)
+-#define HMAC_crunch(ctx, buf, len)	hmac_sha256_update(&ctx, len, buf)
+-#define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig)
++#define HMAC_setup(ctx, key, len)	hmac_sha256_set_key(ctx, len, key)
++#define HMAC_crunch(ctx, buf, len)	hmac_sha256_update(ctx, len, buf)
++#define HMAC_finish(ctx, dig, dlen)	dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(ctx, SHA256_DIGEST_LENGTH, dig)
+ #define HMAC_close(ctx)
+ #else	/* USE_OPENSSL */
+ #include <openssl/ssl.h>
+ #include <openssl/sha.h>
+ #include <openssl/hmac.h>
+ #include <openssl/rc4.h>
+-#define HMAC_setup(ctx, key, len)	HMAC_CTX_init(&ctx); HMAC_Init_ex(&ctx, (unsigned char *)key, len, EVP_sha256(), 0)
+-#define HMAC_crunch(ctx, buf, len)	HMAC_Update(&ctx, (unsigned char *)buf, len)
+-#define HMAC_finish(ctx, dig, dlen)	HMAC_Final(&ctx, (unsigned char *)dig, &dlen);
+-#define HMAC_close(ctx)	HMAC_CTX_cleanup(&ctx)
++#if OPENSSL_VERSION_NUMBER < 0x10100000L
++#define HMAC_setup(ctx, key, len)	HMAC_CTX_init(ctx); HMAC_Init_ex(ctx, (unsigned char *)key, len, EVP_sha256(), 0)
++#else
++#define HMAC_setup(ctx, key, len)	ctx = HMAC_CTX_new(); HMAC_CTX_reset(ctx); HMAC_Init_ex(ctx, key, len, EVP_sha256(), 0)
++#endif
++#define HMAC_crunch(ctx, buf, len)	HMAC_Update(ctx, (unsigned char *)buf, len)
++#define HMAC_finish(ctx, dig, dlen)	HMAC_Final(ctx, (unsigned char *)dig, &dlen);
++#if OPENSSL_VERSION_NUMBER < 0x10100000L
++#define HMAC_close(ctx)	HMAC_CTX_cleanup(ctx)
++#else
++#define HMAC_close(ctx) HMAC_CTX_reset(ctx); HMAC_CTX_free(ctx)
++#endif
+ #endif
+ 
+ extern void RTMP_TLS_Init();
+@@ -289,7 +297,7 @@
+ struct info
+ {
+   z_stream *zs;
+-  HMAC_CTX ctx;
++  HMAC_CTX *ctx;
+   int first;
+   int zlib;
+   int size;
diff --git a/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-swf_vertification_type_2.patch b/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-swf_vertification_type_2.patch
new file mode 100644
index 0000000000..cc7637d849
--- /dev/null
+++ b/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-swf_vertification_type_2.patch
@@ -0,0 +1,14 @@
+https://bugs.gentoo.org/669574
+diff --git a/librtmp/rtmp.c b/librtmp/rtmp.c
+index 5311a8a..79fefae 100644
+--- a/librtmp/rtmp.c
++++ b/librtmp/rtmp.c
+@@ -2854,7 +2854,7 @@ HandleCtrl(RTMP *r, const RTMPPacket *packet)
+   if (nType == 0x1A)
+     {
+       RTMP_Log(RTMP_LOGDEBUG, "%s, SWFVerification ping received: ", __FUNCTION__);
+-      if (packet->m_nBodySize > 2 && packet->m_body[2] > 0x01)
++      if (packet->m_nBodySize > 2 && packet->m_body[2] > 0x02)
+ 	{
+ 	  RTMP_Log(RTMP_LOGERROR,
+             "%s: SWFVerification Type %d request not supported! Patches welcome...",
diff --git a/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-swf_vertification_type_2_part_2.patch b/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-swf_vertification_type_2_part_2.patch
new file mode 100644
index 0000000000..ade0d9baa7
--- /dev/null
+++ b/var/spack/repos/builtin/packages/rtmpdump/rtmpdump-swf_vertification_type_2_part_2.patch
@@ -0,0 +1,22 @@
+https://bugs.gentoo.org/669574
+diff --git a/librtmp/rtmp.c b/librtmp/rtmp.c
+index df2cb27..b72dc64 100644
+--- a/librtmp/rtmp.c
++++ b/librtmp/rtmp.c
+@@ -2857,14 +2857,14 @@ HandleCtrl(RTMP *r, const RTMPPacket *packet)
+       if (packet->m_nBodySize > 2 && packet->m_body[2] > 0x01)
+ 	{
+ 	  RTMP_Log(RTMP_LOGERROR,
+-            "%s: SWFVerification Type %d request not supported! Patches welcome...",
++            "%s: SWFVerification Type %d request not supported, attempting to use SWFVerification Type 1! Patches welcome...",
+ 	    __FUNCTION__, packet->m_body[2]);
+ 	}
+ #ifdef CRYPTO
+       /*RTMP_LogHex(packet.m_body, packet.m_nBodySize); */
+ 
+       /* respond with HMAC SHA256 of decompressed SWF, key is the 30byte player key, also the last 30 bytes of the server handshake are applied */
+-      else if (r->Link.SWFSize)
++      if (r->Link.SWFSize)
+ 	{
+ 	  RTMP_SendCtrl(r, 0x1B, 0, 0);
+ 	}