Another Great Gentoo Hackā¢. "Upstream": https://gitweb.gentoo.org/repo/gentoo.git/plain/media-video/rtmpdump/files/rtmpdump-openssl-1.1-v2.patch Originally from: https://raw.githubusercontent.com/xbmc/inputstream.rtmp/master/depends/common/librtmp/0003-openssl-1.1.patch See also: https://github.com/xbmc/inputstream.rtmp/pull/46 --- a/librtmp/dh.h +++ b/librtmp/dh.h @@ -253,20 +253,42 @@ if (!dh) goto failed; +#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L MP_new(dh->g); if (!dh->g) goto failed; +#else + BIGNUM *g = NULL; + MP_new(g); + if (!g) + goto failed; +#endif +#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L MP_gethex(dh->p, P1024, res); /* prime P1024, see dhgroups.h */ +#else + BIGNUM* p = NULL; + DH_get0_pqg(dh, (BIGNUM const**)&p, NULL, NULL); + MP_gethex(p, P1024, res); /* prime P1024, see dhgroups.h */ +#endif if (!res) { goto failed; } +#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L MP_set_w(dh->g, 2); /* base 2 */ +#else + MP_set_w(g, 2); /* base 2 */ + DH_set0_pqg(dh, p, NULL, g); +#endif +#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L dh->length = nKeyBits; +#else + DH_set_length(dh, nKeyBits); +#endif return dh; failed: @@ -293,12 +315,24 @@ MP_gethex(q1, Q1024, res); assert(res); +#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L res = isValidPublicKey(dh->pub_key, dh->p, q1); +#else + BIGNUM const* pub_key = NULL; + BIGNUM const* p = NULL; + DH_get0_key(dh, &pub_key, NULL); + DH_get0_pqg(dh, &p, NULL, NULL); + res = isValidPublicKey((BIGNUM*)pub_key, (BIGNUM*)p, q1); +#endif if (!res) { +#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L MP_free(dh->pub_key); MP_free(dh->priv_key); dh->pub_key = dh->priv_key = 0; +#else + DH_free(dh); +#endif } MP_free(q1); @@ -314,15 +348,29 @@ DHGetPublicKey(MDH *dh, uint8_t *pubkey, size_t nPubkeyLen) { int len; +#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L if (!dh || !dh->pub_key) +#else + BIGNUM const* pub_key = NULL; + DH_get0_key(dh, &pub_key, NULL); + if (!dh || !pub_key) +#endif return 0; +#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L len = MP_bytes(dh->pub_key); +#else + len = MP_bytes(pub_key); +#endif if (len <= 0 || len > (int) nPubkeyLen) return 0; memset(pubkey, 0, nPubkeyLen); +#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L MP_setbin(dh->pub_key, pubkey + (nPubkeyLen - len), len); +#else + MP_setbin(pub_key, pubkey + (nPubkeyLen - len), len); +#endif return 1; } @@ -364,7 +412,13 @@ MP_gethex(q1, Q1024, len); assert(len); +#if !defined(USE_OPENSSL) || !defined(OPENSSL_VERSION_NUMBER) || OPENSSL_VERSION_NUMBER < 0x10100000L if (isValidPublicKey(pubkeyBn, dh->p, q1)) +#else + BIGNUM const* p = NULL; + DH_get0_pqg(dh, &p, NULL, NULL); + if (isValidPublicKey(pubkeyBn, (BIGNUM*)p, q1)) +#endif res = MDH_compute_key(secret, nPubkeyLen, pubkeyBn, dh); else res = -1; --- a/librtmp/handshake.h +++ b/librtmp/handshake.h @@ -31,9 +31,9 @@ #define SHA256_DIGEST_LENGTH 32 #endif #define HMAC_CTX sha2_context -#define HMAC_setup(ctx, key, len) sha2_hmac_starts(&ctx, (unsigned char *)key, len, 0) -#define HMAC_crunch(ctx, buf, len) sha2_hmac_update(&ctx, buf, len) -#define HMAC_finish(ctx, dig, dlen) dlen = SHA256_DIGEST_LENGTH; sha2_hmac_finish(&ctx, dig) +#define HMAC_setup(ctx, key, len) sha2_hmac_starts(ctx, (unsigned char *)key, len, 0) +#define HMAC_crunch(ctx, buf, len) sha2_hmac_update(ctx, buf, len) +#define HMAC_finish(ctx, dig, dlen) dlen = SHA256_DIGEST_LENGTH; sha2_hmac_finish(ctx, dig) typedef arc4_context * RC4_handle; #define RC4_alloc(h) *h = malloc(sizeof(arc4_context)) @@ -50,9 +50,9 @@ #endif #undef HMAC_CTX #define HMAC_CTX struct hmac_sha256_ctx -#define HMAC_setup(ctx, key, len) hmac_sha256_set_key(&ctx, len, key) -#define HMAC_crunch(ctx, buf, len) hmac_sha256_update(&ctx, len, buf) -#define HMAC_finish(ctx, dig, dlen) dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig) +#define HMAC_setup(ctx, key, len) hmac_sha256_set_key(ctx, len, key) +#define HMAC_crunch(ctx, buf, len) hmac_sha256_update(ctx, len, buf) +#define HMAC_finish(ctx, dig, dlen) dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(ctx, SHA256_DIGEST_LENGTH, dig) #define HMAC_close(ctx) typedef struct arcfour_ctx* RC4_handle; @@ -64,14 +64,23 @@ #else /* USE_OPENSSL */ #include <openssl/sha.h> +#include <openssl/ossl_typ.h> #include <openssl/hmac.h> #include <openssl/rc4.h> #if OPENSSL_VERSION_NUMBER < 0x0090800 || !defined(SHA256_DIGEST_LENGTH) #error Your OpenSSL is too old, need 0.9.8 or newer with SHA256 #endif -#define HMAC_setup(ctx, key, len) HMAC_CTX_init(&ctx); HMAC_Init_ex(&ctx, key, len, EVP_sha256(), 0) -#define HMAC_crunch(ctx, buf, len) HMAC_Update(&ctx, buf, len) -#define HMAC_finish(ctx, dig, dlen) HMAC_Final(&ctx, dig, &dlen); HMAC_CTX_cleanup(&ctx) +#if OPENSSL_VERSION_NUMBER < 0x10100000L +#define HMAC_setup(ctx, key, len) HMAC_CTX_init(ctx); HMAC_Init_ex(ctx, key, len, EVP_sha256(), 0) +#else +#define HMAC_setup(ctx, key, len) ctx = HMAC_CTX_new(); HMAC_CTX_reset(ctx); HMAC_Init_ex(ctx, key, len, EVP_sha256(), 0) +#endif +#define HMAC_crunch(ctx, buf, len) HMAC_Update(ctx, buf, len) +#if OPENSSL_VERSION_NUMBER < 0x10100000L +#define HMAC_finish(ctx, dig, dlen) HMAC_Final(ctx, dig, &dlen); HMAC_CTX_cleanup(ctx) +#else +#define HMAC_finish(ctx, dig, dlen) HMAC_Final(ctx, dig, &dlen); HMAC_CTX_free(ctx) +#endif typedef RC4_KEY * RC4_handle; #define RC4_alloc(h) *h = malloc(sizeof(RC4_KEY)) @@ -117,7 +126,7 @@ { uint8_t digest[SHA256_DIGEST_LENGTH]; unsigned int digestLen = 0; - HMAC_CTX ctx; + HMAC_CTX* ctx = NULL; RC4_alloc(rc4keyIn); RC4_alloc(rc4keyOut); @@ -266,7 +275,7 @@ size_t keylen, uint8_t *digest) { unsigned int digestLen; - HMAC_CTX ctx; + HMAC_CTX* ctx = NULL; HMAC_setup(ctx, key, keylen); HMAC_crunch(ctx, message, messageLen); --- a/librtmp/hashswf.c +++ b/librtmp/hashswf.c @@ -37,9 +37,9 @@ #define SHA256_DIGEST_LENGTH 32 #endif #define HMAC_CTX sha2_context -#define HMAC_setup(ctx, key, len) sha2_hmac_starts(&ctx, (unsigned char *)key, len, 0) -#define HMAC_crunch(ctx, buf, len) sha2_hmac_update(&ctx, buf, len) -#define HMAC_finish(ctx, dig, dlen) dlen = SHA256_DIGEST_LENGTH; sha2_hmac_finish(&ctx, dig) +#define HMAC_setup(ctx, key, len) sha2_hmac_starts(ctx, (unsigned char *)key, len, 0) +#define HMAC_crunch(ctx, buf, len) sha2_hmac_update(ctx, buf, len) +#define HMAC_finish(ctx, dig, dlen) dlen = SHA256_DIGEST_LENGTH; sha2_hmac_finish(ctx, dig) #define HMAC_close(ctx) #elif defined(USE_GNUTLS) #include <nettle/hmac.h> @@ -48,19 +48,27 @@ #endif #undef HMAC_CTX #define HMAC_CTX struct hmac_sha256_ctx -#define HMAC_setup(ctx, key, len) hmac_sha256_set_key(&ctx, len, key) -#define HMAC_crunch(ctx, buf, len) hmac_sha256_update(&ctx, len, buf) -#define HMAC_finish(ctx, dig, dlen) dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(&ctx, SHA256_DIGEST_LENGTH, dig) +#define HMAC_setup(ctx, key, len) hmac_sha256_set_key(ctx, len, key) +#define HMAC_crunch(ctx, buf, len) hmac_sha256_update(ctx, len, buf) +#define HMAC_finish(ctx, dig, dlen) dlen = SHA256_DIGEST_LENGTH; hmac_sha256_digest(ctx, SHA256_DIGEST_LENGTH, dig) #define HMAC_close(ctx) #else /* USE_OPENSSL */ #include <openssl/ssl.h> #include <openssl/sha.h> #include <openssl/hmac.h> #include <openssl/rc4.h> -#define HMAC_setup(ctx, key, len) HMAC_CTX_init(&ctx); HMAC_Init_ex(&ctx, (unsigned char *)key, len, EVP_sha256(), 0) -#define HMAC_crunch(ctx, buf, len) HMAC_Update(&ctx, (unsigned char *)buf, len) -#define HMAC_finish(ctx, dig, dlen) HMAC_Final(&ctx, (unsigned char *)dig, &dlen); -#define HMAC_close(ctx) HMAC_CTX_cleanup(&ctx) +#if OPENSSL_VERSION_NUMBER < 0x10100000L +#define HMAC_setup(ctx, key, len) HMAC_CTX_init(ctx); HMAC_Init_ex(ctx, (unsigned char *)key, len, EVP_sha256(), 0) +#else +#define HMAC_setup(ctx, key, len) ctx = HMAC_CTX_new(); HMAC_CTX_reset(ctx); HMAC_Init_ex(ctx, key, len, EVP_sha256(), 0) +#endif +#define HMAC_crunch(ctx, buf, len) HMAC_Update(ctx, (unsigned char *)buf, len) +#define HMAC_finish(ctx, dig, dlen) HMAC_Final(ctx, (unsigned char *)dig, &dlen); +#if OPENSSL_VERSION_NUMBER < 0x10100000L +#define HMAC_close(ctx) HMAC_CTX_cleanup(ctx) +#else +#define HMAC_close(ctx) HMAC_CTX_reset(ctx); HMAC_CTX_free(ctx) +#endif #endif extern void RTMP_TLS_Init(); @@ -289,7 +297,7 @@ struct info { z_stream *zs; - HMAC_CTX ctx; + HMAC_CTX *ctx; int first; int zlib; int size;