From 2de6fa865778a32ff807f1d716ff7f4b49acd74c Mon Sep 17 00:00:00 2001 From: FX Coudert Date: Wed, 4 Sep 2019 17:03:04 +0200 Subject: [PATCH] rtmpdump: move to OpenSSL 1.1 and patch --- Formula/rtmpdump.rb | 150 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 149 insertions(+), 1 deletion(-) diff --git a/Formula/rtmpdump.rb b/Formula/rtmpdump.rb index 481f4b8c15..c9ee2a7361 100644 --- a/Formula/rtmpdump.rb +++ b/Formula/rtmpdump.rb @@ -4,6 +4,7 @@ class Rtmpdump < Formula url "https://deb.debian.org/debian/pool/main/r/rtmpdump/rtmpdump_2.4+20151223.gitfa8646d.1.orig.tar.gz" version "2.4+20151223" sha256 "5c032f5c8cc2937eb55a81a94effdfed3b0a0304b6376147b86f951e225e3ab5" + revision 1 head "https://git.ffmpeg.org/rtmpdump", :shallow => false bottle do @@ -16,10 +17,14 @@ class Rtmpdump < Formula sha256 "f4c8dbdf3f8a04626a7975abf96eccd5e494a3f6a795b2035c6d418bfbe8079d" => :mavericks end - depends_on "openssl" + depends_on "openssl@1.1" conflicts_with "flvstreamer", :because => "both install 'rtmpsrv', 'rtmpsuck' and 'streams' binary" + # Patch for OpenSSL 1.1 compatibility + # Taken from https://github.com/JudgeZarbi/RTMPDump-OpenSSL-1.1 + patch :p0, :DATA + def install ENV.deparallelize system "make", "CC=#{ENV.cc}", @@ -36,3 +41,146 @@ class Rtmpdump < Formula system "#{bin}/rtmpdump", "-h" end end +__END__ +--- librtmp/dh.h.orig 2016-02-29 01:15:13 UTC ++++ librtmp/dh.h +@@ -253,20 +253,23 @@ DHInit(int nKeyBits) + if (!dh) + goto failed; + +- MP_new(dh->g); ++ const BIGNUM *p; ++ const BIGNUM *g; ++ DH_get0_pqg(dh,&p,NULL,&g); ++ MP_new(g); + +- if (!dh->g) ++ if (!g) + goto failed; + +- MP_gethex(dh->p, P1024, res); /* prime P1024, see dhgroups.h */ ++ MP_gethex(p, P1024, res); /* prime P1024, see dhgroups.h */ + if (!res) + { + goto failed; + } + +- MP_set_w(dh->g, 2); /* base 2 */ ++ MP_set_w(g, 2); /* base 2 */ + +- dh->length = nKeyBits; ++ DH_set_length(dh, nKeyBits); + return dh; + + failed: +@@ -293,12 +296,15 @@ DHGenerateKey(MDH *dh) + MP_gethex(q1, Q1024, res); + assert(res); + +- res = isValidPublicKey(dh->pub_key, dh->p, q1); ++ BIGNUM *pub_key, *priv_key, *p; ++ DH_get0_key(dh, &pub_key, &priv_key); ++ DH_get0_pqg(dh,&p,NULL,NULL); ++ res = isValidPublicKey(pub_key, p, q1); + if (!res) + { +- MP_free(dh->pub_key); +- MP_free(dh->priv_key); +- dh->pub_key = dh->priv_key = 0; ++ MP_free(pub_key); ++ MP_free(priv_key); ++ DH_set0_key(dh, 0, 0); + } + + MP_free(q1); +@@ -314,15 +320,17 @@ static int + DHGetPublicKey(MDH *dh, uint8_t *pubkey, size_t nPubkeyLen) + { + int len; +- if (!dh || !dh->pub_key) ++ BIGNUM *pub_key; ++ DH_get0_key(dh, &pub_key, NULL); ++ if (!dh || !pub_key) + return 0; + +- len = MP_bytes(dh->pub_key); ++ len = MP_bytes(pub_key); + if (len <= 0 || len > (int) nPubkeyLen) + return 0; + + memset(pubkey, 0, nPubkeyLen); +- MP_setbin(dh->pub_key, pubkey + (nPubkeyLen - len), len); ++ MP_setbin(pub_key, pubkey + (nPubkeyLen - len), len); + return 1; + } + +@@ -364,7 +372,9 @@ DHComputeSharedSecretKey(MDH *dh, uint8_t *pubkey, siz + MP_gethex(q1, Q1024, len); + assert(len); + +- if (isValidPublicKey(pubkeyBn, dh->p, q1)) ++ BIGNUM *p; ++ DH_get0_pqg(dh,&p,NULL,NULL); ++ if (isValidPublicKey(pubkeyBn, p, q1)) + res = MDH_compute_key(secret, nPubkeyLen, pubkeyBn, dh); + else + res = -1; +--- librtmp/handshake.h.orig 2016-02-29 01:15:13 UTC ++++ librtmp/handshake.h +@@ -69,9 +69,9 @@ typedef struct arcfour_ctx* RC4_handle; + #if OPENSSL_VERSION_NUMBER < 0x0090800 || !defined(SHA256_DIGEST_LENGTH) + #error Your OpenSSL is too old, need 0.9.8 or newer with SHA256 + #endif +-#define HMAC_setup(ctx, key, len) HMAC_CTX_init(&ctx); HMAC_Init_ex(&ctx, key, len, EVP_sha256(), 0) +-#define HMAC_crunch(ctx, buf, len) HMAC_Update(&ctx, buf, len) +-#define HMAC_finish(ctx, dig, dlen) HMAC_Final(&ctx, dig, &dlen); HMAC_CTX_cleanup(&ctx) ++#define HMAC_setup(ctx, key, len) HMAC_Init_ex(ctx, key, len, EVP_sha256(), 0) ++#define HMAC_crunch(ctx, buf, len) HMAC_Update(ctx, buf, len) ++#define HMAC_finish(ctx, dig, dlen) HMAC_Final(ctx, dig, &dlen); HMAC_CTX_free(ctx) + + typedef RC4_KEY * RC4_handle; + #define RC4_alloc(h) *h = malloc(sizeof(RC4_KEY)) +@@ -117,7 +117,7 @@ static void InitRC4Encryption + { + uint8_t digest[SHA256_DIGEST_LENGTH]; + unsigned int digestLen = 0; +- HMAC_CTX ctx; ++ HMAC_CTX *ctx = HMAC_CTX_new(); + + RC4_alloc(rc4keyIn); + RC4_alloc(rc4keyOut); +@@ -266,7 +266,7 @@ HMACsha256(const uint8_t *message, size_t messageLen, + size_t keylen, uint8_t *digest) + { + unsigned int digestLen; +- HMAC_CTX ctx; ++ HMAC_CTX *ctx = HMAC_CTX_new(); + + HMAC_setup(ctx, key, keylen); + HMAC_crunch(ctx, message, messageLen); +--- librtmp/hashswf.c.orig 2016-02-29 01:15:13 UTC ++++ librtmp/hashswf.c +@@ -57,10 +57,10 @@ + #include + #include + #include +-#define HMAC_setup(ctx, key, len) HMAC_CTX_init(&ctx); HMAC_Init_ex(&ctx, (unsigned char *)key, len, EVP_sha256(), 0) +-#define HMAC_crunch(ctx, buf, len) HMAC_Update(&ctx, (unsigned char *)buf, len) +-#define HMAC_finish(ctx, dig, dlen) HMAC_Final(&ctx, (unsigned char *)dig, &dlen); +-#define HMAC_close(ctx) HMAC_CTX_cleanup(&ctx) ++#define HMAC_setup(ctx, key, len) HMAC_Init_ex(ctx, (unsigned char *)key, len, EVP_sha256(), 0) ++#define HMAC_crunch(ctx, buf, len) HMAC_Update(ctx, (unsigned char *)buf, len) ++#define HMAC_finish(ctx, dig, dlen) HMAC_Final(ctx, (unsigned char *)dig, &dlen); ++#define HMAC_close(ctx) HMAC_CTX_free(ctx) + #endif + + extern void RTMP_TLS_Init(); +@@ -289,7 +289,7 @@ leave: + struct info + { + z_stream *zs; +- HMAC_CTX ctx; ++ HMAC_CTX *ctx; + int first; + int zlib; + int size;