From bd6aa6acddf0ba640a49834807872f4cc0d0a773 Mon Sep 17 00:00:00 2001
From: Jani Hakala <jjhakala@gmail.com>
Date: Thu, 16 Jun 2016 14:28:15 +0300
Subject: [PATCH] Fix OpenSSL 1.1 compability issues

Some data types have been made opaque in OpenSSL version 1.1 so
stack allocation and accessing struct fields directly does not work.

Downloaded from upstream commit
https://github.com/rdesktop/rdesktop/commit/bd6aa6acddf0ba640a49834807872f4cc0d0a773

Signed-off-by: Bernd Kuhls <bernd.kuhls@t-online.de>
---
 ssl.c | 65 ++++++++++++++++++++++++++++++++++++-----------------------
 1 file changed, 40 insertions(+), 25 deletions(-)

diff --git a/ssl.c b/ssl.c
index 48751255..032e9b9e 100644
--- a/ssl.c
+++ b/ssl.c
@@ -88,7 +88,7 @@ rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 *
 		  uint8 * exponent)
 {
 	BN_CTX *ctx;
-	BIGNUM mod, exp, x, y;
+	BIGNUM *mod, *exp, *x, *y;
 	uint8 inr[SEC_MAX_MODULUS_SIZE];
 	int outlen;
 
@@ -98,24 +98,24 @@ rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 *
 	reverse(inr, len);
 
 	ctx = BN_CTX_new();
-	BN_init(&mod);
-	BN_init(&exp);
-	BN_init(&x);
-	BN_init(&y);
-
-	BN_bin2bn(modulus, modulus_size, &mod);
-	BN_bin2bn(exponent, SEC_EXPONENT_SIZE, &exp);
-	BN_bin2bn(inr, len, &x);
-	BN_mod_exp(&y, &x, &exp, &mod, ctx);
-	outlen = BN_bn2bin(&y, out);
+	mod = BN_new();
+	exp = BN_new();
+	x = BN_new();
+	y = BN_new();
+
+	BN_bin2bn(modulus, modulus_size, mod);
+	BN_bin2bn(exponent, SEC_EXPONENT_SIZE, exp);
+	BN_bin2bn(inr, len, x);
+	BN_mod_exp(y, x, exp, mod, ctx);
+	outlen = BN_bn2bin(y, out);
 	reverse(out, outlen);
 	if (outlen < (int) modulus_size)
 		memset(out + outlen, 0, modulus_size - outlen);
 
-	BN_free(&y);
-	BN_clear_free(&x);
-	BN_free(&exp);
-	BN_free(&mod);
+	BN_free(y);
+	BN_clear_free(x);
+	BN_free(exp);
+	BN_free(mod);
 	BN_CTX_free(ctx);
 }
 
@@ -146,12 +146,20 @@ rdssl_cert_to_rkey(RDSSL_CERT * cert, uint32 * key_len)
 
 	   Kudos to Richard Levitte for the following (. intiutive .) 
 	   lines of code that resets the OID and let's us extract the key. */
-	nid = OBJ_obj2nid(cert->cert_info->key->algor->algorithm);
+
+	X509_PUBKEY *key = NULL;
+	X509_ALGOR *algor = NULL;
+
+	key = X509_get_X509_PUBKEY(cert);
+	algor = X509_PUBKEY_get0_param(NULL, NULL, 0, &algor, key);
+
+	nid = OBJ_obj2nid(algor->algorithm);
+
 	if ((nid == NID_md5WithRSAEncryption) || (nid == NID_shaWithRSAEncryption))
 	{
 		DEBUG_RDP5(("Re-setting algorithm type to RSA in server certificate\n"));
-		ASN1_OBJECT_free(cert->cert_info->key->algor->algorithm);
-		cert->cert_info->key->algor->algorithm = OBJ_nid2obj(NID_rsaEncryption);
+		X509_PUBKEY_set0_param(key, OBJ_nid2obj(NID_rsaEncryption),
+				       0, NULL, NULL, 0);
 	}
 	epk = X509_get_pubkey(cert);
 	if (NULL == epk)
@@ -201,14 +209,24 @@ rdssl_rkey_get_exp_mod(RDSSL_RKEY * rkey, uint8 * exponent, uint32 max_exp_len,
 {
 	int len;
 
-	if ((BN_num_bytes(rkey->e) > (int) max_exp_len) ||
-	    (BN_num_bytes(rkey->n) > (int) max_mod_len))
+	BIGNUM *e = NULL;
+	BIGNUM *n = NULL;
+
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+	e = rkey->e;
+	n = rkey->n;
+#else
+	RSA_get0_key(rkey, &e, &n, NULL);
+#endif
+
+	if ((BN_num_bytes(e) > (int) max_exp_len) ||
+	    (BN_num_bytes(n) > (int) max_mod_len))
 	{
 		return 1;
 	}
-	len = BN_bn2bin(rkey->e, exponent);
+	len = BN_bn2bin(e, exponent);
 	reverse(exponent, len);
-	len = BN_bn2bin(rkey->n, modulus);
+	len = BN_bn2bin(n, modulus);
 	reverse(modulus, len);
 	return 0;
 }
@@ -229,8 +247,5 @@ void
 rdssl_hmac_md5(const void *key, int key_len, const unsigned char *msg, int msg_len,
 	       unsigned char *md)
 {
-	HMAC_CTX ctx;
-	HMAC_CTX_init(&ctx);
 	HMAC(EVP_md5(), key, key_len, msg, msg_len, md, NULL);
-	HMAC_CTX_cleanup(&ctx);
 }