main/rdesktop: rebuild against openssl 1.1

3 files changed, 201 insertions, 16 deletions

diff --git a/main/rdesktop/APKBUILD b/main/rdesktop/APKBUILD
index 9391b7ce17..23a81a482c 100644
--- a/main/rdesktop/APKBUILD
+++ b/main/rdesktop/APKBUILD
@@ -1,38 +1,43 @@
 # Maintainer: Natanael Copa <ncopa@alpinelinux.org>
 pkgname=rdesktop
 pkgver=1.8.3
-pkgrel=4
+pkgrel=5
 pkgdesc="rdesktop is used to connect to windows terminal servers"
-url="http://www.rdesktop.org/"
+url="https://www.rdesktop.org/"
 arch="all"
-license="GPL"
+license="GPL-3"
 subpackages="$pkgname-doc"
 depends=
-makedepends="linux-headers libx11-dev libao-dev libressl-dev libsamplerate-dev libgssglue-dev
-	libxrandr-dev"
-source="https://downloads.sourceforge.net/$pkgname/$pkgname-$pkgver.tar.gz"
+makedepends="linux-headers libx11-dev libao-dev openssl-dev libsamplerate-dev
+	libgssglue-dev libxrandr-dev"
+source="https://downloads.sourceforge.net/$pkgname/$pkgname-$pkgver.tar.gz
+	Fix-OpenSSL-1.1-compatibility-issues.patch
+	Fix-crash-in-rdssl_cert_to_rkey.patch
+	"
 
+builddir="$srcdir"/$pkgname-$pkgver
 prepare() {
-	cd "$srcdir"/$pkgname-$pkgver
-	update_config_sub || return 1
+	cd "$builddir"
+	default_prepare
+	update_config_sub
 }
 
 build() {
-	cd "$srcdir"/$pkgname-$pkgver
+	cd "$builddir"
 	./configure \
 		--build=$CBUILD \
 		--host=$CHOST \
 		--prefix=/usr \
 		--with-sound=ao \
-		--disable-smartcard \
-		|| return 1
-	make || return 1
+		--disable-smartcard
+	make
 }
 
 package() {
-	cd "$srcdir"/$pkgname-$pkgver
+	cd "$builddir"
 	make DESTDIR="$pkgdir" install
 }
-md5sums="86e8b368a7c715e74ded92e0d7912dc5  rdesktop-1.8.3.tar.gz"
-sha256sums="88b20156b34eff5f1b453f7c724e0a3ff9370a599e69c01dc2bf0b5e650eece4  rdesktop-1.8.3.tar.gz"
-sha512sums="06b94ad3b09430b05e424ef31a3e6f2388190b4920e348603cb66a414244896e0dc8906b9f12920e9406cf153ffa7f6507b23bf6713c3a675c0540a8ef57902d  rdesktop-1.8.3.tar.gz"
+
+sha512sums="06b94ad3b09430b05e424ef31a3e6f2388190b4920e348603cb66a414244896e0dc8906b9f12920e9406cf153ffa7f6507b23bf6713c3a675c0540a8ef57902d  rdesktop-1.8.3.tar.gz
+fdf39ee3a4bb97add9307b664a523d5a9edeb9cd3897e82e8fc081d447e843d48e82c4c5eddf52400d78988ff7076e3bc0ae05bc661b099fb952188ad0ef4795  Fix-OpenSSL-1.1-compatibility-issues.patch
+faa6de4d71d6b021d053ecf17bd2f718118185e6aaa3c146fb2e4b18a0188f410d8a3ba27f7288da70dbce4be6f40b9bf78bc4a1e456f11dd5feb13212b74c8d  Fix-crash-in-rdssl_cert_to_rkey.patch"
diff --git a/main/rdesktop/Fix-OpenSSL-1.1-compatibility-issues.patch b/main/rdesktop/Fix-OpenSSL-1.1-compatibility-issues.patch
new file mode 100644
index 0000000000..17c41eefe2
--- /dev/null
+++ b/main/rdesktop/Fix-OpenSSL-1.1-compatibility-issues.patch
@@ -0,0 +1,125 @@
+From bd6aa6acddf0ba640a49834807872f4cc0d0a773 Mon Sep 17 00:00:00 2001
+From: Jani Hakala <jjhakala@gmail.com>
+Date: Thu, 16 Jun 2016 14:28:15 +0300
+Subject: [PATCH] Fix OpenSSL 1.1 compability issues
+
+Some data types have been made opaque in OpenSSL version 1.1 so
+stack allocation and accessing struct fields directly does not work.
+---
+ ssl.c | 65 ++++++++++++++++++++++++++++++++++++++++-------------------------
+ 1 file changed, 40 insertions(+), 25 deletions(-)
+
+diff --git a/ssl.c b/ssl.c
+index 4875125..032e9b9 100644
+--- a/ssl.c
++++ b/ssl.c
+@@ -88,7 +88,7 @@ rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 *
+ 		  uint8 * exponent)
+ {
+ 	BN_CTX *ctx;
+-	BIGNUM mod, exp, x, y;
++	BIGNUM *mod, *exp, *x, *y;
+ 	uint8 inr[SEC_MAX_MODULUS_SIZE];
+ 	int outlen;
+ 
+@@ -98,24 +98,24 @@ rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 *
+ 	reverse(inr, len);
+ 
+ 	ctx = BN_CTX_new();
+-	BN_init(&mod);
+-	BN_init(&exp);
+-	BN_init(&x);
+-	BN_init(&y);
+-
+-	BN_bin2bn(modulus, modulus_size, &mod);
+-	BN_bin2bn(exponent, SEC_EXPONENT_SIZE, &exp);
+-	BN_bin2bn(inr, len, &x);
+-	BN_mod_exp(&y, &x, &exp, &mod, ctx);
+-	outlen = BN_bn2bin(&y, out);
++	mod = BN_new();
++	exp = BN_new();
++	x = BN_new();
++	y = BN_new();
++
++	BN_bin2bn(modulus, modulus_size, mod);
++	BN_bin2bn(exponent, SEC_EXPONENT_SIZE, exp);
++	BN_bin2bn(inr, len, x);
++	BN_mod_exp(y, x, exp, mod, ctx);
++	outlen = BN_bn2bin(y, out);
+ 	reverse(out, outlen);
+ 	if (outlen < (int) modulus_size)
+ 		memset(out + outlen, 0, modulus_size - outlen);
+ 
+-	BN_free(&y);
+-	BN_clear_free(&x);
+-	BN_free(&exp);
+-	BN_free(&mod);
++	BN_free(y);
++	BN_clear_free(x);
++	BN_free(exp);
++	BN_free(mod);
+ 	BN_CTX_free(ctx);
+ }
+ 
+@@ -146,12 +146,20 @@ rdssl_cert_to_rkey(RDSSL_CERT * cert, uint32 * key_len)
+ 
+ 	   Kudos to Richard Levitte for the following (. intiutive .) 
+ 	   lines of code that resets the OID and let's us extract the key. */
+-	nid = OBJ_obj2nid(cert->cert_info->key->algor->algorithm);
++
++	X509_PUBKEY *key = NULL;
++	X509_ALGOR *algor = NULL;
++
++	key = X509_get_X509_PUBKEY(cert);
++	algor = X509_PUBKEY_get0_param(NULL, NULL, 0, &algor, key);
++
++	nid = OBJ_obj2nid(algor->algorithm);
++
+ 	if ((nid == NID_md5WithRSAEncryption) || (nid == NID_shaWithRSAEncryption))
+ 	{
+ 		DEBUG_RDP5(("Re-setting algorithm type to RSA in server certificate\n"));
+-		ASN1_OBJECT_free(cert->cert_info->key->algor->algorithm);
+-		cert->cert_info->key->algor->algorithm = OBJ_nid2obj(NID_rsaEncryption);
++		X509_PUBKEY_set0_param(key, OBJ_nid2obj(NID_rsaEncryption),
++				       0, NULL, NULL, 0);
+ 	}
+ 	epk = X509_get_pubkey(cert);
+ 	if (NULL == epk)
+@@ -201,14 +209,24 @@ rdssl_rkey_get_exp_mod(RDSSL_RKEY * rkey, uint8 * exponent, uint32 max_exp_len,
+ {
+ 	int len;
+ 
+-	if ((BN_num_bytes(rkey->e) > (int) max_exp_len) ||
+-	    (BN_num_bytes(rkey->n) > (int) max_mod_len))
++	BIGNUM *e = NULL;
++	BIGNUM *n = NULL;
++
++#if OPENSSL_VERSION_NUMBER < 0x10100000L
++	e = rkey->e;
++	n = rkey->n;
++#else
++	RSA_get0_key(rkey, &e, &n, NULL);
++#endif
++
++	if ((BN_num_bytes(e) > (int) max_exp_len) ||
++	    (BN_num_bytes(n) > (int) max_mod_len))
+ 	{
+ 		return 1;
+ 	}
+-	len = BN_bn2bin(rkey->e, exponent);
++	len = BN_bn2bin(e, exponent);
+ 	reverse(exponent, len);
+-	len = BN_bn2bin(rkey->n, modulus);
++	len = BN_bn2bin(n, modulus);
+ 	reverse(modulus, len);
+ 	return 0;
+ }
+@@ -229,8 +247,5 @@ void
+ rdssl_hmac_md5(const void *key, int key_len, const unsigned char *msg, int msg_len,
+ 	       unsigned char *md)
+ {
+-	HMAC_CTX ctx;
+-	HMAC_CTX_init(&ctx);
+ 	HMAC(EVP_md5(), key, key_len, msg, msg_len, md, NULL);
+-	HMAC_CTX_cleanup(&ctx);
+ }
diff --git a/main/rdesktop/Fix-crash-in-rdssl_cert_to_rkey.patch b/main/rdesktop/Fix-crash-in-rdssl_cert_to_rkey.patch
new file mode 100644
index 0000000000..58bed87383
--- /dev/null
+++ b/main/rdesktop/Fix-crash-in-rdssl_cert_to_rkey.patch
@@ -0,0 +1,55 @@
+From c6e8e1074b8ac57de6c80c4e3ed38e105b4d94f1 Mon Sep 17 00:00:00 2001
+From: Henrik Andersson <hean01@cendio.com>
+Date: Mon, 24 Oct 2016 10:24:35 +0200
+Subject: [PATCH] Fix crash in rdssl_cert_to_rkey.
+
+This crash was introduced by merging OpenSSL 1.1 PR done on
+commit 50b39d11. Where algor was overwritten with return value
+of X509_PUBKEY_get0_param(). I also added additional error
+handling for X509_get_X509_PUBKEY.
+
+Thanks to TingPing that found this error in PR.
+---
+ ssl.c | 15 ++++++++++++++-
+ 1 file changed, 14 insertions(+), 1 deletion(-)
+
+diff --git a/ssl.c b/ssl.c
+index 032e9b9..07d7aa5 100644
+--- a/ssl.c
++++ b/ssl.c
+@@ -3,6 +3,7 @@
+    Secure sockets abstraction layer
+    Copyright (C) Matthew Chapman <matthewc.unsw.edu.au> 1999-2008
+    Copyright (C) Jay Sorg <j@american-data.com> 2006-2008
++   Copyright (C) Henrik Andersson <hean01@cendio.com> 2016
+ 
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+@@ -140,6 +141,7 @@ rdssl_cert_to_rkey(RDSSL_CERT * cert, uint32 * key_len)
+ 	EVP_PKEY *epk = NULL;
+ 	RDSSL_RKEY *lkey;
+ 	int nid;
++	int ret;
+ 
+ 	/* By some reason, Microsoft sets the OID of the Public RSA key to
+ 	   the oid for "MD5 with RSA Encryption" instead of "RSA Encryption"
+@@ -151,7 +153,18 @@ rdssl_cert_to_rkey(RDSSL_CERT * cert, uint32 * key_len)
+ 	X509_ALGOR *algor = NULL;
+ 
+ 	key = X509_get_X509_PUBKEY(cert);
+-	algor = X509_PUBKEY_get0_param(NULL, NULL, 0, &algor, key);
++	if (key == NULL)
++	{
++		error("Failed to get public key from certificate.\n");
++		return NULL;
++	}
++
++	ret = X509_PUBKEY_get0_param(NULL, NULL, 0, &algor, key);
++	if (ret != 1)
++	{
++		error("Faild to get algorithm used for public key.\n");
++		return NULL;
++	}
+ 
+ 	nid = OBJ_obj2nid(algor->algorithm);
+