checking mpz_export return value properly

fixes a potential DoS attack if a DH value of zero gets processed

3 files changed, 28 insertions, 8 deletions

diff --git a/src/libstrongswan/plugins/gmp/gmp_diffie_hellman.c b/src/libstrongswan/plugins/gmp/gmp_diffie_hellman.c
index 7f549ef95..1d885d1c7 100644
--- a/src/libstrongswan/plugins/gmp/gmp_diffie_hellman.c
+++ b/src/libstrongswan/plugins/gmp/gmp_diffie_hellman.c
@@ -343,7 +343,7 @@ struct private_gmp_diffie_hellman_t {
 	 * Generator value.
 	 */	
 	mpz_t g;
-
+	
 	/**
 	 * My private value.
 	 */
@@ -353,7 +353,7 @@ struct private_gmp_diffie_hellman_t {
 	 * My public value.
 	 */
 	mpz_t ya;
-
+	
 	/**
 	 * Other public value.
 	 */	
@@ -373,7 +373,7 @@ struct private_gmp_diffie_hellman_t {
 	 * Modulus length.
 	 */
 	size_t p_len;
-
+	
 	/**
 	 * True if shared secret is computed and stored in my_public_value.
 	 */
@@ -440,7 +440,11 @@ static status_t get_other_public_value(private_gmp_diffie_hellman_t *this,
 		return FAILED;
 	}
 	value->len = this->p_len;
-    value->ptr = mpz_export(NULL, NULL, 1, value->len, 1, 0, this->yb);
+	value->ptr = mpz_export(NULL, NULL, 1, value->len, 1, 0, this->yb);
+	if (value->ptr == NULL)
+	{
+		return FAILED;
+	}
 	return SUCCESS;
 }
 
@@ -451,6 +455,10 @@ static void get_my_public_value(private_gmp_diffie_hellman_t *this,chunk_t *valu
 {
 	value->len = this->p_len;
     value->ptr = mpz_export(NULL, NULL, 1, value->len, 1, 0, this->ya);
+    if (value->ptr == NULL)
+    {
+    	value->len = 0;
+    }
 }
 
 /**
@@ -463,7 +471,11 @@ static status_t get_shared_secret(private_gmp_diffie_hellman_t *this, chunk_t *s
 		return FAILED;
 	}
 	secret->len = this->p_len;
-    secret->ptr = mpz_export(NULL, NULL, 1, secret->len, 1, 0, this->zz);
+	secret->ptr = mpz_export(NULL, NULL, 1, secret->len, 1, 0, this->zz);
+	if (secret->ptr == NULL)
+	{
+		return FAILED;
+	}
 	return SUCCESS;
 }
 
diff --git a/src/libstrongswan/plugins/gmp/gmp_rsa_private_key.c b/src/libstrongswan/plugins/gmp/gmp_rsa_private_key.c
index c3fc464e4..84fbb3266 100644
--- a/src/libstrongswan/plugins/gmp/gmp_rsa_private_key.c
+++ b/src/libstrongswan/plugins/gmp/gmp_rsa_private_key.c
@@ -191,6 +191,10 @@ static chunk_t rsadp(private_gmp_rsa_private_key_t *this, chunk_t data)
 	
 	decrypted.len = this->k;
 	decrypted.ptr = mpz_export(NULL, NULL, 1, decrypted.len, 1, 0, t1);
+	if (decrypted.ptr == NULL)
+	{
+		decrypted.len = 0;
+	}
 	
 	mpz_clear_randomized(t1);
 	mpz_clear_randomized(t2);
diff --git a/src/libstrongswan/plugins/gmp/gmp_rsa_public_key.c b/src/libstrongswan/plugins/gmp/gmp_rsa_public_key.c
index f58e28f7d..c5f8244a7 100644
--- a/src/libstrongswan/plugins/gmp/gmp_rsa_public_key.c
+++ b/src/libstrongswan/plugins/gmp/gmp_rsa_public_key.c
@@ -93,11 +93,15 @@ static chunk_t rsaep(private_gmp_rsa_public_key_t *this, chunk_t data)
 	
 	mpz_powm(c, m, this->e, this->n);
 
-    encrypted.len = this->k;
-    encrypted.ptr = mpz_export(NULL, NULL, 1, encrypted.len, 1, 0, c);
+	encrypted.len = this->k;
+	encrypted.ptr = mpz_export(NULL, NULL, 1, encrypted.len, 1, 0, c);
+	if (encrypted.ptr == NULL)
+	{
+		encrypted.len = 0;
+	}
 	
 	mpz_clear(c);
-	mpz_clear(m);	
+	mpz_clear(m);
 	
 	return encrypted;
 }