crypto: Define MODP_CUSTOM outside of IKE DH range

Before this fix it was possible to crash charon with an IKE_SA_INIT
message containing a KE payload with DH group MODP_CUSTOM(1025).
Defining MODP_CUSTOM outside of the two byte IKE DH identifier range
prevents it from getting negotiated.

Fixes CVE-2014-9221.

9 files changed, 17 insertions, 14 deletions

diff --git a/src/charon-tkm/src/tkm/tkm_diffie_hellman.c b/src/charon-tkm/src/tkm/tkm_diffie_hellman.c
index 67db5e6d8..836e0b7f0 100644
--- a/src/charon-tkm/src/tkm/tkm_diffie_hellman.c
+++ b/src/charon-tkm/src/tkm/tkm_diffie_hellman.c
@@ -41,7 +41,7 @@ struct private_tkm_diffie_hellman_t {
 	/**
 	 * Diffie Hellman group number.
 	 */
-	u_int16_t group;
+	diffie_hellman_group_t group;
 
 	/**
 	 * Diffie Hellman public value.
diff --git a/src/libstrongswan/crypto/diffie_hellman.c b/src/libstrongswan/crypto/diffie_hellman.c
index bada1c529..ac106e9c4 100644
--- a/src/libstrongswan/crypto/diffie_hellman.c
+++ b/src/libstrongswan/crypto/diffie_hellman.c
@@ -42,15 +42,16 @@ ENUM_NEXT(diffie_hellman_group_names, MODP_1024_160, ECP_512_BP, ECP_521_BIT,
 	"ECP_256_BP",
 	"ECP_384_BP",
 	"ECP_512_BP");
-ENUM_NEXT(diffie_hellman_group_names, MODP_NULL, MODP_CUSTOM, ECP_512_BP,
-	"MODP_NULL",
-	"MODP_CUSTOM");
-ENUM_NEXT(diffie_hellman_group_names, NTRU_112_BIT, NTRU_256_BIT, MODP_CUSTOM,
+ENUM_NEXT(diffie_hellman_group_names, MODP_NULL, MODP_NULL, ECP_512_BP,
+	"MODP_NULL");
+ENUM_NEXT(diffie_hellman_group_names, NTRU_112_BIT, NTRU_256_BIT, MODP_NULL,
 	"NTRU_112",
 	"NTRU_128",
 	"NTRU_192",
 	"NTRU_256");
-ENUM_END(diffie_hellman_group_names, NTRU_256_BIT);
+ENUM_NEXT(diffie_hellman_group_names, MODP_CUSTOM, MODP_CUSTOM, NTRU_256_BIT,
+	"MODP_CUSTOM");
+ENUM_END(diffie_hellman_group_names, MODP_CUSTOM);
 
 
 /**
diff --git a/src/libstrongswan/crypto/diffie_hellman.h b/src/libstrongswan/crypto/diffie_hellman.h
index 105db22f1..d5161d077 100644
--- a/src/libstrongswan/crypto/diffie_hellman.h
+++ b/src/libstrongswan/crypto/diffie_hellman.h
@@ -63,12 +63,14 @@ enum diffie_hellman_group_t {
 	/** insecure NULL diffie hellman group for testing, in PRIVATE USE */
 	MODP_NULL = 1024,
 	/** MODP group with custom generator/prime */
-	MODP_CUSTOM = 1025,
 	/** Parameters defined by IEEE 1363.1, in PRIVATE USE */
 	NTRU_112_BIT = 1030,
 	NTRU_128_BIT = 1031,
 	NTRU_192_BIT = 1032,
-	NTRU_256_BIT = 1033
+	NTRU_256_BIT = 1033,
+	/** internally used DH group with additional parameters g and p, outside
+	 * of PRIVATE USE (i.e. IKEv2 DH group range) so it can't be negotiated */
+	MODP_CUSTOM = 65536,
 };
 
 /**
diff --git a/src/libstrongswan/plugins/gcrypt/gcrypt_dh.c b/src/libstrongswan/plugins/gcrypt/gcrypt_dh.c
index f418b941d..299865da2 100644
--- a/src/libstrongswan/plugins/gcrypt/gcrypt_dh.c
+++ b/src/libstrongswan/plugins/gcrypt/gcrypt_dh.c
@@ -35,7 +35,7 @@ struct private_gcrypt_dh_t {
 	/**
 	 * Diffie Hellman group number
 	 */
-	u_int16_t group;
+	diffie_hellman_group_t group;
 
 	/*
 	 * Generator value
diff --git a/src/libstrongswan/plugins/gmp/gmp_diffie_hellman.c b/src/libstrongswan/plugins/gmp/gmp_diffie_hellman.c
index b74d35169..9936f7e45 100644
--- a/src/libstrongswan/plugins/gmp/gmp_diffie_hellman.c
+++ b/src/libstrongswan/plugins/gmp/gmp_diffie_hellman.c
@@ -42,7 +42,7 @@ struct private_gmp_diffie_hellman_t {
 	/**
 	 * Diffie Hellman group number.
 	 */
-	u_int16_t group;
+	diffie_hellman_group_t group;
 
 	/*
 	 * Generator value.
diff --git a/src/libstrongswan/plugins/ntru/ntru_ke.c b/src/libstrongswan/plugins/ntru/ntru_ke.c
index abaa22336..e64f32b91 100644
--- a/src/libstrongswan/plugins/ntru/ntru_ke.c
+++ b/src/libstrongswan/plugins/ntru/ntru_ke.c
@@ -56,7 +56,7 @@ struct private_ntru_ke_t {
 	/**
 	 * Diffie Hellman group number.
 	 */
-	u_int16_t group;
+	diffie_hellman_group_t group;
 
 	/**
 	 * NTRU Parameter Set
diff --git a/src/libstrongswan/plugins/openssl/openssl_diffie_hellman.c b/src/libstrongswan/plugins/openssl/openssl_diffie_hellman.c
index ff3382473..1e68ac59b 100644
--- a/src/libstrongswan/plugins/openssl/openssl_diffie_hellman.c
+++ b/src/libstrongswan/plugins/openssl/openssl_diffie_hellman.c
@@ -38,7 +38,7 @@ struct private_openssl_diffie_hellman_t {
 	/**
 	 * Diffie Hellman group number.
 	 */
-	u_int16_t group;
+	diffie_hellman_group_t group;
 
 	/**
 	 * Diffie Hellman object
diff --git a/src/libstrongswan/plugins/openssl/openssl_ec_diffie_hellman.c b/src/libstrongswan/plugins/openssl/openssl_ec_diffie_hellman.c
index b487d59a5..50853d6f0 100644
--- a/src/libstrongswan/plugins/openssl/openssl_ec_diffie_hellman.c
+++ b/src/libstrongswan/plugins/openssl/openssl_ec_diffie_hellman.c
@@ -40,7 +40,7 @@ struct private_openssl_ec_diffie_hellman_t {
 	/**
 	 * Diffie Hellman group number.
 	 */
-	u_int16_t group;
+	diffie_hellman_group_t group;
 
 	/**
 	 * EC private (public) key
diff --git a/src/libstrongswan/plugins/pkcs11/pkcs11_dh.c b/src/libstrongswan/plugins/pkcs11/pkcs11_dh.c
index 36cc284bf..23b63d238 100644
--- a/src/libstrongswan/plugins/pkcs11/pkcs11_dh.c
+++ b/src/libstrongswan/plugins/pkcs11/pkcs11_dh.c
@@ -47,7 +47,7 @@ struct private_pkcs11_dh_t {
 	/**
 	 * Diffie Hellman group number.
 	 */
-	u_int16_t group;
+	diffie_hellman_group_t group;
 
 	/**
 	 * Handle for own private value