Correct multiple vulnerabilities in crypto(3).

Limit the size of public keys used in order to protect applications
from a denial of service via insane key sizes.

Security:	FreeBSD-SA-06:23.openssl
Approved by:	so (cperciva)

3 files changed, 57 insertions, 0 deletions

diff --git a/crypto/openssl/crypto/rsa/rsa.h b/crypto/openssl/crypto/rsa/rsa.h
index 1ea0fe10a535..050715aec74c 100644
--- a/crypto/openssl/crypto/rsa/rsa.h
+++ b/crypto/openssl/crypto/rsa/rsa.h
@@ -151,6 +151,17 @@ struct rsa_st
 	BN_BLINDING *blinding;
 	};
 
+#ifndef OPENSSL_RSA_MAX_MODULUS_BITS
+# define OPENSSL_RSA_MAX_MODULUS_BITS	16400
+#endif
+
+#ifndef OPENSSL_RSA_SMALL_MODULUS_BITS
+# define OPENSSL_RSA_SMALL_MODULUS_BITS	4112
+#endif
+#ifndef OPENSSL_RSA_MAX_PUBEXP_BITS
+# define OPENSSL_RSA_MAX_PUBEXP_BITS	72 /* exponent limit enforced for "large" modulus only */
+#endif
+
 #define RSA_3	0x3L
 #define RSA_F4	0x10001L
 
@@ -344,6 +355,7 @@ void ERR_load_RSA_strings(void);
 #define RSA_R_INVALID_MESSAGE_LENGTH			 131
 #define RSA_R_IQMP_NOT_INVERSE_OF_Q			 126
 #define RSA_R_KEY_SIZE_TOO_SMALL			 120
+#define RSA_R_MODULUS_TOO_LARGE				 105
 #define RSA_R_NULL_BEFORE_BLOCK_MISSING			 113
 #define RSA_R_N_DOES_NOT_EQUAL_P_Q			 127
 #define RSA_R_OAEP_DECODING_ERROR			 121
diff --git a/crypto/openssl/crypto/rsa/rsa_eay.c b/crypto/openssl/crypto/rsa/rsa_eay.c
index 5f7a1f20a359..43737cd00b97 100644
--- a/crypto/openssl/crypto/rsa/rsa_eay.c
+++ b/crypto/openssl/crypto/rsa/rsa_eay.c
@@ -105,6 +105,28 @@ static int RSA_eay_public_encrypt(int flen, const unsigned char *from,
 	unsigned char *buf=NULL;
 	BN_CTX *ctx=NULL;
 
+	if (BN_num_bits(rsa->n) > OPENSSL_RSA_MAX_MODULUS_BITS)
+		{
+		RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_MODULUS_TOO_LARGE);
+		return -1;
+		}
+
+	if (BN_ucmp(rsa->n, rsa->e) <= 0)
+		{
+		RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_BAD_E_VALUE);
+		return -1;
+		}
+
+	/* for large moduli, enforce exponent limit */
+	if (BN_num_bits(rsa->n) > OPENSSL_RSA_SMALL_MODULUS_BITS)
+		{
+		if (BN_num_bits(rsa->e) > OPENSSL_RSA_MAX_PUBEXP_BITS)
+			{
+			RSAerr(RSA_F_RSA_EAY_PUBLIC_ENCRYPT, RSA_R_BAD_E_VALUE);
+			return -1;
+			}
+		}
+	
 	BN_init(&f);
 	BN_init(&ret);
 	if ((ctx=BN_CTX_new()) == NULL) goto err;
@@ -505,6 +527,28 @@ static int RSA_eay_public_decrypt(int flen, const unsigned char *from,
 	unsigned char *buf=NULL;
 	BN_CTX *ctx=NULL;
 
+	if (BN_num_bits(rsa->n) > OPENSSL_RSA_MAX_MODULUS_BITS)
+		{
+		RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT, RSA_R_MODULUS_TOO_LARGE);
+		return -1;
+		}
+
+	if (BN_ucmp(rsa->n, rsa->e) <= 0)
+		{
+		RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT, RSA_R_BAD_E_VALUE);
+		return -1;
+		}
+
+	/* for large moduli, enforce exponent limit */
+	if (BN_num_bits(rsa->n) > OPENSSL_RSA_SMALL_MODULUS_BITS)
+		{
+		if (BN_num_bits(rsa->e) > OPENSSL_RSA_MAX_PUBEXP_BITS)
+			{
+			RSAerr(RSA_F_RSA_EAY_PUBLIC_DECRYPT, RSA_R_BAD_E_VALUE);
+			return -1;
+			}
+		}
+	
 	BN_init(&f);
 	BN_init(&ret);
 	ctx=BN_CTX_new();
diff --git a/crypto/openssl/crypto/rsa/rsa_err.c b/crypto/openssl/crypto/rsa/rsa_err.c
index a7766c3b762e..8221a921e7b3 100644
--- a/crypto/openssl/crypto/rsa/rsa_err.c
+++ b/crypto/openssl/crypto/rsa/rsa_err.c
@@ -116,6 +116,7 @@ static ERR_STRING_DATA RSA_str_reasons[]=
 {RSA_R_INVALID_MESSAGE_LENGTH            ,"invalid message length"},
 {RSA_R_IQMP_NOT_INVERSE_OF_Q             ,"iqmp not inverse of q"},
 {RSA_R_KEY_SIZE_TOO_SMALL                ,"key size too small"},
+{RSA_R_MODULUS_TOO_LARGE                 ,"modulus too large"},
 {RSA_R_NULL_BEFORE_BLOCK_MISSING         ,"null before block missing"},
 {RSA_R_N_DOES_NOT_EQUAL_P_Q              ,"n does not equal p q"},
 {RSA_R_OAEP_DECODING_ERROR               ,"oaep decoding error"},