Don't allow BIGNUMs to become so large that computations with dmax

might overflow.

crypto/bn/bn.h

@@ -505,6 +505,7 @@ BN_ULONG bn_sub_words(BN_ULONG *rp, BN_ULONG *ap, BN_ULONG *bp,int num);
 #define BN_R_INVALID_LENGTH				 106
 #define BN_R_NOT_INITIALIZED				 107
 #define BN_R_NO_INVERSE					 108
+#define BN_R_TOO_LARGE					 114
 #define BN_R_TOO_MANY_TEMPORARY_VARIABLES		 109
 
 #ifdef  __cplusplus

crypto/bn/bn_err.c

@@ -99,6 +99,7 @@ static ERR_STRING_DATA BN_str_reasons[]=
 {BN_R_INVALID_LENGTH                     ,"invalid length"},
 {BN_R_NOT_INITIALIZED                    ,"not initialized"},
 {BN_R_NO_INVERSE                         ,"no inverse"},
+{BN_R_TOO_LARGE                          ,"too large"},
 {BN_R_TOO_MANY_TEMPORARY_VARIABLES       ,"too many temporary variables"},
 {0,NULL}
 	};

crypto/bn/bn_lib.c

@@ -319,6 +319,12 @@ BIGNUM *bn_expand2(BIGNUM *b, int words)
 
 	if (words > b->dmax)
 		{
+		if (words > (INT_MAX/(4*BN_BITS2)))
+			{
+			BNerr(BN_F_BN_EXPAND2,BN_R_TOO_LARGE);
+			return NULL;
+			}
+			
 		bn_check_top(b);	
 		if (BN_get_flags(b,BN_FLG_STATIC_DATA))
 			{