Ensure that rc5 doesn't try to use a key longer than 2040 bits

The maximum key length for rc5 is 2040 bits so we should not attempt to
use keys longer than this.

Issue found by OSS-Fuzz and Guido Vranken.

Reviewed-by: Paul Dale <paul.dale@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/8834)

crypto/err/openssl.txt

@@ -889,6 +889,7 @@ EVP_F_PKEY_SET_TYPE:158:pkey_set_type
 EVP_F_POLY1305_CTRL:216:poly1305_ctrl
 EVP_F_RC2_MAGIC_TO_METH:109:rc2_magic_to_meth
 EVP_F_RC5_CTRL:125:rc5_ctrl
+EVP_F_R_32_12_16_INIT_KEY:242:r_32_12_16_init_key
 EVP_F_S390X_AES_GCM_CTRL:201:s390x_aes_gcm_ctrl
 EVP_F_S390X_AES_GCM_TLS_CIPHER:208:s390x_aes_gcm_tls_cipher
 EVP_F_SCRYPT_ALG:228:scrypt_alg
@@ -2385,6 +2386,7 @@ ESS_R_ESS_SIGNING_CERT_V2_ADD_ERROR:101:ess signing cert v2 add error
 EVP_R_AES_KEY_SETUP_FAILED:143:aes key setup failed
 EVP_R_ARIA_KEY_SETUP_FAILED:176:aria key setup failed
 EVP_R_BAD_DECRYPT:100:bad decrypt
+EVP_R_BAD_KEY_LENGTH:195:bad key length
 EVP_R_BUFFER_TOO_SMALL:155:buffer too small
 EVP_R_CAMELLIA_KEY_SETUP_FAILED:157:camellia key setup failed
 EVP_R_CIPHER_NOT_GCM_MODE:184:cipher not gcm mode

crypto/evp/e_rc5.c

@@ -66,6 +66,10 @@ static int rc5_ctrl(EVP_CIPHER_CTX *c, int type, int arg, void *ptr)
 static int r_32_12_16_init_key(EVP_CIPHER_CTX *ctx, const unsigned char *key,
                                const unsigned char *iv, int enc)
 {
+    if (EVP_CIPHER_CTX_key_length(ctx) > 255) {
+        EVPerr(EVP_F_R_32_12_16_INIT_KEY, EVP_R_BAD_KEY_LENGTH);
+        return 0;
+    }
     RC5_32_set_key(&data(ctx)->ks, EVP_CIPHER_CTX_key_length(ctx),
                    key, data(ctx)->rounds);
     return 1;

crypto/evp/evp_err.c

@@ -185,6 +185,8 @@ static const ERR_STRING_DATA EVP_str_functs[] = {
     {ERR_PACK(ERR_LIB_EVP, EVP_F_POLY1305_CTRL, 0), "poly1305_ctrl"},
     {ERR_PACK(ERR_LIB_EVP, EVP_F_RC2_MAGIC_TO_METH, 0), "rc2_magic_to_meth"},
     {ERR_PACK(ERR_LIB_EVP, EVP_F_RC5_CTRL, 0), "rc5_ctrl"},
+    {ERR_PACK(ERR_LIB_EVP, EVP_F_R_32_12_16_INIT_KEY, 0),
+     "r_32_12_16_init_key"},
     {ERR_PACK(ERR_LIB_EVP, EVP_F_S390X_AES_GCM_CTRL, 0), "s390x_aes_gcm_ctrl"},
     {ERR_PACK(ERR_LIB_EVP, EVP_F_S390X_AES_GCM_TLS_CIPHER, 0),
      "s390x_aes_gcm_tls_cipher"},
@@ -199,6 +201,7 @@ static const ERR_STRING_DATA EVP_str_reasons[] = {
     {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_ARIA_KEY_SETUP_FAILED),
     "aria key setup failed"},
     {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_BAD_DECRYPT), "bad decrypt"},
+    {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_BAD_KEY_LENGTH), "bad key length"},
     {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_BUFFER_TOO_SMALL), "buffer too small"},
     {ERR_PACK(ERR_LIB_EVP, 0, EVP_R_CAMELLIA_KEY_SETUP_FAILED),
     "camellia key setup failed"},

doc/man3/EVP_rc5_32_12_16_cbc.pod

@@ -33,7 +33,26 @@ EVP_rc5_32_12_16_ofb()
 
 RC5 encryption algorithm in CBC, CFB, ECB and OFB modes respectively. This is a
 variable key length cipher with an additional "number of rounds" parameter. By
-default the key length is set to 128 bits and 12 rounds.
+default the key length is set to 128 bits and 12 rounds. Alternative key lengths
+can be set using L<EVP_CIPHER_CTX_set_key_length(3)>. The maximum key length is
+2040 bits.
+
+The following rc5 specific I<ctrl>s are supported (see
+L<EVP_CIPHER_CTX_ctrl(3)>).
+
+=over 4
+
+=item EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_SET_RC5_ROUNDS, rounds, NULL)
+
+Sets the number of rounds to B<rounds>. This must be one of RC5_8_ROUNDS,
+RC5_12_ROUNDS or RC5_16_ROUNDS.
+
+=item EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GET_RC5_ROUNDS, 0, &rounds)
+
+Stores the number of rounds currently configured in B<*rounds> where B<*rounds>
+is an int.
+
+=back
 
 =back
 
@@ -43,10 +62,6 @@ These functions return an B<EVP_CIPHER> structure that contains the
 implementation of the symmetric cipher. See L<EVP_CIPHER_meth_new(3)> for
 details of the B<EVP_CIPHER> structure.
 
-=head1 BUGS
-
-Currently the number of rounds in RC5 can only be set to 8, 12 or 16.
-This is a limitation of the current RC5 code rather than the EVP interface.
 
 =head1 SEE ALSO
 

include/openssl/evperr.h

@@ -151,6 +151,7 @@ int ERR_load_EVP_strings(void);
 # define EVP_F_POLY1305_CTRL                              216
 # define EVP_F_RC2_MAGIC_TO_METH                          109
 # define EVP_F_RC5_CTRL                                   125
+# define EVP_F_R_32_12_16_INIT_KEY                        242
 # define EVP_F_S390X_AES_GCM_CTRL                         201
 # define EVP_F_S390X_AES_GCM_TLS_CIPHER                   208
 # define EVP_F_SCRYPT_ALG                                 228
@@ -162,6 +163,7 @@ int ERR_load_EVP_strings(void);
 # define EVP_R_AES_KEY_SETUP_FAILED                       143
 # define EVP_R_ARIA_KEY_SETUP_FAILED                      176
 # define EVP_R_BAD_DECRYPT                                100
+# define EVP_R_BAD_KEY_LENGTH                             195
 # define EVP_R_BUFFER_TOO_SMALL                           155
 # define EVP_R_CAMELLIA_KEY_SETUP_FAILED                  157
 # define EVP_R_CIPHER_NOT_GCM_MODE                        184