Defines and strings for special salt length values, add tests

Reviewed-by: Rich Salz <rsalz@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2236)

crypto/rsa/rsa_ameth.c

@@ -540,7 +540,7 @@ static RSA_PSS_PARAMS *rsa_ctx_to_pss(EVP_PKEY_CTX *pkctx)
         saltlen = EVP_MD_size(sigmd);
     else if (saltlen == -2) {
         saltlen = EVP_PKEY_size(pk) - EVP_MD_size(sigmd) - 2;
-        if (((EVP_PKEY_bits(pk) - 1) & 0x7) == 0)
+        if ((EVP_PKEY_bits(pk) & 0x7) == 1)
             saltlen--;
     }
 

crypto/rsa/rsa_pmeth.c

@@ -58,7 +58,8 @@ static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
         rctx->pad_mode = RSA_PKCS1_PSS_PADDING;
     else
         rctx->pad_mode = RSA_PKCS1_PADDING;
-    rctx->saltlen = -2;
+    /* Maximum for sign, auto for verify */
+    rctx->saltlen = RSA_PSS_SALTLEN_AUTO;
     rctx->min_saltlen = -1;
     ctx->data = rctx;
     ctx->keygen_info = rctx->gentmp;
@@ -430,14 +431,16 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
         if (type == EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN) {
             *(int *)p2 = rctx->saltlen;
         } else {
-            if (p1 < -2)
+            if (p1 < RSA_PSS_SALTLEN_MAX)
                 return -2;
             if (rsa_pss_restricted(rctx)) {
-                if (p1 == -2 && ctx->operation == EVP_PKEY_OP_VERIFY) {
+                if (p1 == RSA_PSS_SALTLEN_AUTO
+                    && ctx->operation == EVP_PKEY_OP_VERIFY) {
                     RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_INVALID_PSS_SALTLEN);
                     return -2;
                 }
-                if ((p1 == -1 && rctx->min_saltlen > EVP_MD_size(rctx->md))
+                if ((p1 == RSA_PSS_SALTLEN_DIGEST
+                     && rctx->min_saltlen > EVP_MD_size(rctx->md))
                     || (p1 >= 0 && p1 < rctx->min_saltlen)) {
                     RSAerr(RSA_F_PKEY_RSA_CTRL, RSA_R_PSS_SALTLEN_TOO_SMALL);
                     return 0;
@@ -596,6 +599,13 @@ static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
 
     if (strcmp(type, "rsa_pss_saltlen") == 0) {
         int saltlen;
+        if (!strcmp(value, "digest"))
+            saltlen = RSA_PSS_SALTLEN_DIGEST;
+        else if (!strcmp(value, "max"))
+            saltlen = RSA_PSS_SALTLEN_MAX;
+        else if (!strcmp(value, "auto"))
+            saltlen = RSA_PSS_SALTLEN_AUTO;
+        else
             saltlen = atoi(value);
         return EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, saltlen);
     }

crypto/rsa/rsa_pss.c

@@ -41,7 +41,6 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
     EVP_MD_CTX *ctx = EVP_MD_CTX_new();
     unsigned char H_[EVP_MAX_MD_SIZE];
 
-
     if (ctx == NULL)
         goto err;
 
@@ -57,11 +56,9 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
      *      -2      salt length is autorecovered from signature
      *      -N      reserved
      */
-    if (sLen == -1)
+    if (sLen == RSA_PSS_SALTLEN_DIGEST)
         sLen = hLen;
-    else if (sLen == -2)
+    else if (sLen < RSA_PSS_SALTLEN_MAX) {
-        sLen = -2;
-    else if (sLen < -2) {
         RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_SLEN_CHECK_FAILED);
         goto err;
     }
@@ -76,7 +73,9 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
         EM++;
         emLen--;
     }
-    if (emLen < (hLen + sLen + 2)) { /* sLen can be small negative */
+    if (sLen == RSA_PSS_SALTLEN_MAX) {
+        sLen = emLen - hLen - 2;
+    } else if (emLen < (hLen + sLen + 2)) { /* sLen can be small negative */
         RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_DATA_TOO_LARGE);
         goto err;
     }
@@ -102,7 +101,7 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const unsigned char *mHash,
         RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_SLEN_RECOVERY_FAILED);
         goto err;
     }
-    if (sLen >= 0 && (maskedDBLen - i) != sLen) {
+    if (sLen != RSA_PSS_SALTLEN_AUTO && (maskedDBLen - i) != sLen) {
         RSAerr(RSA_F_RSA_VERIFY_PKCS1_PSS_MGF1, RSA_R_SLEN_CHECK_FAILED);
         goto err;
     }
@@ -160,11 +159,11 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
      *      -2      salt length is maximized
      *      -N      reserved
      */
-    if (sLen == -1)
+    if (sLen == RSA_PSS_SALTLEN_DIGEST)
         sLen = hLen;
-    else if (sLen == -2)
+    else if (sLen == RSA_PSS_SALTLEN_MAX_SIGN)
-        sLen = -2;
+        sLen = RSA_PSS_SALTLEN_MAX;
-    else if (sLen < -2) {
+    else if (sLen < RSA_PSS_SALTLEN_MAX) {
         RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_PSS_MGF1, RSA_R_SLEN_CHECK_FAILED);
         goto err;
     }
@@ -175,7 +174,7 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
         *EM++ = 0;
         emLen--;
     }
-    if (sLen == -2) {
+    if (sLen == RSA_PSS_SALTLEN_MAX) {
         sLen = emLen - hLen - 2;
     } else if (emLen < (hLen + sLen + 2)) {
         RSAerr(RSA_F_RSA_PADDING_ADD_PKCS1_PSS_MGF1,

doc/man1/pkeyutl.pod

@@ -215,11 +215,11 @@ specified.
 
 =item B<rsa_pss_saltlen:len>
 
-For B<pss> mode only this option specifies the salt length. Two special values
+For B<pss> mode only this option specifies the salt length. Three special
-are supported: -1 sets the salt length to the digest length. When signing -2
+values are supported: "digest" sets the salt length to the digest length,
-sets the salt length to the maximum permissible value. When verifying -2 causes
+"max" sets the salt length to the maximum permissible value. When verifying
-the salt length to be automatically determined based on the B<PSS> block
+"auto" causes the salt length to be automatically determined based on the
-structure.
+B<PSS> block structure.
 
 =item B<rsa_mgf1_md:digest>
 

doc/man3/EVP_PKEY_CTX_ctrl.pod

@@ -82,12 +82,13 @@ if this control is called. If it is not called then the first byte of the plaint
 buffer is expected to be the algorithm identifier byte.
 
 The EVP_PKEY_CTX_set_rsa_pss_saltlen() macro sets the RSA PSS salt length to
-B<len> as its name implies it is only supported for PSS padding.  Two special
+B<len> as its name implies it is only supported for PSS padding.  Three special
-values are supported: -1 sets the salt length to the digest length. When
+values are supported: RSA_PSS_SALTLEN_DIGEST sets the salt length to the
-signing -2 sets the salt length to the maximum permissible value. When
+digest length, RSA_PSS_SALTLEN_MAX sets the salt length to the maximum
-verifying -2 causes the salt length to be automatically determined based on the
+permissible value. When verifying RSA_PSS_SALTLEN_AUTO causes the salt length
-B<PSS> block structure. If this macro is not called a salt length value of -2
+to be automatically determined based on the B<PSS> block structure. If this
-is used by default.
+macro is not called maximum salt length is used when signing and auto detection
+when verifying is used by default.
 
 The EVP_PKEY_CTX_set_rsa_rsa_keygen_bits() macro sets the RSA key length for
 RSA key generation to B<bits>. If not specified 1024 bits is used.

doc/man3/EVP_PKEY_CTX_set_rsa_pss_keygen_md.pod

@@ -44,8 +44,9 @@ than B<PSS>. It is otherwise similar to the B<RSA> version.
 The EVP_PKEY_CTX_set_rsa_pss_saltlen() macro is used to set the salt length.
 If the key has usage restrictions then an error is returned if an attempt is
 made to set the salt length below the minimum value. It is otherwise similar
-to the B<RSA> operation except detection of the salt length (using -2) is
+to the B<RSA> operation except detection of the salt length (using
-not supported for verification if the key has usage restrictions.
+RSA_PSS_SALTLEN_AUTO is not supported for verification if the key has
+usage restrictions.
 
 The EVP_PKEY_CTX_set_signature_md() and EVP_PKEY_CTX_set_rsa_mgf1_md() macros
 are used to set the digest and MGF1 algorithms respectively. If the key has

include/openssl/rsa.h

@@ -94,6 +94,14 @@ extern "C" {
 # define EVP_PKEY_CTX_set_rsa_pss_saltlen(ctx, len) \
         RSA_pkey_ctx_ctrl(ctx, (EVP_PKEY_OP_SIGN|EVP_PKEY_OP_VERIFY), \
                           EVP_PKEY_CTRL_RSA_PSS_SALTLEN, len, NULL)
+/* Salt length matches digest */
+# define RSA_PSS_SALTLEN_DIGEST -1
+/* Verify only: auto detect salt length */
+# define RSA_PSS_SALTLEN_AUTO   -2
+/* Set salt length to maximum possible */
+# define RSA_PSS_SALTLEN_MAX    -3
+/* Old compatible max salt length for sign only */
+# define RSA_PSS_SALTLEN_MAX_SIGN    -2
 
 # define EVP_PKEY_CTX_set_rsa_pss_keygen_saltlen(ctx, len) \
         EVP_PKEY_CTX_ctrl(ctx, EVP_PKEY_RSA_PSS, EVP_PKEY_OP_KEYGEN, \

test/evptests.txt

@@ -2931,6 +2931,13 @@ Ctrl = digest:sha256
 Input="0123456789ABCDEF0123456789ABCDEF"
 Output=4DE433D5844043EF08D354DA03CB29068780D52706D7D1E4D50EFB7D58C9D547D83A747DDD0635A96B28F854E50145518482CB49E963054621B53C60C498D07C16E9C2789C893CF38D4D86900DE71BDE463BD2761D1271E358C7480A1AC0BAB930DDF39602AD1BC165B5D7436B516B7A7858E8EB7AB1C420EEB482F4D207F0E462B1724959320A084E13848D11D10FB593E66BF680BF6D3F345FC3E9C3DE60ABBAC37E1C6EC80A268C8D9FC49626C679097AA690BC1AA662B95EB8DB70390861AA0898229F9349B4B5FDD030D4928C47084708A933144BE23BD3C6E661B85B2C0EF9ED36D498D5B7320E8194D363D4AD478C059BAE804181965E0B81B663158A
 
+# Verify using salt length auto detect
+Verify = RSA-2048-PUBLIC
+Ctrl = rsa_padding_mode:pss
+Ctrl = rsa_pss_saltlen:auto
+Input="0123456789ABCDEF0123"
+Output = 6BF7EDC63A0BA184EEEC7F3020FEC8F5EBF38C2B76481881F48BCCE5796E7AB294548BA9AE810457C7723CABD1BDE94CF59CF7C0FC7461B22760C8ED703DD98E97BFDD61FA8D1181C411F6DEE5FF159F4850746D78EDEE385A363DC28E2CB373D5CAD7953F3BD5E639BE345732C03A1BDEA268814DA036EB1891C82D4012F3B903D86636055F87B96FC98806AD1B217685A4D754046A5DE0B0D7870664BE07902153EC85BA457BE7D7F89D7FE0F626D02A9CBBB2BB479DDA1A5CAE75247FB7BF6BFB15C1D3FD9E6B1573CCDBC72011C3B97716058BB11C7EA2E4E56ADAFE1F5DE6A7FD405AC5890100F9C3408EFFB5C73BF73F48177FF743B4B819D0699D507B
+
 # Digest too short
 Verify = RSA-2048-PUBLIC
 Ctrl = rsa_padding_mode:pss
@@ -3049,10 +3056,10 @@ Ctrl = digest:sha1
 Input="0123456789ABCDEF0123"
 Output = 3EFE09D88509027D837BFA5F8471CF7B69E6DF395DD999BB9CA42021F15722D9AC76670507C6BCFB73F64FB2211B611B8F140E76EBDB064BD762FDBA89D019E304A0D6B274E1C2FE1DF50005598A0306AF805416094E2A5BA60BC72BDE38CE061E853ED40F14967A8B9CA4DC739B462F89558F12FDF2D8D19FBEF16AD66FE2DDDA8BEE983ECBD873064244849D8D94B5B33F45E076871A47ED653E73257A2BE2DB3C0878094B0D2B6B682C8007DFD989425FB39A1FEEC9EED5876414601A49176EC344F5E3EDEE81CA2DDD29B7364F4638112CB3A547E2BC170E28CB66BDABE863754BE8AD5BA230567B575266F4B6B4CF81F28310ABF05351CC9E2DB85D00BF
 
-# Verify using default parameters, explicitly setting parameters -1 salt length
+# Verify explicitly setting parameters "digest" salt length
 Verify = RSA-PSS-DEFAULT
 Ctrl = rsa_padding_mode:pss
-Ctrl = rsa_pss_saltlen:-1
+Ctrl = rsa_pss_saltlen:digest
 Ctrl = digest:sha1
 Input="0123456789ABCDEF0123"
 Output = 3EFE09D88509027D837BFA5F8471CF7B69E6DF395DD999BB9CA42021F15722D9AC76670507C6BCFB73F64FB2211B611B8F140E76EBDB064BD762FDBA89D019E304A0D6B274E1C2FE1DF50005598A0306AF805416094E2A5BA60BC72BDE38CE061E853ED40F14967A8B9CA4DC739B462F89558F12FDF2D8D19FBEF16AD66FE2DDDA8BEE983ECBD873064244849D8D94B5B33F45E076871A47ED653E73257A2BE2DB3C0878094B0D2B6B682C8007DFD989425FB39A1FEEC9EED5876414601A49176EC344F5E3EDEE81CA2DDD29B7364F4638112CB3A547E2BC170E28CB66BDABE863754BE8AD5BA230567B575266F4B6B4CF81F28310ABF05351CC9E2DB85D00BF
@@ -3063,6 +3070,12 @@ Ctrl = rsa_pss_saltlen:30
 Input="0123456789ABCDEF0123"
 Output = 6BF7EDC63A0BA184EEEC7F3020FEC8F5EBF38C2B76481881F48BCCE5796E7AB294548BA9AE810457C7723CABD1BDE94CF59CF7C0FC7461B22760C8ED703DD98E97BFDD61FA8D1181C411F6DEE5FF159F4850746D78EDEE385A363DC28E2CB373D5CAD7953F3BD5E639BE345732C03A1BDEA268814DA036EB1891C82D4012F3B903D86636055F87B96FC98806AD1B217685A4D754046A5DE0B0D7870664BE07902153EC85BA457BE7D7F89D7FE0F626D02A9CBBB2BB479DDA1A5CAE75247FB7BF6BFB15C1D3FD9E6B1573CCDBC72011C3B97716058BB11C7EA2E4E56ADAFE1F5DE6A7FD405AC5890100F9C3408EFFB5C73BF73F48177FF743B4B819D0699D507B
 
+# Verify using maximum salt length
+Verify = RSA-PSS-DEFAULT
+Ctrl = rsa_pss_saltlen:max
+Input="0123456789ABCDEF0123"
+Output = 4470DCFE812DEE2E58E4301D4ED274AB348FE040B724B2CD1D8CD0914BFF375F0B86FCB32BFA8AEA9BD22BD7C4F1ADD4F3D215A5CFCC99055BAFECFC23800E9BECE19A08C66BEBC5802122D13A732E5958FC228DCC0B49B5B4B1154F032D8FA2F3564AA949C1310CC9266B0C47F86D449AC9D2E7678347E7266E2D7C888CCE1ADF44A109A293F8516AE2BD94CE220F26E137DB8E7A66BB9FCE052CDC1D0BE24D8CEBB20D10125F26B069F117044B9E1D16FDDAABCA5340AE1702F37D0E1C08A2E93801C0A41035C6C73DA02A0E32227EAFB0B85E79107B59650D0EE7DC32A6772CCCE90F06369B2880FE87ED76997BA61F5EA818091EE88F8B0D6F24D02A3FC6
+
 # Attempt to change salt length below minimum
 Verify = RSA-PSS-DEFAULT
 Ctrl = rsa_pss_saltlen:0