Merge pull request #6938 from SparkiDev/rsa_pss_salt_len_openssl_compat_fix

RSA PSS OpenSSL compatibility verification: support AUTO

Author: dgarske

src/pk.c

@@ -3556,13 +3556,16 @@ int wolfSSL_RSA_verify_PKCS1_PSS(WOLFSSL_RSA *rsa, const unsigned char *mHash,
 
     if (ret == 1) {
         /* Calculate the salt length to use for special cases. */
-        /* TODO: use special case wolfCrypt values. */
         switch (saltLen) {
         /* Negative saltLen values are treated differently */
         case RSA_PSS_SALTLEN_DIGEST:
             saltLen = hashLen;
             break;
-        case RSA_PSS_SALTLEN_MAX_SIGN:
+        case RSA_PSS_SALTLEN_AUTO:
+        #ifdef WOLFSSL_PSS_SALT_LEN_DISCOVER
+            saltLen = RSA_PSS_SALT_LEN_DISCOVER;
+            break;
+        #endif
         case RSA_PSS_SALTLEN_MAX:
         #ifdef WOLFSSL_PSS_LONG_SALT
             saltLen = emLen - hashLen - 2;

wolfssl/openssl/rsa.h

@@ -54,6 +54,8 @@
 #define RSA_PSS_SALTLEN_DIGEST   (-1)
 /* Old max salt length */
 #define RSA_PSS_SALTLEN_MAX_SIGN (-2)
+/* Verification only value to indicate to discover salt length. */
+#define RSA_PSS_SALTLEN_AUTO     (-2)
 /* Max salt length */
 #define RSA_PSS_SALTLEN_MAX      (-3)
 #endif /* OPENSSL_EXTRA || OPENSSL_EXTRA_X509_SMALL */