When verifying RSA-PSS signature with auto-saltlen always fallback (#34)

+ Previously would only fallback for RSA keys, not specifically RSA-PSS
  keys
+ Add test cases for DigestSign/Verify with RSA-PSS

SslPlay/SslPlay.cpp

@@ -654,6 +654,15 @@ void TestRsaDigestSignVerify(
         //     goto end;
         // }
     }
+    if (padding == RSA_PKCS1_PSS_PADDING)
+    {
+        printf("Command EVP_PKEY_CTX_set_rsa_pss_saltlen RSA_PSS_SALTLEN_DIGEST\n");
+        if (EVP_PKEY_CTX_set_rsa_pss_saltlen(pSigningKeyContext, RSA_PSS_SALTLEN_DIGEST) <= 0)
+        {
+            handleOpenSSLError("");
+            goto end;
+        }
+    }
 
     printf("Command EVP_DigestSignUpdate\n");
     if (EVP_DigestSignUpdate(RSASignCtx, message, message_len) <= 0) {
@@ -703,6 +712,15 @@ void TestRsaDigestSignVerify(
         //     goto end;
         // }
     }
+    if (padding == RSA_PKCS1_PSS_PADDING)
+    {
+        printf("Command EVP_PKEY_CTX_set_rsa_pss_saltlen RSA_PSS_SALTLEN_DIGEST\n");
+        if (EVP_PKEY_CTX_set_rsa_pss_saltlen(pVerificationKeyContext, RSA_PSS_SALTLEN_DIGEST) <= 0)
+        {
+            handleOpenSSLError("");
+            goto end;
+        }
+    }
     printf("Command EVP_DigestVerifyUpdate\n");
     if (EVP_DigestVerifyUpdate(RSAVerifyCtx, message, message_len) <= 0) {
         handleOpenSSLError("");
@@ -1015,7 +1033,10 @@ void TestRsaEvp(int modulus, uint32_t exponent)
     TestRsaDigestSignVerify(privateKey, publicKey, "RSA_PKCS1_PADDING", RSA_PKCS1_PADDING, "EVP_sha256", EVP_sha256());
     TestRsaDigestSignVerify(privateKey, publicKey, "RSA_PKCS1_PADDING", RSA_PKCS1_PADDING, "EVP_sha384", EVP_sha384());
     TestRsaDigestSignVerify(privateKey, publicKey, "RSA_PKCS1_PADDING", RSA_PKCS1_PADDING, "EVP_sha512", EVP_sha512());
+    printf("%s", SeparatorLine);
 
+    TestRsaDigestSignVerify(privateKey, publicKey, "RSA_PKCS1_PSS_PADDING", RSA_PKCS1_PSS_PADDING, "EVP_sha256", EVP_sha256());
+    TestRsaDigestSignVerify(privateKeyPss, publicKeyPss, "RSA_PKCS1_PSS_PADDING", RSA_PKCS1_PSS_PADDING, "EVP_sha256", EVP_sha256());
     printf("%s", SeparatorLine);
 
     //

SymCryptEngine/src/scossl_pkey_meths.c

@@ -50,12 +50,31 @@ static int scossl_pkey_rsa_sign(_Inout_ EVP_PKEY_CTX *ctx, _Out_writes_bytes_(*s
     return _openssl_pkey_rsa_sign(ctx, sig, siglen, tbs, tbslen);
 }
 
+// Call SymCrypt engine RSA-PSS verify, unless auto salt-length specified (not yet supported by SymCrypt)
+static int scossl_pkey_rsapss_verify(_Inout_ EVP_PKEY_CTX *ctx, _In_reads_bytes_(siglen) const unsigned char *sig, size_t siglen,
+                                      _In_reads_bytes_(tbslen) const unsigned char *tbs, size_t tbslen)
+{
+    int cbSalt = RSA_PSS_SALTLEN_DIGEST;
+
+    if( EVP_PKEY_CTX_get_rsa_pss_saltlen(ctx, &cbSalt) <= 0 )
+    {
+        SCOSSL_LOG_ERROR("Failed to get cbSalt");
+        return SCOSSL_UNSUPPORTED;
+    }
+    if( cbSalt != RSA_PSS_SALTLEN_AUTO )
+    {
+        return scossl_rsapss_verify(ctx, sig, siglen, tbs, tbslen);
+    }
+    SCOSSL_LOG_INFO("SymCrypt Engine does not support RSA_PSS_SALTLEN_AUTO saltlen - falling back to OpenSSL");
+
+    return _openssl_pkey_rsa_verify(ctx, sig, siglen, tbs, tbslen);
+}
+
 // Call SymCrypt engine verify if PSS padding, otherwise OpenSSL version.
 static int scossl_pkey_rsa_verify(_Inout_ EVP_PKEY_CTX *ctx, _In_reads_bytes_(siglen) const unsigned char *sig, size_t siglen,
                                     _In_reads_bytes_(tbslen) const unsigned char *tbs, size_t tbslen)
 {
     int padding;
-    int cbSalt = RSA_PSS_SALTLEN_DIGEST;
 
     if( EVP_PKEY_CTX_get_rsa_padding(ctx, &padding) <= 0 )
     {
@@ -65,17 +84,7 @@ static int scossl_pkey_rsa_verify(_Inout_ EVP_PKEY_CTX *ctx, _In_reads_bytes_(si
 
     if( padding == RSA_PKCS1_PSS_PADDING )
     {
-        if( EVP_PKEY_CTX_get_rsa_pss_saltlen(ctx, &cbSalt) <= 0 )
-        {
-            SCOSSL_LOG_ERROR("Failed to get cbSalt");
-            return SCOSSL_UNSUPPORTED;
-        }
-        if( cbSalt != RSA_PSS_SALTLEN_AUTO )
-        {
-
-            return scossl_rsapss_verify(ctx, sig, siglen, tbs, tbslen);
-        }
-        SCOSSL_LOG_INFO("SymCrypt Engine does not support RSA_PSS_SALTLEN_AUTO saltlen - falling back to OpenSSL");
+        return scossl_pkey_rsapss_verify(ctx, sig, siglen, tbs, tbslen);
     }
 
     return _openssl_pkey_rsa_verify(ctx, sig, siglen, tbs, tbslen);
@@ -132,7 +141,7 @@ static EVP_PKEY_METHOD *scossl_pkey_rsa_pss(void)
         EVP_PKEY_meth_get_verify(_scossl_pkey_rsa_pss, &pverify_init, &pverify);
 
         EVP_PKEY_meth_set_sign(_scossl_pkey_rsa_pss, psign_init, scossl_rsapss_sign);
-        EVP_PKEY_meth_set_verify(_scossl_pkey_rsa_pss, pverify_init, scossl_rsapss_verify);
+        EVP_PKEY_meth_set_verify(_scossl_pkey_rsa_pss, pverify_init, scossl_pkey_rsapss_verify);
     }
     return _scossl_pkey_rsa_pss;
 }