winpr/crypto: Load legacy provider to fix rc4 with OpenSSL 3.0 (#7210)

* winpr/crypto: Exit cleanly when EVP_EncryptInit_ex fails

The `EVP_EncryptInit_ex` function may fail in certain configurations.
Consequently, FreeRDP segfaults in `EVP_CIPHER_CTX_set_key_length`.
Let's handle the `EVP_EncryptInit_ex` failures and exit cleanly in
such case.

* winpr/crypto: Load legacy provider to fix rc4 with OpenSSL 3.0

Currently, the `EVP_EncryptInit_ex` function fails for rc4 with OpenSSL 3.0.
This is becuase rc4 is provided by the legacy provider which is not loaded
by default. Let's explicitly load the legacy provider to make FreeRDP work
with OpenSSL 3.0.

Relates: https://github.com/openssl/openssl/issues/14392
Fixes: https://github.com/FreeRDP/FreeRDP/issues/6604
(cherry picked from commit 67f3fff2c8)

Conflicts:
	winpr/libwinpr/crypto/cipher.c

winpr/libwinpr/crypto/cipher.c

@@ -29,6 +29,9 @@
 #include <openssl/rc4.h>
 #include <openssl/des.h>
 #include <openssl/evp.h>
+#if defined(OPENSSL_VERSION_MAJOR) && (OPENSSL_VERSION_MAJOR >= 3)
+#include <openssl/provider.h>
+#endif
 #endif
 
 #ifdef WITH_MBEDTLS
@@ -55,6 +58,14 @@ static WINPR_RC4_CTX* winpr_RC4_New_Internal(const BYTE* key, size_t keylen, BOO
 
 #if defined(WITH_OPENSSL)
 
+	if (keylen > INT_MAX)
+		return NULL;
+
+#if defined(OPENSSL_VERSION_MAJOR) && (OPENSSL_VERSION_MAJOR >= 3)
+	if (OSSL_PROVIDER_load(NULL, "legacy") == NULL)
+		return NULL;
+#endif
+
 	if (!(ctx = (WINPR_RC4_CTX*)EVP_CIPHER_CTX_new()))
 		return NULL;
 
@@ -64,7 +75,12 @@ static WINPR_RC4_CTX* winpr_RC4_New_Internal(const BYTE* key, size_t keylen, BOO
 		return NULL;
 
 	EVP_CIPHER_CTX_init((EVP_CIPHER_CTX*)ctx);
-	EVP_EncryptInit_ex((EVP_CIPHER_CTX*)ctx, evp, NULL, NULL, NULL);
+	if (EVP_EncryptInit_ex((EVP_CIPHER_CTX*)ctx, evp, NULL, NULL, NULL) != 1)
+	{
+		EVP_CIPHER_CTX_free ((EVP_CIPHER_CTX*)ctx);
+		return NULL;
+	}
+
 	/* EVP_CIPH_FLAG_NON_FIPS_ALLOW does not exist before openssl 1.0.1 */
 #if !(OPENSSL_VERSION_NUMBER < 0x10001000L)
 
@@ -72,8 +88,12 @@ static WINPR_RC4_CTX* winpr_RC4_New_Internal(const BYTE* key, size_t keylen, BOO
 		EVP_CIPHER_CTX_set_flags((EVP_CIPHER_CTX*)ctx, EVP_CIPH_FLAG_NON_FIPS_ALLOW);
 
 #endif
-	EVP_CIPHER_CTX_set_key_length((EVP_CIPHER_CTX*)ctx, keylen);
-	EVP_EncryptInit_ex((EVP_CIPHER_CTX*)ctx, NULL, NULL, key, NULL);
+	EVP_CIPHER_CTX_set_key_length((EVP_CIPHER_CTX*)ctx, (int)keylen);
+	if (EVP_EncryptInit_ex((EVP_CIPHER_CTX*)ctx, NULL, NULL, key, NULL) != 1)
+	{
+		EVP_CIPHER_CTX_free ((EVP_CIPHER_CTX*)ctx);
+		return NULL;
+	}
 #elif defined(WITH_MBEDTLS) && defined(MBEDTLS_ARC4_C)
 
 	if (!(ctx = (WINPR_RC4_CTX*)calloc(1, sizeof(mbedtls_arc4_context))))