winpr/crypto: Load legacy provider to fix rc4 with OpenSSL 3.0 (#7210)

* winpr/crypto: Exit cleanly when EVP_EncryptInit_ex fails

The `EVP_EncryptInit_ex` function may fail in certain configurations.
Consequently, FreeRDP segfaults in `EVP_CIPHER_CTX_set_key_length`.
Let's handle the `EVP_EncryptInit_ex` failures and exit cleanly in
such case.

* winpr/crypto: Load legacy provider to fix rc4 with OpenSSL 3.0

Currently, the `EVP_EncryptInit_ex` function fails for rc4 with OpenSSL 3.0.
This is becuase rc4 is provided by the legacy provider which is not loaded
by default. Let's explicitly load the legacy provider to make FreeRDP work
with OpenSSL 3.0.

Relates: https://github.com/openssl/openssl/issues/14392
Fixes: https://github.com/FreeRDP/FreeRDP/issues/6604
This commit is contained in:
Ondrej Holy 2021-08-03 15:28:58 +02:00 коммит произвёл GitHub
Родитель b7d612d843
Коммит 67f3fff2c8
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
1 изменённых файлов: 20 добавлений и 2 удалений

Просмотреть файл

@ -29,6 +29,9 @@
#include <openssl/rc4.h>
#include <openssl/des.h>
#include <openssl/evp.h>
#if defined(OPENSSL_VERSION_MAJOR) && (OPENSSL_VERSION_MAJOR >= 3)
#include <openssl/provider.h>
#endif
#endif
#ifdef WITH_MBEDTLS
@ -57,6 +60,12 @@ static WINPR_RC4_CTX* winpr_RC4_New_Internal(const BYTE* key, size_t keylen, BOO
if (keylen > INT_MAX)
return NULL;
#if defined(OPENSSL_VERSION_MAJOR) && (OPENSSL_VERSION_MAJOR >= 3)
if (OSSL_PROVIDER_load(NULL, "legacy") == NULL)
return NULL;
#endif
if (!(ctx = (WINPR_RC4_CTX*)EVP_CIPHER_CTX_new()))
return NULL;
@ -66,7 +75,12 @@ static WINPR_RC4_CTX* winpr_RC4_New_Internal(const BYTE* key, size_t keylen, BOO
return NULL;
EVP_CIPHER_CTX_init((EVP_CIPHER_CTX*)ctx);
EVP_EncryptInit_ex((EVP_CIPHER_CTX*)ctx, evp, NULL, NULL, NULL);
if (EVP_EncryptInit_ex((EVP_CIPHER_CTX*)ctx, evp, NULL, NULL, NULL) != 1)
{
EVP_CIPHER_CTX_free ((EVP_CIPHER_CTX*)ctx);
return NULL;
}
/* EVP_CIPH_FLAG_NON_FIPS_ALLOW does not exist before openssl 1.0.1 */
#if !(OPENSSL_VERSION_NUMBER < 0x10001000L)
@ -75,7 +89,11 @@ static WINPR_RC4_CTX* winpr_RC4_New_Internal(const BYTE* key, size_t keylen, BOO
#endif
EVP_CIPHER_CTX_set_key_length((EVP_CIPHER_CTX*)ctx, (int)keylen);
EVP_EncryptInit_ex((EVP_CIPHER_CTX*)ctx, NULL, NULL, key, NULL);
if (EVP_EncryptInit_ex((EVP_CIPHER_CTX*)ctx, NULL, NULL, key, NULL) != 1)
{
EVP_CIPHER_CTX_free ((EVP_CIPHER_CTX*)ctx);
return NULL;
}
#elif defined(WITH_MBEDTLS) && defined(MBEDTLS_ARC4_C)
if (!(ctx = (WINPR_RC4_CTX*)calloc(1, sizeof(mbedtls_arc4_context))))