[ruby/openssl] pkey/dh, pkey/ec: use EVP_PKEY_check() family

Use EVP_PKEY_param_check() instead of DH_check() if available. Also,
use EVP_PKEY_public_check() instead of EC_KEY_check_key().

EVP_PKEY_*check() is part of the EVP API and is meant to replace those
low-level functions. They were added by OpenSSL 1.1.1. It is currently
not provided by LibreSSL.

https://github.com/ruby/openssl/commit/797e9f8e08

ext/openssl/extconf.rb

@@ -165,6 +165,9 @@ have_func("TS_RESP_CTX_set_time_cb")
 have_func("EVP_PBE_scrypt")
 have_func("SSL_CTX_set_post_handshake_auth")
 
+# added in 1.1.1
+have_func("EVP_PKEY_check")
+
 Logging::message "=== Checking done. ===\n"
 
 create_header

ext/openssl/ossl_pkey_dh.c

@@ -273,19 +273,38 @@ ossl_dh_get_params(VALUE self)
  * Validates the Diffie-Hellman parameters associated with this instance.
  * It checks whether a safe prime and a suitable generator are used. If this
  * is not the case, +false+ is returned.
+ *
+ * See also the man page EVP_PKEY_param_check(3).
  */
 static VALUE
 ossl_dh_check_params(VALUE self)
 {
+    int ret;
+#ifdef HAVE_EVP_PKEY_CHECK
+    EVP_PKEY *pkey;
+    EVP_PKEY_CTX *pctx;
+
+    GetPKey(self, pkey);
+    pctx = EVP_PKEY_CTX_new(pkey, /* engine */NULL);
+    if (!pctx)
+        ossl_raise(eDHError, "EVP_PKEY_CTX_new");
+    ret = EVP_PKEY_param_check(pctx);
+    EVP_PKEY_CTX_free(pctx);
+#else
     DH *dh;
     int codes;
 
     GetDH(self, dh);
-    if (!DH_check(dh, &codes)) {
-	return Qfalse;
-    }
+    ret = DH_check(dh, &codes) == 1 && codes == 0;
+#endif
 
-    return codes == 0 ? Qtrue : Qfalse;
+    if (ret == 1)
+        return Qtrue;
+    else {
+        /* DH_check_ex() will put error entry on failure */
+        ossl_clear_error();
+        return Qfalse;
+    }
 }
 
 /*

ext/openssl/ossl_pkey_ec.c

@@ -438,20 +438,35 @@ static VALUE ossl_ec_key_generate_key(VALUE self)
 }
 
 /*
- *  call-seq:
- *     key.check_key   => true
+ * call-seq:
+ *    key.check_key   => true
  *
- *  Raises an exception if the key is invalid.
+ * Raises an exception if the key is invalid.
  *
- *  See the OpenSSL documentation for EC_KEY_check_key()
+ * See also the man page EVP_PKEY_public_check(3).
  */
 static VALUE ossl_ec_key_check_key(VALUE self)
 {
+#ifdef HAVE_EVP_PKEY_CHECK
+    EVP_PKEY *pkey;
+    EVP_PKEY_CTX *pctx;
+    int ret;
+
+    GetPKey(self, pkey);
+    pctx = EVP_PKEY_CTX_new(pkey, /* engine */NULL);
+    if (!pctx)
+        ossl_raise(eDHError, "EVP_PKEY_CTX_new");
+    ret = EVP_PKEY_public_check(pctx);
+    EVP_PKEY_CTX_free(pctx);
+    if (ret != 1)
+        ossl_raise(eECError, "EVP_PKEY_public_check");
+#else
     EC_KEY *ec;
 
     GetEC(self, ec);
     if (EC_KEY_check_key(ec) != 1)
 	ossl_raise(eECError, "EC_KEY_check_key");
+#endif
 
     return Qtrue;
 }

test/openssl/test_pkey_dh.rb

@@ -86,6 +86,22 @@ class OpenSSL::TestPKeyDH < OpenSSL::PKeyTestCase
     assert_equal(dh.compute_key(dh2.pub_key), dh2.compute_key(dh.pub_key))
   end
 
+  def test_params_ok?
+    dh0 = Fixtures.pkey("dh1024")
+
+    dh1 = OpenSSL::PKey::DH.new(OpenSSL::ASN1::Sequence([
+      OpenSSL::ASN1::Integer(dh0.p),
+      OpenSSL::ASN1::Integer(dh0.g)
+    ]))
+    assert_equal(true, dh1.params_ok?)
+
+    dh2 = OpenSSL::PKey::DH.new(OpenSSL::ASN1::Sequence([
+      OpenSSL::ASN1::Integer(dh0.p + 1),
+      OpenSSL::ASN1::Integer(dh0.g)
+    ]))
+    assert_equal(false, dh2.params_ok?)
+  end
+
   def test_dup
     dh = Fixtures.pkey("dh1024")
     dh2 = dh.dup