ossl_pkey.c: fix memory leak

* ext/openssl/ossl_pkey.c (ossl_pkey_verify): as EVP_VerifyFinal()
  finalizes only a copy of the digest context, the context must be
  cleaned up after initialization by EVP_MD_CTX_cleanup() or a
  memory leak will occur.  [ruby-core:62038] [Bug #9743]

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@45595 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,10 @@
+Wed Apr 16 09:51:16 2014  Nobuyoshi Nakada  <nobu@ruby-lang.org>
+
+	* ext/openssl/ossl_pkey.c (ossl_pkey_verify): as EVP_VerifyFinal()
+	  finalizes only a copy of the digest context, the context must be
+	  cleaned up after initialization by EVP_MD_CTX_cleanup() or a
+	  memory leak will occur.  [ruby-core:62038] [Bug #9743]
+
 Tue Apr 15 19:36:42 2014  NAKAMURA Usaku  <usa@ruby-lang.org>
 
 	* include/ruby/win32.h (rb_w32_cmdvector): removed.

ext/openssl/ossl_pkey.c

@@ -318,13 +318,16 @@ ossl_pkey_verify(VALUE self, VALUE digest, VALUE sig, VALUE data)
 {
     EVP_PKEY *pkey;
     EVP_MD_CTX ctx;
+    int result;
 
     GetPKey(self, pkey);
-    EVP_VerifyInit(&ctx, GetDigestPtr(digest));
     StringValue(sig);
     StringValue(data);
+    EVP_VerifyInit(&ctx, GetDigestPtr(digest));
     EVP_VerifyUpdate(&ctx, RSTRING_PTR(data), RSTRING_LEN(data));
-    switch (EVP_VerifyFinal(&ctx, (unsigned char *)RSTRING_PTR(sig), RSTRING_LENINT(sig), pkey)) {
+    result = EVP_VerifyFinal(&ctx, (unsigned char *)RSTRING_PTR(sig), RSTRING_LENINT(sig), pkey);
+    EVP_MD_CTX_cleanup(&ctx);
+    switch (result) {
     case 0:
 	return Qfalse;
     case 1:

test/openssl/test_pkey_rsa.rb

@@ -75,6 +75,36 @@ class OpenSSL::TestPKeyRSA < Test::Unit::TestCase
     assert(key.verify(digest, sig, data))
   end
 
+  def test_sign_verify_memory_leak
+    bug9743 = '[ruby-core:62038] [Bug #9743]'
+    assert_no_memory_leak(%w[-ropenssl], <<-PREP, <<-CODE, bug9743, rss: true)
+    data = 'Sign me!'
+    digest = OpenSSL::Digest::SHA512.new
+    pkey = OpenSSL::PKey::RSA.new(2048)
+    signature = pkey.sign(digest, data)
+    pub_key = pkey.public_key
+    PREP
+    20_000.times {
+      pub_key.verify(digest, signature, data)
+    }
+    CODE
+
+    assert_no_memory_leak(%w[-ropenssl], <<-PREP, <<-CODE, bug9743, rss: true)
+    data = 'Sign me!'
+    digest = OpenSSL::Digest::SHA512.new
+    pkey = OpenSSL::PKey::RSA.new(2048)
+    signature = pkey.sign(digest, data)
+    pub_key = pkey.public_key
+    PREP
+    20_000.times {
+      begin
+        pub_key.verify(digest, signature, 1)
+      rescue TypeError
+      end
+    }
+    CODE
+  end
+
   def test_digest_state_irrelevant_sign
     key = OpenSSL::TestUtils::TEST_KEY_RSA1024
     digest1 = OpenSSL::Digest::SHA1.new