Bug 1204155 - Check if we're OOM when calling SECKEY_CopyPrivateKey() and SECKEY_CopyPublicKey() r=mt

dom/crypto/CryptoKey.cpp

@@ -394,26 +394,32 @@ nsresult CryptoKey::SetSymKey(const CryptoBuffer& aSymKey)
   return NS_OK;
 }
 
-void
+nsresult
 CryptoKey::SetPrivateKey(SECKEYPrivateKey* aPrivateKey)
 {
   nsNSSShutDownPreventionLock locker;
+
   if (!aPrivateKey || isAlreadyShutDown()) {
     mPrivateKey = nullptr;
-    return;
+    return NS_OK;
   }
+
   mPrivateKey = SECKEY_CopyPrivateKey(aPrivateKey);
+  return mPrivateKey ? NS_OK : NS_ERROR_OUT_OF_MEMORY;
 }
 
-void
+nsresult
 CryptoKey::SetPublicKey(SECKEYPublicKey* aPublicKey)
 {
   nsNSSShutDownPreventionLock locker;
+
   if (!aPublicKey || isAlreadyShutDown()) {
     mPublicKey = nullptr;
-    return;
+    return NS_OK;
   }
+
   mPublicKey = SECKEY_CopyPublicKey(aPublicKey);
+  return mPublicKey ? NS_OK : NS_ERROR_OUT_OF_MEMORY;
 }
 
 const CryptoBuffer&

dom/crypto/CryptoKey.h

@@ -128,8 +128,8 @@ public:
   static bool AllUsagesRecognized(const Sequence<nsString>& aUsages);
 
   nsresult SetSymKey(const CryptoBuffer& aSymKey);
-  void SetPrivateKey(SECKEYPrivateKey* aPrivateKey);
-  void SetPublicKey(SECKEYPublicKey* aPublicKey);
+  nsresult SetPrivateKey(SECKEYPrivateKey* aPrivateKey);
+  nsresult SetPublicKey(SECKEYPublicKey* aPublicKey);
 
   // Accessors for the keys themselves
   // Note: GetPrivateKey and GetPublicKey return copies of the internal

dom/crypto/WebCryptoTask.cpp

@@ -1581,7 +1581,10 @@ private:
         return NS_ERROR_DOM_DATA_ERR;
       }
 
-      mKey->SetPublicKey(pubKey.get());
+      if (NS_FAILED(mKey->SetPublicKey(pubKey.get()))) {
+        return NS_ERROR_DOM_OPERATION_ERR;
+      }
+
       mKey->SetType(CryptoKey::PUBLIC);
     } else if (mFormat.EqualsLiteral(WEBCRYPTO_KEY_FORMAT_PKCS8) ||
         (mFormat.EqualsLiteral(WEBCRYPTO_KEY_FORMAT_JWK) &&
@@ -1597,7 +1600,10 @@ private:
         return NS_ERROR_DOM_DATA_ERR;
       }
 
-      mKey->SetPrivateKey(privKey.get());
+      if (NS_FAILED(mKey->SetPrivateKey(privKey.get()))) {
+        return NS_ERROR_DOM_OPERATION_ERR;
+      }
+
       mKey->SetType(CryptoKey::PRIVATE);
       pubKey = SECKEY_ConvertToPublicKey(privKey.get());
       if (!pubKey) {
@@ -1715,7 +1721,10 @@ private:
         return NS_ERROR_DOM_DATA_ERR;
       }
 
-      mKey->SetPrivateKey(privKey.get());
+      if (NS_FAILED(mKey->SetPrivateKey(privKey.get()))) {
+        return NS_ERROR_DOM_OPERATION_ERR;
+      }
+
       mKey->SetType(CryptoKey::PRIVATE);
     } else if (mFormat.EqualsLiteral(WEBCRYPTO_KEY_FORMAT_RAW) ||
                mFormat.EqualsLiteral(WEBCRYPTO_KEY_FORMAT_SPKI) ||
@@ -1752,7 +1761,10 @@ private:
         }
       }
 
-      mKey->SetPublicKey(pubKey.get());
+      if (NS_FAILED(mKey->SetPublicKey(pubKey.get()))) {
+        return NS_ERROR_DOM_OPERATION_ERR;
+      }
+
       mKey->SetType(CryptoKey::PUBLIC);
     } else {
       return NS_ERROR_DOM_NOT_SUPPORTED_ERR;
@@ -1874,7 +1886,10 @@ private:
         ATTEMPT_BUFFER_ASSIGN(mGenerator, &pubKey->u.dh.base);
       }
 
-      mKey->SetPublicKey(pubKey.get());
+      if (NS_FAILED(mKey->SetPublicKey(pubKey.get()))) {
+        return NS_ERROR_DOM_OPERATION_ERR;
+      }
+
       mKey->SetType(CryptoKey::PUBLIC);
     } else {
       return NS_ERROR_DOM_NOT_SUPPORTED_ERR;
@@ -2388,13 +2403,15 @@ GenerateAsymmetricKeyTask::DoCrypto()
     return NS_ERROR_DOM_UNKNOWN_ERR;
   }
 
-  mKeyPair.mPrivateKey.get()->SetPrivateKey(mPrivateKey);
-  mKeyPair.mPublicKey.get()->SetPublicKey(mPublicKey);
+  nsresult rv = mKeyPair.mPrivateKey.get()->SetPrivateKey(mPrivateKey);
+  NS_ENSURE_SUCCESS(rv, NS_ERROR_DOM_OPERATION_ERR);
+  rv = mKeyPair.mPublicKey.get()->SetPublicKey(mPublicKey);
+  NS_ENSURE_SUCCESS(rv, NS_ERROR_DOM_OPERATION_ERR);
 
   // PK11_GenerateKeyPair() does not set a CKA_EC_POINT attribute on the
   // private key, we need this later when exporting to PKCS8 and JWK though.
   if (mMechanism == CKM_EC_KEY_PAIR_GEN) {
-    nsresult rv = mKeyPair.mPrivateKey->AddPublicKeyData(mPublicKey);
+    rv = mKeyPair.mPrivateKey->AddPublicKeyData(mPublicKey);
     NS_ENSURE_SUCCESS(rv, NS_ERROR_DOM_OPERATION_ERR);
   }