Bug 1788856 - initialize NSS as needed in nsNSSCertificate r=jschanck

Previously, instantiating an nsIX509Cert (implemented by nsNSSCertificate)
would cause NSS to be initialized. However, if 'new nsNSSCertificate()' was
called directly (rather than going through XPCOM), NSS would not be
initialized. This didn't seem to be a problem until bug 1787942 changed how
nsITransportSecurityInfo was sent between processes for PHttpChannel and
HttpChannelOnStartRequestArgs (namely, by using the direct IPC support rather
than first serializing to a string, sending it over IPC, and then deserializing
it). That direct IPC implementation uses 'new nsNSSCertificate()', which is now
a problem.

nsNSSCertificate used to make extensive use of NSS, which warranted ensuring
NSS was initialized before creating one at all. Now, as of bug 1748341, the
cases where nsNSSCertificate uses NSS are limited and clearly delineated.
Accordinly, this change makes it so nsNSSCertificate only initializes NSS if
and when it needs it, rather than relying on the XPCOM boilerplate to
initialize NSS first.

Differential Revision: https://phabricator.services.mozilla.com/D156353

security/manager/ssl/components.conf

@@ -51,7 +51,7 @@ Classes = [
         'cid': '{660a3226-915c-4ffb-bb20-8985a632df05}',
         'contract_ids': [],
         'type': 'nsNSSCertificate',
-        'legacy_constructor': 'mozilla::psm::NSSConstructor<nsNSSCertificate>',
+        'headers': ['nsNSSCertificate.h'],
     },
     {
         'cid': '{fb0bbc5c-452e-4783-b32c-80124693d871}',

security/manager/ssl/nsNSSCertificate.cpp

@@ -78,6 +78,11 @@ UniqueCERTCertificate nsNSSCertificate::GetOrInstantiateCert() {
   if (maybeCert.isSome()) {
     return UniqueCERTCertificate(CERT_DupCertificate((*maybeCert).get()));
   }
+
+  if (!EnsureNSSInitializedChromeOrContent()) {
+    return nullptr;
+  }
+
   SECItem derItem = {siBuffer, mDER.Elements(),
                      static_cast<unsigned int>(mDER.Length())};
   UniqueCERTCertificate cert(CERT_NewTempCertificate(
@@ -450,6 +455,11 @@ nsNSSCertificate::GetSerialNumber(nsAString& _serialNumber) {
 nsresult nsNSSCertificate::GetCertificateHash(nsAString& aFingerprint,
                                               SECOidTag aHashAlg) {
   aFingerprint.Truncate();
+
+  if (!EnsureNSSInitializedChromeOrContent()) {
+    return NS_ERROR_NOT_AVAILABLE;
+  }
+
   nsTArray<uint8_t> digestArray;
   nsresult rv =
       Digest::DigestBuf(aHashAlg, mDER.Elements(), mDER.Length(), digestArray);
@@ -505,6 +515,10 @@ nsNSSCertificate::GetSha256SubjectPublicKeyInfoDigest(
     nsACString& aSha256SPKIDigest) {
   aSha256SPKIDigest.Truncate();
 
+  if (!EnsureNSSInitializedChromeOrContent()) {
+    return NS_ERROR_NOT_AVAILABLE;
+  }
+
   pkix::Input certInput;
   pkix::Result result = certInput.Init(mDER.Elements(), mDER.Length());
   if (result != pkix::Result::Success) {

security/manager/ssl/nsNSSModule.cpp

@@ -7,7 +7,6 @@
 #include "nsNSSModule.h"
 
 #include "ContentSignatureVerifier.h"
-#include "NSSErrorsService.h"
 #include "OSKeyStore.h"
 #include "OSReauthenticator.h"
 #include "PKCS11ModuleDB.h"
@@ -16,20 +15,12 @@
 #include "mozilla/MacroArgs.h"
 #include "mozilla/ModuleUtils.h"
 #include "mozilla/SyncRunnable.h"
-#include "nsCURILoader.h"
-#include "nsCryptoHash.h"
-#include "nsNSSCertificate.h"
-#include "nsNSSCertificateDB.h"
-#include "nsNSSComponent.h"
-#include "nsNSSVersion.h"
-#include "nsNetCID.h"
-#include "nsPK11TokenDB.h"
-#include "nsPKCS11Slot.h"
-#include "nsRandomGenerator.h"
-#include "nsSecureBrowserUI.h"
-#include "nsXULAppAPI.h"
-
 #include "nsCertTree.h"
+#include "nsCryptoHash.h"
+#include "nsNSSCertificateDB.h"
+#include "nsPK11TokenDB.h"
+#include "nsRandomGenerator.h"
+#include "nsXULAppAPI.h"
 
 namespace mozilla {
 namespace psm {
@@ -104,7 +95,6 @@ static nsresult Constructor(REFNSIID aIID, void** aResult) {
 IMPL(SecretDecoderRing, nullptr)
 IMPL(nsPK11TokenDB, nullptr)
 IMPL(PKCS11ModuleDB, nullptr)
-IMPL(nsNSSCertificate, nullptr, ProcessRestriction::AnyProcess)
 IMPL(nsNSSCertificateDB, nullptr)
 IMPL(nsCertTree, nullptr)
 IMPL(nsCryptoHash, nullptr, ProcessRestriction::AnyProcess)