bug 1182742 - allow users to override small key size errors r=rbarnes

Key size enforcement for TLS certificates happens at two levels: PSM and NSS.
PSM enforces a minimum of 1024 bits. NSS enforces a minimum of 1023 bits by
default. The NSS error is not overridable, but the PSM error is. This change
allows users to connect to devices with small RSA keys (as little as 512 bits)
using the certificate error override functionality.

MozReview-Commit-ID: 2TZ8c4I3hXC

--HG--
extra : rebase_source : a9c550f15261c711e789a670c90c129c65802ff0

config/external/nss/nss.symbols

@@ -268,6 +268,7 @@ NSS_Init
 NSS_Initialize
 NSS_InitWithMerge
 NSS_IsInitialized
+NSS_OptionSet
 NSS_NoDB_Init
 NSS_SecureMemcmp
 NSS_SetAlgorithmPolicy

security/manager/ssl/nsNSSComponent.cpp

@@ -2137,6 +2137,13 @@ InitializeCipherSuite()
   SEC_PKCS12SetPreferredCipher(PKCS12_DES_EDE3_168, 1);
   PORT_SetUCS2_ASCIIConversionFunction(pip_ucs2_ascii_conversion_fn);
 
+  // PSM enforces a minimum RSA key size of 1024 bits, which is overridable.
+  // NSS has its own minimum, which is not overridable (the default is 1023
+  // bits). This sets the NSS minimum to 512 bits so users can still connect to
+  // devices like wifi routers with woefully small keys (they would have to add
+  // an override to do so, but they already do for such devices).
+  NSS_OptionSet(NSS_RSA_MIN_KEY_SIZE, 512);
+
   // Observe preference change around cipher suite setting.
   return CipherSuiteChangeObserver::StartObserve();
 }

security/manager/ssl/tests/unit/test_cert_overrides.js

@@ -41,7 +41,7 @@ function check_telemetry() {
         "Actual and expected MOZILLA_PKIX_ERROR_CA_CERT_USED_AS_END_ENTITY counts should match");
   equal(histogram.counts[12], 1,
         "Actual and expected MOZILLA_PKIX_ERROR_V1_CERT_USED_AS_CA counts should match");
-  equal(histogram.counts[13], 0,
+  equal(histogram.counts[13], 1,
         "Actual and expected MOZILLA_PKIX_ERROR_INADEQUATE_KEY_SIZE counts should match");
   equal(histogram.counts[14], 2,
         "Actual and expected MOZILLA_PKIX_ERROR_NOT_YET_VALID_CERTIFICATE counts should match");
@@ -232,11 +232,10 @@ function add_simple_tests() {
                          Ci.nsICertOverrideService.ERROR_UNTRUSTED,
                          SEC_ERROR_CA_CERT_INVALID);
 
-  // This host presents a 1016-bit RSA key. NSS determines this key is too
-  // small and terminates the connection. The error is not overridable.
-  add_prevented_cert_override_test("inadequate-key-size-ee.example.com",
+  // This host presents a 1016-bit RSA key.
+  add_cert_override_test("inadequate-key-size-ee.example.com",
                          Ci.nsICertOverrideService.ERROR_UNTRUSTED,
-                                   SSL_ERROR_WEAK_SERVER_CERT_KEY);
+                         MOZILLA_PKIX_ERROR_INADEQUATE_KEY_SIZE);
 
   add_cert_override_test("ipAddressAsDNSNameInSAN.example.com",
                          Ci.nsICertOverrideService.ERROR_MISMATCH,