Bug 1874962 - add Xyber768 support under a pref. r=keeler,necko-reviewers,kershaw

Differential Revision: https://phabricator.services.mozilla.com/D198744

modules/libpref/init/StaticPrefList.yaml

@@ -14190,6 +14190,11 @@
   value: true
   mirror: always
 
+- name: security.tls.enable_kyber
+  type: RelaxedAtomicBool
+  value: false
+  mirror: always
+
 - name: security.ssl.treat_unsafe_negotiation_as_broken
   type: RelaxedAtomicBool
   value: false

netwerk/protocol/http/Http2Session.cpp

@@ -4107,7 +4107,13 @@ nsresult Http2Session::ConfirmTLSProfile() {
   }
 
   uint16_t kea = ssl->GetKEAUsed();
-  if (kea != ssl_kea_dh && kea != ssl_kea_ecdh) {
+  if (kea == ssl_kea_ecdh_hybrid && !StaticPrefs::security_tls_enable_kyber()) {
+    LOG3(("Http2Session::ConfirmTLSProfile %p FAILED due to disabled KEA %d\n",
+          this, kea));
+    return SessionError(INADEQUATE_SECURITY);
+  }
+
+  if (kea != ssl_kea_dh && kea != ssl_kea_ecdh && kea != ssl_kea_ecdh_hybrid) {
     LOG3(("Http2Session::ConfirmTLSProfile %p FAILED due to invalid KEA %d\n",
           this, kea));
     return SessionError(INADEQUATE_SECURITY);

security/manager/ssl/nsNSSCallbacks.cpp

@@ -656,6 +656,9 @@ nsCString getKeaGroupName(uint32_t aKeaGroup) {
     case ssl_grp_ec_curve25519:
       groupName = "x25519"_ns;
       break;
+    case ssl_grp_kem_xyber768d00:
+      groupName = "xyber768d00"_ns;
+      break;
     case ssl_grp_ffdhe_2048:
       groupName = "FF 2048"_ns;
       break;
@@ -820,6 +823,8 @@ SECStatus CanFalseStartCallback(PRFileDesc* fd, void* client_data,
   }
 
   // See bug 952863 for why ECDHE is allowed, but DHE (and RSA) are not.
+  // Also note that ecdh_hybrid groups are not supported in TLS 1.2 and are out
+  // of scope.
   if (channelInfo.keaType != ssl_kea_ecdh) {
     MOZ_LOG(gPIPNSSLog, LogLevel::Debug,
             ("CanFalseStartCallback [%p] failed - "
@@ -1019,7 +1024,7 @@ void HandshakeCallback(PRFileDesc* fd, void* client_data) {
   if (rv != SECSuccess) {
     return;
   }
-  // keyExchange null=0, rsa=1, dh=2, fortezza=3, ecdh=4
+  // keyExchange null=0, rsa=1, dh=2, fortezza=3, ecdh=4, ecdh_hybrid=5
   Telemetry::Accumulate(infoObject->IsFullHandshake()
                             ? Telemetry::SSL_KEY_EXCHANGE_ALGORITHM_FULL
                             : Telemetry::SSL_KEY_EXCHANGE_ALGORITHM_RESUMED,
@@ -1039,6 +1044,9 @@ void HandshakeCallback(PRFileDesc* fd, void* client_data) {
         AccumulateECCCurve(Telemetry::SSL_KEA_ECDHE_CURVE_FULL,
                            channelInfo.keaKeyBits);
         break;
+      case ssl_kea_ecdh_hybrid:
+        // Bug 1874963: Add probes for Xyber768d00
+        break;
       default:
         MOZ_CRASH("impossible KEA");
         break;

security/manager/ssl/nsNSSIOLayer.cpp

@@ -1430,19 +1430,39 @@ static nsresult nsSSLIOLayerSetOptions(PRFileDesc* fd, bool forSTARTTLS,
   }
 
   // Include a modest set of named groups.
-  // Please change getKeaGroupName in nsNSSCallbacks.cpp when changing the list
+  // Please change getKeaGroupName in nsNSSCallbacks.cpp when changing the lists
   // here.
-  const SSLNamedGroup namedGroups[] = {
-      ssl_grp_ec_curve25519, ssl_grp_ec_secp256r1, ssl_grp_ec_secp384r1,
-      ssl_grp_ec_secp521r1,  ssl_grp_ffdhe_2048,   ssl_grp_ffdhe_3072};
-  if (SECSuccess != SSL_NamedGroupConfig(fd, namedGroups,
-                                         mozilla::ArrayLength(namedGroups))) {
-    return NS_ERROR_FAILURE;
-  }
-  // This ensures that we send key shares for X25519 and P-256 in TLS 1.3, so
-  // that servers are less likely to use HelloRetryRequest.
-  if (SECSuccess != SSL_SendAdditionalKeyShares(fd, 1)) {
-    return NS_ERROR_FAILURE;
+  if (StaticPrefs::security_tls_enable_kyber() &&
+      range.max >= SSL_LIBRARY_VERSION_TLS_1_3 &&
+      !(infoObject->GetProviderFlags() &
+        (nsISocketProvider::BE_CONSERVATIVE | nsISocketTransport::IS_RETRY))) {
+    const SSLNamedGroup namedGroups[] = {
+        ssl_grp_kem_xyber768d00, ssl_grp_ec_curve25519, ssl_grp_ec_secp256r1,
+        ssl_grp_ec_secp384r1,    ssl_grp_ec_secp521r1,  ssl_grp_ffdhe_2048,
+        ssl_grp_ffdhe_3072};
+    if (SECSuccess != SSL_NamedGroupConfig(fd, namedGroups,
+                                           mozilla::ArrayLength(namedGroups))) {
+      return NS_ERROR_FAILURE;
+    }
+    // This ensures that we send key shares for Xyber768D00, X25519, and P-256
+    // in TLS 1.3, so that servers are less likely to use HelloRetryRequest.
+    if (SECSuccess != SSL_SendAdditionalKeyShares(fd, 2)) {
+      return NS_ERROR_FAILURE;
+    }
+  } else {
+    const SSLNamedGroup namedGroups[] = {
+        ssl_grp_ec_curve25519, ssl_grp_ec_secp256r1, ssl_grp_ec_secp384r1,
+        ssl_grp_ec_secp521r1,  ssl_grp_ffdhe_2048,   ssl_grp_ffdhe_3072};
+    // Skip the |ssl_grp_kem_xyber768d00| entry.
+    if (SECSuccess != SSL_NamedGroupConfig(fd, namedGroups,
+                                           mozilla::ArrayLength(namedGroups))) {
+      return NS_ERROR_FAILURE;
+    }
+    // This ensures that we send key shares for X25519 and P-256 in TLS 1.3, so
+    // that servers are less likely to use HelloRetryRequest.
+    if (SECSuccess != SSL_SendAdditionalKeyShares(fd, 1)) {
+      return NS_ERROR_FAILURE;
+    }
   }
 
   // NOTE: Should this list ever include ssl_sig_rsa_pss_pss_sha* (or should