Bug 1887870 - Reduce the number of mutexes used by mojo in TSAN builds, r=ipc-reviewers,mccr8

This changes the locking behaviour for IPC port mutexes in TSAN builds
to use a single shared mutex for all ports, rather than individual
mutexes per-port. This avoids the need to potentially lock a large
number of mutexes simultaneously when sending a large number of ports in
an IPC message.

I've tried to leave in the various debug assertions such that it still
acts like there are multiple mutexes under the hood. It is likely that
this could harm performance somewhat due to the increased contention,
however it should have no impact on actual release builds.

Differential Revision: https://phabricator.services.mozilla.com/D207073

ipc/chromium/src/mojo/core/ports/port.cc

@@ -8,6 +8,10 @@ namespace mojo {
 namespace core {
 namespace ports {
 
+#ifdef MOZ_USE_SINGLETON_PORT_MUTEX
+mozilla::StaticMutex detail::PortMutex::sSingleton;
+#endif
+
 // Used by std::{push,pop}_heap functions
 inline bool operator<(const mozilla::UniquePtr<Event>& a,
                       const mozilla::UniquePtr<Event>& b) {

ipc/chromium/src/mojo/core/ports/port.h

@@ -18,6 +18,18 @@
 #include "mozilla/RefPtr.h"
 #include "nsISupportsImpl.h"
 
+#ifdef MOZ_TSAN
+// In TSAN builds, a single static mutex is used for all ports, rather than
+// per-port mutexes, to avoid overloading the maximum 64 concurrently-held locks
+// limit of its deadlock detector when sending messages with a large number of
+// attached ports.
+#  define MOZ_USE_SINGLETON_PORT_MUTEX 1
+#endif
+
+#ifdef MOZ_USE_SINGLETON_PORT_MUTEX
+#  include "mozilla/StaticMutex.h"
+#endif
+
 namespace mojo {
 namespace core {
 namespace ports {
@@ -28,11 +40,18 @@ namespace detail {
 
 // Ports cannot use mozilla::Mutex, as the acquires-before relationships handled
 // by PortLocker can overload the debug-only deadlock detector.
-class MOZ_CAPABILITY("mutex") PortMutex : private ::mozilla::detail::MutexImpl {
+class MOZ_CAPABILITY("mutex") PortMutex
+#ifndef MOZ_USE_SINGLETON_PORT_MUTEX
+    : private ::mozilla::detail::MutexImpl
+#endif
+{
  public:
   void AssertCurrentThreadOwns() const MOZ_ASSERT_CAPABILITY(this) {
 #ifdef DEBUG
     MOZ_ASSERT(mOwningThread == PR_GetCurrentThread());
+#endif
+#ifdef MOZ_USE_SINGLETON_PORT_MUTEX
+    sSingleton.AssertCurrentThreadOwns();
 #endif
   }
 
@@ -40,8 +59,19 @@ class MOZ_CAPABILITY("mutex") PortMutex : private ::mozilla::detail::MutexImpl {
   // PortMutex should only be locked/unlocked via PortLocker
   friend class ::mojo::core::ports::PortLocker;
 
+#ifdef MOZ_USE_SINGLETON_PORT_MUTEX
+  // If the singleton mutex is in use, it must be locked before calling `Lock()`
+  // on any port, and must only be unlocked after calling `Unlock()` on every
+  // locked port.
+  static ::mozilla::StaticMutex sSingleton;
+#endif
+
   void Lock() MOZ_CAPABILITY_ACQUIRE() {
+#ifdef MOZ_USE_SINGLETON_PORT_MUTEX
+    sSingleton.AssertCurrentThreadOwns();
+#else
     ::mozilla::detail::MutexImpl::lock();
+#endif
 #ifdef DEBUG
     mOwningThread = PR_GetCurrentThread();
 #endif
@@ -51,7 +81,11 @@ class MOZ_CAPABILITY("mutex") PortMutex : private ::mozilla::detail::MutexImpl {
     MOZ_ASSERT(mOwningThread == PR_GetCurrentThread());
     mOwningThread = nullptr;
 #endif
+#ifdef MOZ_USE_SINGLETON_PORT_MUTEX
+    sSingleton.AssertCurrentThreadOwns();
+#else
     ::mozilla::detail::MutexImpl::unlock();
+#endif
   }
 
 #ifdef DEBUG

ipc/chromium/src/mojo/core/ports/port_locker.cc

@@ -36,6 +36,10 @@ PortLocker::PortLocker(const PortRef** port_refs, size_t num_ports)
   UpdateTLS(nullptr, this);
 #endif
 
+#ifdef MOZ_USE_SINGLETON_PORT_MUTEX
+  detail::PortMutex::sSingleton.Lock();
+#endif
+
   // Sort the ports by address to lock them in a globally consistent order.
   std::sort(
       port_refs_, port_refs_ + num_ports_,
@@ -52,6 +56,10 @@ PortLocker::~PortLocker() {
     port_refs_[i]->port()->lock_.Unlock();
   }
 
+#ifdef MOZ_USE_SINGLETON_PORT_MUTEX
+  detail::PortMutex::sSingleton.Unlock();
+#endif
+
 #ifdef DEBUG
   UpdateTLS(this, nullptr);
 #endif