Bug 1390476 - Lock helper thread state while changing simulated OOM thread r=lth

js/src/builtin/TestingFunctions.cpp

@@ -1478,7 +1478,6 @@ SetupOOMFailure(JSContext* cx, bool failAlways, unsigned argc, Value* vp)
         return false;
     }
 
-    HelperThreadState().waitForAllThreads();
     js::oom::SimulateOOMAfter(count, targetThread, failAlways);
     args.rval().setUndefined();
     return true;
@@ -1587,9 +1586,6 @@ OOMTest(JSContext* cx, unsigned argc, Value* vp)
         if (verbose)
             fprintf(stderr, "thread %d\n", thread);
 
-        HelperThreadState().waitForAllThreads();
-        js::oom::targetThread = thread;
-
         unsigned allocation = 1;
         bool handledOOM;
         do {

js/src/jsutil.cpp

@@ -10,6 +10,7 @@
 
 #include "mozilla/Assertions.h"
 #include "mozilla/MathAlgorithms.h"
+#include "mozilla/Maybe.h"
 #include "mozilla/PodOperations.h"
 #include "mozilla/ThreadLocal.h"
 
@@ -59,8 +60,20 @@ GetThreadType(void) {
     return threadType.get();
 }
 
+static inline bool
+IsHelperThreadType(uint32_t thread)
+{
+    return thread != THREAD_TYPE_NONE && thread != THREAD_TYPE_COOPERATING;
+}
+
 void
 SimulateOOMAfter(uint64_t allocations, uint32_t thread, bool always) {
+    Maybe<AutoLockHelperThreadState> lock;
+    if (IsHelperThreadType(targetThread) || IsHelperThreadType(thread)) {
+        lock.emplace();
+        HelperThreadState().waitForAllThreadsLocked(lock.ref());
+    }
+
     MOZ_ASSERT(counter + allocations > counter);
     MOZ_ASSERT(thread > js::THREAD_TYPE_NONE && thread < js::THREAD_TYPE_MAX);
     targetThread = thread;
@@ -70,8 +83,12 @@ SimulateOOMAfter(uint64_t allocations, uint32_t thread, bool always) {
 
 void
 ResetSimulatedOOM() {
-    if (targetThread != THREAD_TYPE_NONE && targetThread != THREAD_TYPE_COOPERATING)
+    Maybe<AutoLockHelperThreadState> lock;
-        HelperThreadState().waitForAllThreads();
+    if (IsHelperThreadType(targetThread)) {
+        lock.emplace();
+        HelperThreadState().waitForAllThreadsLocked(lock.ref());
+    }
+
     targetThread = THREAD_TYPE_NONE;
     maxAllocations = UINT64_MAX;
     failAlways = false;

js/src/vm/HelperThreads.cpp

@@ -113,11 +113,9 @@ js::StartOffThreadWasmTier2Generator(wasm::Tier2GeneratorTask* task)
     return true;
 }
 
-void
+static void
-js::CancelOffThreadWasmTier2Generator()
+CancelOffThreadWasmTier2GeneratorLocked(AutoLockHelperThreadState& lock)
 {
-    AutoLockHelperThreadState lock;
-
     if (!HelperThreadState().threads)
         return;
 
@@ -160,6 +158,13 @@ js::CancelOffThreadWasmTier2Generator()
     }
 }
 
+void
+js::CancelOffThreadWasmTier2Generator()
+{
+    AutoLockHelperThreadState lock;
+    CancelOffThreadWasmTier2GeneratorLocked(lock);
+}
+
 bool
 js::StartOffThreadIonCompile(JSContext* cx, jit::IonBuilder* builder)
 {
@@ -254,14 +259,10 @@ IonBuilderMatches(const CompilationSelector& selector, jit::IonBuilder* builder)
     return selector.match(BuilderMatches{builder});
 }
 
-void
+static void
-js::CancelOffThreadIonCompile(const CompilationSelector& selector, bool discardLazyLinkList)
+CancelOffThreadIonCompileLocked(const CompilationSelector& selector, bool discardLazyLinkList,
+                                AutoLockHelperThreadState& lock)
 {
-    if (!JitDataStructuresExist(selector))
-        return;
-
-    AutoLockHelperThreadState lock;
-
     if (!HelperThreadState().threads)
         return;
 
@@ -325,6 +326,16 @@ js::CancelOffThreadIonCompile(const CompilationSelector& selector, bool discardL
     }
 }
 
+void
+js::CancelOffThreadIonCompile(const CompilationSelector& selector, bool discardLazyLinkList)
+{
+    if (!JitDataStructuresExist(selector))
+        return;
+
+    AutoLockHelperThreadState lock;
+    CancelOffThreadIonCompileLocked(selector, discardLazyLinkList, lock);
+}
+
 #ifdef DEBUG
 bool
 js::HasOffThreadIonCompile(JSCompartment* comp)
@@ -1020,10 +1031,16 @@ GlobalHelperThreadState::hasActiveThreads(const AutoLockHelperThreadState&)
 void
 GlobalHelperThreadState::waitForAllThreads()
 {
-    CancelOffThreadIonCompile();
-    CancelOffThreadWasmTier2Generator();
-
     AutoLockHelperThreadState lock;
+    waitForAllThreadsLocked(lock);
+}
+
+void
+GlobalHelperThreadState::waitForAllThreadsLocked(AutoLockHelperThreadState& lock)
+{
+    CancelOffThreadIonCompileLocked(CompilationSelector(AllCompilations()), false, lock);
+    CancelOffThreadWasmTier2GeneratorLocked(lock);
+
     while (hasActiveThreads(lock))
         wait(lock, CONSUMER);
 }

js/src/vm/HelperThreads.h

@@ -425,12 +425,13 @@ class GlobalHelperThreadState
     JSObject* finishModuleParseTask(JSContext* cx, void* token);
 
     bool hasActiveThreads(const AutoLockHelperThreadState&);
-    void waitForAllThreads();
+    void waitForAllThreadsLocked(AutoLockHelperThreadState&);
 
     template <typename T>
     bool checkTaskThreadLimit(size_t maxThreads, bool isMaster = false) const;
 
   private:
+    void waitForAllThreads();
 
     /*
      * Lock protecting all mutable shared state accessed by helper threads, and