Bug 1247687 - Implement initial Static Module Loading for Workers; r=jonco

This is the first pass of getting static module loading to work. This roughly implements
https://html.spec.whatwg.org/multipage/webappapis.html#fetch-a-worklet/module-worker-script-graph --
without some of the settings objects correctly set.

Th WorkerModuleLoader itself is implementing step 5, with the StartFetch method handling moving the work back to the main thread. The algorithm in step 5 is generic to all modules.

Depends on D147329

Differential Revision: https://phabricator.services.mozilla.com/D147327

dom/workers/ScriptLoader.cpp

@@ -265,6 +265,17 @@ void LoadAllScripts(WorkerPrivate* aWorkerPrivate,
     return;
   }
 
+  if (aIsMainScript) {
+    // Module Load
+    RefPtr<JS::loader::ScriptLoadRequest> mainScript = loader->GetMainScript();
+    if (mainScript && mainScript->IsModuleRequest()) {
+      if (NS_FAILED(mainScript->AsModuleRequest()->StartModuleLoad())) {
+        return;
+      }
+      syncLoop.Run();
+      return;
+    }
+  }
   if (loader->DispatchLoadScripts()) {
     syncLoop.Run();
   }
@@ -371,6 +382,10 @@ class ScriptExecutorRunnable final : public MainThreadWorkerSyncRunnable {
 
   virtual bool PreRun(WorkerPrivate* aWorkerPrivate) override;
 
+  bool ProcessModuleScript(JSContext* aCx, WorkerPrivate* aWorkerPrivate);
+
+  bool ProcessClassicScripts(JSContext* aCx, WorkerPrivate* aWorkerPrivate);
+
   virtual bool WorkerRun(JSContext* aCx,
                          WorkerPrivate* aWorkerPrivate) override;
 
@@ -424,6 +439,15 @@ WorkerScriptLoader::WorkerScriptLoader(
   }
 }
 
+ScriptLoadRequest* WorkerScriptLoader::GetMainScript() {
+  mWorkerRef->Private()->AssertIsOnWorkerThread();
+  ScriptLoadRequest* request = mLoadingRequests.getFirst();
+  if (request->GetWorkerLoadContext()->IsTopLevel()) {
+    return request;
+  }
+  return nullptr;
+}
+
 void WorkerScriptLoader::InitModuleLoader() {
   mWorkerRef->Private()->AssertIsOnWorkerThread();
   RefPtr<WorkerModuleLoader> moduleLoader =
@@ -439,6 +463,7 @@ void WorkerScriptLoader::InitModuleLoader() {
 bool WorkerScriptLoader::CreateScriptRequests(
     const nsTArray<nsString>& aScriptURLs,
     const mozilla::Encoding* aDocumentEncoding, bool aIsMainScript) {
+  mWorkerRef->Private()->AssertIsOnWorkerThread();
   // If a worker has been loaded as a module worker, ImportScripts calls are
   // disallowed -- then the operation is invalid.
   //
@@ -478,6 +503,7 @@ nsTArray<RefPtr<ThreadSafeRequestHandle>> WorkerScriptLoader::GetLoadingList() {
 already_AddRefed<ScriptLoadRequest> WorkerScriptLoader::CreateScriptLoadRequest(
     const nsString& aScriptURL, const mozilla::Encoding* aDocumentEncoding,
     bool aIsMainScript) {
+  mWorkerRef->Private()->AssertIsOnWorkerThread();
   WorkerLoadContext::Kind kind =
       WorkerLoadContext::GetKind(aIsMainScript, IsDebuggerScript());
 
@@ -503,13 +529,34 @@ already_AddRefed<ScriptLoadRequest> WorkerScriptLoader::CreateScriptLoadRequest(
     loadContext->mLoadResult = rv;
   }
 
+  RefPtr<ScriptLoadRequest> request;
+  if (mWorkerRef->Private()->WorkerType() == WorkerType::Classic) {
     RefPtr<ScriptFetchOptions> fetchOptions =
         new ScriptFetchOptions(CORSMode::CORS_NONE, referrerPolicy, nullptr);
 
-  RefPtr<ScriptLoadRequest> request =
-      new ScriptLoadRequest(ScriptKind::eClassic, uri, fetchOptions,
+    request = new ScriptLoadRequest(ScriptKind::eClassic, uri, fetchOptions,
                                     SRIMetadata(), nullptr, /* = aReferrer */
                                     loadContext);
+  } else {
+    // Implements part of "To fetch a worklet/module worker script graph"
+    // including, setting up the request with a credentials mode (TODO),
+    // destination (CSP, TODO).
+
+    // Step 1. Let options be a script fetch options. TODO: credentials mode.
+    RefPtr<ScriptFetchOptions> fetchOptions =
+        new ScriptFetchOptions(CORSMode::CORS_NONE, referrerPolicy, nullptr);
+
+    // Part of Step 2. This sets the Top-level flag to true
+    RefPtr<WorkerModuleLoader::ModuleLoaderBase> moduleLoader =
+        GetGlobal()->GetModuleLoader(nullptr);
+
+    request = new ModuleLoadRequest(
+        uri, fetchOptions, SRIMetadata(), nullptr, loadContext,
+        true,  /* is top level */
+        false, /* is dynamic import */
+        moduleLoader, ModuleLoadRequest::NewVisitedSetForTopLevelImport(uri),
+        nullptr);
+  }
 
   // Set the mURL, it will be used for error handling and debugging.
   request->mURL = NS_ConvertUTF16toUTF8(aScriptURL);
@@ -601,7 +648,11 @@ nsIGlobalObject* WorkerScriptLoader::GetGlobal() {
 
 void WorkerScriptLoader::MaybeMoveToLoadedList(ScriptLoadRequest* aRequest) {
   mWorkerRef->Private()->AssertIsOnWorkerThread();
+  // Only set to ready for regular scripts. Module loader will set the script to
+  // ready if it is a Module Request.
+  if (!aRequest->IsModuleRequest()) {
     aRequest->SetReady();
+  }
 
   // If the request is not in a list, we are in an illegal state.
   MOZ_RELEASE_ASSERT(aRequest->isInList());
@@ -900,6 +951,25 @@ bool WorkerScriptLoader::EvaluateScript(JSContext* aCx,
     mWorkerRef->Private()->ExecutionReady();
   }
 
+  if (aRequest->IsModuleRequest()) {
+    // Only the top level module of the module graph will be executed from here,
+    // the rest will be executed from SpiderMonkey as part of the execution of
+    // the module graph.
+    MOZ_ASSERT(aRequest->IsTopLevel());
+    ModuleLoadRequest* request = aRequest->AsModuleRequest();
+    if (!request->mModuleScript) {
+      return false;
+    }
+    // Implements To fetch a worklet/module worker script graph
+    // Step 5. Fetch the descendants of and link result.
+    if (!request->InstantiateModuleGraph()) {
+      return false;
+    }
+
+    nsresult rv = request->EvaluateModule();
+    return NS_SUCCEEDED(rv);
+  }
+
   JS::CompileOptions options(aCx);
   // The introduction script is used by the DOM script loader as a way
   // to fill the Debugger Metadata for the JS Execution context. We don't use
@@ -1285,10 +1355,48 @@ bool ScriptExecutorRunnable::PreRun(WorkerPrivate* aWorkerPrivate) {
   return mScriptLoader->StoreCSP();
 }
 
-bool ScriptExecutorRunnable::WorkerRun(JSContext* aCx,
-                                       WorkerPrivate* aWorkerPrivate) {
-  aWorkerPrivate->AssertIsOnWorkerThread();
+bool ScriptExecutorRunnable::ProcessModuleScript(
+    JSContext* aCx, WorkerPrivate* aWorkerPrivate) {
+  // We should only ever have one script when processing modules
+  MOZ_ASSERT(mLoadedRequests.Length() == 1);
+  RefPtr<ScriptLoadRequest> request;
+  {
+    // There is a possibility that we cleaned up while this task was waiting to
+    // run. If this has happened, return and exit.
+    MutexAutoLock lock(mScriptLoader->CleanUpLock());
+    if (mScriptLoader->CleanedUp()) {
+      return true;
+    }
 
+    const auto& requestHandle = mLoadedRequests.begin()->get();
+    // The request must be valid.
+    MOZ_ASSERT(!requestHandle->IsEmpty());
+
+    // Release the request to the worker. From this point on, the Request Handle
+    // is empty.
+    request = requestHandle->ReleaseRequest();
+
+    // release lock. We will need it later if we cleanup.
+  }
+
+  MOZ_ASSERT(request->IsModuleRequest());
+
+  WorkerLoadContext* loadContext = request->GetWorkerLoadContext();
+  ModuleLoadRequest* moduleRequest = request->AsModuleRequest();
+  if (NS_FAILED(loadContext->mLoadResult)) {
+    if (!moduleRequest->IsTopLevel()) {
+      moduleRequest->Cancel();
+    } else {
+      moduleRequest->LoadFailed();
+    }
+  }
+
+  moduleRequest->OnFetchComplete(loadContext->mLoadResult);
+  return true;
+}
+
+bool ScriptExecutorRunnable::ProcessClassicScripts(
+    JSContext* aCx, WorkerPrivate* aWorkerPrivate) {
   // There is a possibility that we cleaned up while this task was waiting to
   // run. If this has happened, return and exit.
   {
@@ -1297,11 +1405,6 @@ bool ScriptExecutorRunnable::WorkerRun(JSContext* aCx,
       return true;
     }
 
-    // We must be on the same worker as we started on.
-    MOZ_ASSERT(
-        mScriptLoader->mSyncLoopTarget == mSyncLoopTarget,
-        "Unexpected SyncLoopTarget. Check if the sync loop was closed early");
-
     for (const auto& requestHandle : mLoadedRequests) {
       // The request must be valid.
       MOZ_ASSERT(!requestHandle->IsEmpty());
@@ -1309,13 +1412,28 @@ bool ScriptExecutorRunnable::WorkerRun(JSContext* aCx,
       // Release the request to the worker. From this point on, the Request
       // Handle is empty.
       RefPtr<ScriptLoadRequest> request = requestHandle->ReleaseRequest();
-
       mScriptLoader->MaybeMoveToLoadedList(request);
     }
   }
   return mScriptLoader->ProcessPendingRequests(aCx);
 }
 
+bool ScriptExecutorRunnable::WorkerRun(JSContext* aCx,
+                                       WorkerPrivate* aWorkerPrivate) {
+  aWorkerPrivate->AssertIsOnWorkerThread();
+
+  // We must be on the same worker as we started on.
+  MOZ_ASSERT(
+      mScriptLoader->mSyncLoopTarget == mSyncLoopTarget,
+      "Unexpected SyncLoopTarget. Check if the sync loop was closed early");
+
+  if (aWorkerPrivate->WorkerType() == WorkerType::Module) {
+    return ProcessModuleScript(aCx, aWorkerPrivate);
+  }
+
+  return ProcessClassicScripts(aCx, aWorkerPrivate);
+}
+
 nsresult ScriptExecutorRunnable::Cancel() {
   // We need to check first if cancel is called twice
   nsresult rv = MainThreadWorkerSyncRunnable::Cancel();

dom/workers/ScriptLoader.h

@@ -8,6 +8,7 @@
 #define mozilla_dom_workers_scriptloader_h__
 
 #include "js/loader/ScriptLoadRequest.h"
+#include "js/loader/ModuleLoadRequest.h"
 #include "js/loader/ModuleLoaderBase.h"
 #include "mozilla/dom/WorkerCommon.h"
 #include "mozilla/dom/WorkerLoadContext.h"
@@ -142,6 +143,7 @@ class WorkerScriptLoader : public JS::loader::ScriptLoaderInterface,
   friend class CacheLoadHandler;
   friend class CacheCreator;
   friend class NetworkLoadHandler;
+  friend class WorkerModuleLoader;
 
   RefPtr<ThreadSafeWorkerRef> mWorkerRef;
   UniquePtr<SerializedStackHolder> mOriginStack;
@@ -189,6 +191,8 @@ class WorkerScriptLoader : public JS::loader::ScriptLoaderInterface,
                             const mozilla::Encoding* aDocumentEncoding,
                             bool aIsMainScript);
 
+  ScriptLoadRequest* GetMainScript();
+
   already_AddRefed<ScriptLoadRequest> CreateScriptLoadRequest(
       const nsString& aScriptURL, const mozilla::Encoding* aDocumentEncoding,
       bool aIsMainScript);
@@ -202,6 +206,8 @@ class WorkerScriptLoader : public JS::loader::ScriptLoaderInterface,
 
   nsIURI* GetInitialBaseURI();
 
+  nsIGlobalObject* GetGlobal();
+
   void MaybeMoveToLoadedList(ScriptLoadRequest* aRequest);
 
   bool StoreCSP();
@@ -239,8 +245,6 @@ class WorkerScriptLoader : public JS::loader::ScriptLoaderInterface,
 
   nsTArray<RefPtr<ThreadSafeRequestHandle>> GetLoadingList();
 
-  nsIGlobalObject* GetGlobal();
-
   bool EvaluateScript(JSContext* aCx, ScriptLoadRequest* aRequest);
 
   nsresult FillCompileOptionsForRequest(

dom/workers/WorkerScope.h

@@ -165,7 +165,8 @@ class WorkerGlobalScopeBase : public DOMEventTargetHelper,
 
   // The nullptr here is not used, but is required to make the override method
   // have the same signature as other GetModuleLoader methods on globals.
-  JS::loader::ModuleLoaderBase* GetModuleLoader(JSContext* aCx) override {
+  JS::loader::ModuleLoaderBase* GetModuleLoader(
+      JSContext* aCx = nullptr) override {
     return mModuleLoader;
   };
 

dom/workers/loader/WorkerLoadContext.h

@@ -85,6 +85,10 @@ class WorkerLoadContext : public JS::loader::LoadContextBase {
     // We are importing a script from the worker via ImportScript. This may only
     // be a Classic script.
     ImportScript,
+    // We are importing a script from the worker via a Static Import. This may
+    // only
+    // be a Module script.
+    StaticImport,
     // We have an attached debugger, and these should be treated specially and
     // not like a main script (regardless of their type). This is not part of
     // the specification.

dom/workers/loader/WorkerModuleLoader.cpp

@@ -4,6 +4,9 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+#include "js/experimental/JSStencil.h"  // JS::Stencil, JS::CompileModuleScriptToStencil, JS::InstantiateModuleStencil
+#include "js/loader/ModuleLoadRequest.h"
+#include "mozilla/dom/WorkerLoadContext.h"
 #include "mozilla/dom/workerinternals/ScriptLoader.h"
 #include "WorkerModuleLoader.h"
 
@@ -31,7 +34,18 @@ WorkerModuleLoader::WorkerModuleLoader(WorkerScriptLoader* aScriptLoader,
 
 already_AddRefed<ModuleLoadRequest> WorkerModuleLoader::CreateStaticImport(
     nsIURI* aURI, ModuleLoadRequest* aParent) {
-  MOZ_CRASH("Not implemented yet");
+  Maybe<ClientInfo> clientInfo = GetGlobalObject()->GetClientInfo();
+
+  RefPtr<WorkerLoadContext> loadContext =
+      new WorkerLoadContext(WorkerLoadContext::Kind::StaticImport, clientInfo);
+  RefPtr<ModuleLoadRequest> request = new ModuleLoadRequest(
+      aURI, aParent->mFetchOptions, SRIMetadata(), aParent->mURI, loadContext,
+      false, /* is top level */
+      false, /* is dynamic import */
+      this, aParent->mVisitedSet, aParent->GetRootModule());
+
+  request->mURL = request->mURI->GetSpecOrDefault();
+  return request.forget();
 }
 
 already_AddRefed<ModuleLoadRequest> WorkerModuleLoader::CreateDynamicImport(
@@ -48,19 +62,53 @@ bool WorkerModuleLoader::CanStartLoad(ModuleLoadRequest* aRequest,
 }
 
 nsresult WorkerModuleLoader::StartFetch(ModuleLoadRequest* aRequest) {
+  if (!GetScriptLoader()->DispatchLoadScript(aRequest)) {
     return NS_ERROR_FAILURE;
   }
+  return NS_OK;
+}
 
 nsresult WorkerModuleLoader::CompileFetchedModule(
     JSContext* aCx, JS::Handle<JSObject*> aGlobal, JS::CompileOptions& aOptions,
     ModuleLoadRequest* aRequest, JS::MutableHandle<JSObject*> aModuleScript) {
+  RefPtr<JS::Stencil> stencil;
+  MOZ_ASSERT(aRequest->IsTextSource());
+  MaybeSourceText maybeSource;
+  nsresult rv = aRequest->GetScriptSource(aCx, &maybeSource);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  auto compile = [&](auto& source) {
+    return JS::CompileModuleScriptToStencil(aCx, aOptions, source);
+  };
+  stencil = maybeSource.mapNonEmpty(compile);
+
+  if (!stencil) {
     return NS_ERROR_FAILURE;
   }
 
+  JS::InstantiateOptions instantiateOptions(aOptions);
+  aModuleScript.set(
+      JS::InstantiateModuleStencil(aCx, instantiateOptions, stencil));
+  if (!aModuleScript) {
+    return NS_ERROR_FAILURE;
+  }
+
+  return NS_OK;
+}
+
 WorkerScriptLoader* WorkerModuleLoader::GetScriptLoader() {
   return static_cast<WorkerScriptLoader*>(mLoader.get());
 }
 
-void WorkerModuleLoader::OnModuleLoadComplete(ModuleLoadRequest* aRequest) {}
+void WorkerModuleLoader::OnModuleLoadComplete(ModuleLoadRequest* aRequest) {
+  if (aRequest->IsTopLevel()) {
+    AutoJSAPI jsapi;
+    if (NS_WARN_IF(!jsapi.Init(GetGlobalObject()))) {
+      return;
+    }
+    GetScriptLoader()->MaybeMoveToLoadedList(aRequest);
+    GetScriptLoader()->ProcessPendingRequests(jsapi.cx());
+  }
+}
 
 }  // namespace mozilla::dom::workerinternals::loader

dom/workers/loader/WorkerModuleLoader.h

@@ -19,6 +19,16 @@ using ScriptLoadRequest = JS::loader::ScriptLoadRequest;
 using ScriptLoadRequestList = JS::loader::ScriptLoadRequestList;
 using ModuleLoadRequest = JS::loader::ModuleLoadRequest;
 
+// WorkerModuleLoader
+//
+// The WorkerModuleLoader provides the methods that implement specification
+// step 5 from "To fetch a worklet/module worker script graph", specifically for
+// workers. In addition, this implements worker specific initialization for
+// Static imports and Dynamic imports.
+//
+// The steps are outlined in "To fetch the descendants of and link a module
+// script" and are common for all Modules. Thus we delegate to ModuleLoaderBase
+// for those steps.
 class WorkerModuleLoader : public JS::loader::ModuleLoaderBase {
  public:
   NS_DECL_ISUPPORTS_INHERITED
@@ -45,6 +55,8 @@ class WorkerModuleLoader : public JS::loader::ModuleLoaderBase {
 
   bool CanStartLoad(ModuleLoadRequest* aRequest, nsresult* aRvOut) override;
 
+  // StartFetch is special for worker modules, as we need to move back to the
+  // main thread to start a new load.
   nsresult StartFetch(ModuleLoadRequest* aRequest) override;
 
   nsresult CompileFetchedModule(