Bug 1837079 - [7/10] Open Windows file picker out-of-process r=gstoll,handyman,ipc-reviewers,nika,win-reviewers,mhowell

When opening a new Windows file dialog, open it out-of-process if
possible. Fall back to opening it in-process if that fails. (This
behavior is configurable with a pref.)

Differential Revision: https://phabricator.services.mozilla.com/D180343

ipc/glue/UtilityProcessChild.cpp

@@ -6,6 +6,7 @@
 #include "UtilityProcessChild.h"
 
 #include "mozilla/AppShutdown.h"
+#include "mozilla/Logging.h"
 #include "mozilla/dom/ContentParent.h"
 #include "mozilla/dom/JSOracleChild.h"
 #include "mozilla/dom/MemoryReportRequest.h"

modules/libpref/init/StaticPrefList.yaml

@@ -15555,6 +15555,19 @@
   value: true
   mirror: always
 
+# Whether to open the Windows file and folder pickers "remotely" (in a utility
+# process) or "locally" (in the main process).
+#
+# Valid values:
+#  *  0: auto (possibly release-channel-dependent)
+#  *  1: remotely, falling back to locally
+#  *  2: remotely, no fallback
+#  * -1: locally, no fallback
+- name: widget.windows.utility_process_file_picker
+  type: RelaxedAtomicInt32
+  value: -1
+  mirror: always
+
 # The number of messages of each type to keep for display in
 # about:windows-messages
 - name: widget.windows.messages_to_log

widget/windows/nsFilePicker.cpp

@@ -9,23 +9,29 @@
 #include <shlobj.h>
 #include <shlwapi.h>
 #include <cderr.h>
+#include <winerror.h>
+#include <utility>
 
 #include "mozilla/Assertions.h"
 #include "mozilla/BackgroundHangMonitor.h"
+#include "mozilla/Logging.h"
+#include "mozilla/ipc/UtilityProcessManager.h"
 #include "mozilla/ProfilerLabels.h"
+#include "mozilla/StaticPrefs_widget.h"
 #include "mozilla/UniquePtr.h"
 #include "mozilla/WindowsVersion.h"
-#include "nsReadableUtils.h"
-#include "nsNetUtil.h"
-#include "nsWindow.h"
-#include "nsEnumeratorUtils.h"
 #include "nsCRT.h"
+#include "nsEnumeratorUtils.h"
+#include "nsNetUtil.h"
+#include "nsPIDOMWindow.h"
+#include "nsReadableUtils.h"
 #include "nsString.h"
 #include "nsToolkit.h"
+#include "nsWindow.h"
 #include "WinUtils.h"
-#include "nsPIDOMWindow.h"
 
 #include "mozilla/widget/filedialog/WinFileDialogCommands.h"
+#include "mozilla/widget/filedialog/WinFileDialogParent.h"
 
 using mozilla::Maybe;
 using mozilla::Result;
@@ -36,6 +42,9 @@ using namespace mozilla::widget;
 UniquePtr<char16_t[], nsFilePicker::FreeDeleter>
     nsFilePicker::sLastUsedUnicodeDirectory;
 
+using mozilla::LogLevel;
+static mozilla::LazyLogModule sLogFileDialog("FileDialog");
+
 #define MAX_EXTENSION_LENGTH 10
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -80,6 +89,165 @@ NS_IMETHODIMP nsFilePicker::Init(mozIDOMWindowProxy* aParent,
   return nsBaseFilePicker::Init(aParent, aTitle, aMode);
 }
 
+namespace mozilla::detail {
+namespace {
+// Commit crimes against asynchrony.
+//
+// More specifically, drive a MozPromise to completion (resolution or rejection)
+// on the main thread. This is only even vaguely acceptable in the context of
+// the Windows file picker because a) this is essentially what `IFileDialog`'s
+// `ShowModal()` will do if we open it in-process anyway, and b) there exist
+// concrete plans [0] to remove this, making the Windows file picker fully
+// asynchronous.
+//
+// Do not take this as a model for use in other contexts; SpinEventLoopUntil
+// alone is bad enough.
+//
+// [0] Although see, _e.g._, http://www.thecodelesscode.com/case/234.
+//
+template <typename T, typename E, bool B>
+static auto ImmorallyDrivePromiseToCompletion(
+    RefPtr<MozPromise<T, E, B>>&& promise) -> Result<T, E> {
+  Maybe<Result<T, E>> val = Nothing();
+
+  AssertIsOnMainThread();
+  promise->Then(
+      mozilla::GetMainThreadSerialEventTarget(), "DrivePromiseToCompletion",
+      [&](T ret) { val = Some(std::move(ret)); },
+      [&](E error) { val = Some(Err(error)); });
+
+  SpinEventLoopUntil("DrivePromiseToCompletion"_ns,
+                     [&]() -> bool { return val.isSome(); });
+
+  MOZ_RELEASE_ASSERT(val.isSome());
+  return val.extract();
+}
+
+// Boilerplate for remotely showing a file dialog.
+template <typename ActionType,
+          typename ReturnType = typename decltype(std::declval<ActionType>()(
+              nullptr))::element_type::ResolveValueType>
+static auto ShowRemote(ActionType&& action)
+    -> RefPtr<MozPromise<ReturnType, HRESULT, false>> {
+  using RetPromise = MozPromise<ReturnType, HRESULT, false>;
+
+  constexpr static const auto fail = []() {
+    return RetPromise::CreateAndReject(E_FAIL, __PRETTY_FUNCTION__);
+  };
+
+  auto mgr = mozilla::ipc::UtilityProcessManager::GetSingleton();
+  if (!mgr) {
+    MOZ_ASSERT(false);
+    return fail();
+  }
+
+  auto wfda = mgr->CreateWinFileDialogAsync();
+  if (!wfda) {
+    return fail();
+  }
+
+  return wfda->Then(
+      mozilla::GetMainThreadSerialEventTarget(),
+      "nsFilePicker ShowRemote acquire",
+      [action = std::forward<ActionType>(action)](
+          filedialog::ProcessProxy p) -> RefPtr<RetPromise> {
+        MOZ_LOG(sLogFileDialog, LogLevel::Info,
+                ("nsFilePicker ShowRemote first callback: p = [%p]", p.get()));
+        // false positive: not actually redundant
+        // NOLINTNEXTLINE(readability-redundant-smartptr-get)
+        return action(p.get())->Then(
+            mozilla::GetMainThreadSerialEventTarget(),
+            "nsFilePicker ShowRemote call",
+            [p](ReturnType ret) {
+              return RetPromise::CreateAndResolve(std::move(ret),
+                                                  __PRETTY_FUNCTION__);
+            },
+            [](mozilla::ipc::ResponseRejectReason error) {
+              MOZ_LOG(sLogFileDialog, LogLevel::Error,
+                      ("IPC call rejected: %zu", size_t(error)));
+              return fail();
+            });
+      },
+      [](nsresult error) -> RefPtr<RetPromise> {
+        MOZ_LOG(sLogFileDialog, LogLevel::Error,
+                ("could not acquire WinFileDialog: %zu", size_t(error)));
+        return fail();
+      });
+}
+}  // namespace
+}  // namespace mozilla::detail
+
+Result<Maybe<filedialog::Results>, HRESULT> nsFilePicker::ShowFilePickerImpl(
+    HWND parent, filedialog::FileDialogType type,
+    nsTArray<filedialog::Command> const& commands) {
+  int32_t const pref =
+      mozilla::StaticPrefs::widget_windows_utility_process_file_picker();
+
+  switch (pref) {
+#ifndef NIGHTLY_BUILD
+    default:  // remain local-only on release and beta, for now
+#endif
+    case -1:
+      return ShowFilePickerLocal(parent, type, commands);
+    case 2:
+      return ShowFilePickerRemote(parent, type, commands);
+#ifdef NIGHTLY_BUILD
+    default:  // fall back to local on failure on Nightly builds
+#endif
+    case 1:
+      return ShowFilePickerRemote(parent, type, commands)
+          .orElse([&](HRESULT err) {
+            return ShowFilePickerLocal(parent, type, commands);
+          });
+  }
+}
+
+Result<Maybe<nsString>, HRESULT> nsFilePicker::ShowFolderPickerImpl(
+    HWND parent, nsTArray<filedialog::Command> const& commands) {
+  int32_t const pref =
+      mozilla::StaticPrefs::widget_windows_utility_process_file_picker();
+
+  switch (pref) {
+    case -1:
+      return ShowFolderPickerLocal(parent, commands);
+    case 2:
+      return ShowFolderPickerRemote(parent, commands);
+    default:
+    case 1:
+    case 0:
+      return ShowFolderPickerRemote(parent, commands).orElse([&](HRESULT err) {
+        return ShowFolderPickerLocal(parent, commands);
+      });
+  }
+}
+
+/* static */
+Result<Maybe<filedialog::Results>, HRESULT> nsFilePicker::ShowFilePickerRemote(
+    HWND parent, filedialog::FileDialogType type,
+    nsTArray<filedialog::Command> const& commands) {
+  auto promise =
+      mozilla::detail::ShowRemote([parent, type, commands = commands.Clone()](
+                                      filedialog::WinFileDialogParent* p) {
+        MOZ_LOG(sLogFileDialog, LogLevel::Info,
+                ("%s: p = [%p]", __PRETTY_FUNCTION__, p));
+        return p->SendShowFileDialog((uintptr_t)parent, type, commands);
+      });
+
+  return mozilla::detail::ImmorallyDrivePromiseToCompletion(std::move(promise));
+}
+
+/* static */
+Result<Maybe<nsString>, HRESULT> nsFilePicker::ShowFolderPickerRemote(
+    HWND parent, nsTArray<filedialog::Command> const& commands) {
+  auto promise =
+      mozilla::detail::ShowRemote([parent, commands = commands.Clone()](
+                                      filedialog::WinFileDialogParent* p) {
+        return p->SendShowFolderDialog((uintptr_t)parent, commands);
+      });
+
+  return mozilla::detail::ImmorallyDrivePromiseToCompletion(std::move(promise));
+}
+
 /* static */
 Result<Maybe<filedialog::Results>, HRESULT> nsFilePicker::ShowFilePickerLocal(
     HWND parent, filedialog::FileDialogType type,
@@ -168,7 +336,7 @@ bool nsFilePicker::ShowFolderPicker(const nsString& aInitialDir) {
     AutoWidgetPickerState awps(mParentWidget);
 
     mozilla::BackgroundHangMonitor().NotifyWait();
-    auto res = ShowFolderPickerLocal(shim.get(), commands);
+    auto res = ShowFolderPickerImpl(shim.get(), commands);
     if (res.isErr()) {
       NS_WARNING("ShowFolderPickerImpl failed");
       return false;
@@ -286,7 +454,7 @@ bool nsFilePicker::ShowFilePicker(const nsString& aInitialDir) {
     AutoWidgetPickerState awps(mParentWidget);
 
     mozilla::BackgroundHangMonitor().NotifyWait();
-    auto res = ShowFilePickerLocal(
+    auto res = ShowFilePickerImpl(
         shim.get(),
         mMode == modeSave ? FileDialogType::Save : FileDialogType::Open,
         commands);

widget/windows/nsFilePicker.h

@@ -83,6 +83,20 @@ class nsFilePicker : public nsBaseWinFilePicker {
   bool ShowFilePicker(const nsString& aInitialDir);
 
  private:
+  // Show the dialog (by default, remotely falling back to locally, or whatever
+  // is specified by the current config).
+  static Result<Maybe<Results>> ShowFilePickerImpl(
+      HWND aParent, FileDialogType type, nsTArray<Command> const& commands);
+  static Result<Maybe<nsString>> ShowFolderPickerImpl(
+      HWND aParent, nsTArray<Command> const& commands);
+
+  // Show the dialog out-of-process.
+  static Result<Maybe<Results>> ShowFilePickerRemote(
+      HWND aParent, FileDialogType type, nsTArray<Command> const& commands);
+  static Result<Maybe<nsString>> ShowFolderPickerRemote(
+      HWND aParent, nsTArray<Command> const& commands);
+
+  // Show the dialog in-process.
   static Result<Maybe<Results>> ShowFilePickerLocal(
       HWND aParent, FileDialogType type, nsTArray<Command> const& commands);
   static Result<Maybe<nsString>> ShowFolderPickerLocal(