Bug 1382251: Part 7 - Add mechanism for automatically brokering DLL functions; r=jimm

The FunctionBroker is a special kind of FunctionHook that brokers the hooked function on another process.  In the child process, it uses the FunctionBrokerChild to request that the FunctionBrokerParent run a function and return the response.  It handles most cases of parameter, return value and error marshaling on its own.  It also guarantees that requests are issued from the proper thread.

dom/plugins/ipc/FunctionBroker.cpp

@@ -0,0 +1,38 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "FunctionBroker.h"
+#include "PluginQuirks.h"
+#include <commdlg.h>
+
+using namespace mozilla;
+using namespace mozilla::ipc;
+using namespace mozilla::plugins;
+
+namespace mozilla {
+namespace plugins {
+
+template <int QuirkFlag>
+static bool CheckQuirks(int aQuirks)
+{
+  return static_cast<bool>(aQuirks & QuirkFlag);
+}
+
+void FreeDestructor(void* aObj) { free(aObj); }
+
+/*****************************************************************************/
+
+#define FUN_HOOK(x) static_cast<FunctionHook*>(x)
+
+void
+AddBrokeredFunctionHooks(FunctionHookArray& aHooks)
+{
+}
+
+#undef FUN_HOOK
+
+} // namespace plugins
+} // namespace mozilla

dom/plugins/ipc/FunctionBrokerParent.cpp

@@ -11,6 +11,13 @@
 namespace mozilla {
 namespace plugins {
 
+#if defined(XP_WIN)
+UlongPairToIdMap sPairToIdMap;
+IdToUlongPairMap sIdToPairMap;
+PtrToIdMap sPtrToIdMap;
+IdToPtrMap sIdToPtrMap;
+#endif // defined(XP_WIN)
+
 /* static */ FunctionBrokerParent*
 FunctionBrokerParent::Create(Endpoint<PFunctionBrokerParent>&& aParentEnd)
 {
@@ -18,6 +25,11 @@ FunctionBrokerParent::Create(Endpoint<PFunctionBrokerParent>&& aParentEnd)
   if (!thread) {
     return nullptr;
   }
+
+  // We get the FunctionHooks so that they are created here, not on the
+  // message thread.
+  FunctionHook::GetHooks();
+
   return new FunctionBrokerParent(thread, Move(aParentEnd));
 }
 
@@ -80,5 +92,39 @@ FunctionBrokerParent::ActorDestroy(ActorDestroyReason aWhy)
   MOZ_RELEASE_ASSERT(mThread->IsOnThread());
 }
 
+mozilla::ipc::IPCResult
+FunctionBrokerParent::RecvBrokerFunction(const FunctionHookId &aFunctionId,
+                                         const IpdlTuple &aInTuple,
+                                         IpdlTuple *aOutTuple)
+{
+#if defined(XP_WIN)
+  MOZ_ASSERT(mThread->IsOnThread());
+  if (RunBrokeredFunction(OtherPid(), aFunctionId, aInTuple, aOutTuple)) {
+    return IPC_OK();
+  }
+  return IPC_FAIL_NO_REASON(this);
+#else
+  MOZ_ASSERT_UNREACHABLE("BrokerFunction is currently only implemented on Windows.");
+  return IPC_FAIL_NO_REASON(this);
+#endif
+}
+
+// static
+bool
+FunctionBrokerParent::RunBrokeredFunction(base::ProcessId aClientId,
+                                        const FunctionHookId &aFunctionId,
+                                        const IPC::IpdlTuple &aInTuple,
+                                        IPC::IpdlTuple *aOutTuple)
+{
+  if ((size_t)aFunctionId >= FunctionHook::GetHooks()->Length()) {
+    MOZ_ASSERT_UNREACHABLE("Invalid function ID");
+    return false;
+  }
+
+  FunctionHook* hook = FunctionHook::GetHooks()->ElementAt(aFunctionId);
+  MOZ_ASSERT(hook->FunctionId() == aFunctionId);
+  return hook->RunOriginalFunction(aClientId, aInTuple, aOutTuple);
+}
+
 } // namespace plugins
 } // namespace mozilla

dom/plugins/ipc/FunctionBrokerParent.h

@@ -26,12 +26,21 @@ public:
 
   void ActorDestroy(ActorDestroyReason aWhy) override;
 
+  mozilla::ipc::IPCResult
+  RecvBrokerFunction(const FunctionHookId &aFunctionId, const IpdlTuple &aInTuple,
+                   IpdlTuple *aOutTuple) override;
+
 private:
   explicit FunctionBrokerParent(FunctionBrokerThread* aThread,
                                 Endpoint<PFunctionBrokerParent>&& aParentEnd);
   void ShutdownOnBrokerThread();
   void Bind(Endpoint<PFunctionBrokerParent>&& aEnd);
 
+  static bool RunBrokeredFunction(base::ProcessId aClientId,
+                                  const FunctionHookId &aFunctionId,
+                                  const IPC::IpdlTuple &aInTuple,
+                                  IPC::IpdlTuple *aOutTuple);
+
   nsAutoPtr<FunctionBrokerThread> mThread;
   Monitor mMonitor;
   bool mShutdownDone;

dom/plugins/ipc/PluginQuirks.h

@@ -7,6 +7,8 @@
 #ifndef dom_plugins_PluginQuirks_h
 #define dom_plugins_PluginQuirks_h
 
+#include "nsString.h"
+
 namespace mozilla {
 namespace plugins {
 

ipc/glue/IPCMessageUtils.h

@@ -310,6 +310,12 @@ struct ParamTraits<int8_t>
   {
     return aMsg->ReadBytesInto(aIter, aResult, sizeof(*aResult));
   }
+
+  static void Log(const paramType& aParam, std::wstring* aLog)
+  {
+    // Use 0xff to avoid sign extension.
+    aLog->append(StringPrintf(L"0x%02x", aParam & 0xff));
+  }
 };
 
 template<>
@@ -326,6 +332,11 @@ struct ParamTraits<uint8_t>
   {
     return aMsg->ReadBytesInto(aIter, aResult, sizeof(*aResult));
   }
+
+  static void Log(const paramType& aParam, std::wstring* aLog)
+  {
+    aLog->append(StringPrintf(L"0x%02x", aParam));
+  }
 };
 
 #if !defined(OS_POSIX)