Bug 1382251: Part 7 - Add mechanism for automatically brokering DLL functions r=jimm

The FunctionBroker is a special kind of FunctionHook that brokers the hooked function on another process.  In the child process, it uses the FunctionBrokerChild to request that the FunctionBrokerParent run a function and return the response.  It handles most cases of parameter, return value and error marshaling on its own.  It also guarantees that requests are issued from the proper thread.

--HG--
extra : rebase_source : 99e4864f3e6396695eae0ff9aec75aa3c69f082f

dom/plugins/ipc/FunctionBroker.cpp

@@ -0,0 +1,38 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "FunctionBroker.h"
+#include "PluginQuirks.h"
+#include <commdlg.h>
+
+using namespace mozilla;
+using namespace mozilla::ipc;
+using namespace mozilla::plugins;
+
+namespace mozilla {
+namespace plugins {
+
+template <int QuirkFlag>
+static bool CheckQuirks(int aQuirks)
+{
+  return static_cast<bool>(aQuirks & QuirkFlag);
+}
+
+void FreeDestructor(void* aObj) { free(aObj); }
+
+/*****************************************************************************/
+
+#define FUN_HOOK(x) static_cast<FunctionHook*>(x)
+
+void
+AddBrokeredFunctionHooks(FunctionHookArray& aHooks)
+{
+}
+
+#undef FUN_HOOK
+
+} // namespace plugins
+} // namespace mozilla

dom/plugins/ipc/FunctionBrokerParent.cpp

@@ -11,6 +11,13 @@
 namespace mozilla {
 namespace plugins {
 
+#if defined(XP_WIN)
+UlongPairToIdMap sPairToIdMap;
+IdToUlongPairMap sIdToPairMap;
+PtrToIdMap sPtrToIdMap;
+IdToPtrMap sIdToPtrMap;
+#endif // defined(XP_WIN)
+
 /* static */ FunctionBrokerParent*
 FunctionBrokerParent::Create(Endpoint<PFunctionBrokerParent>&& aParentEnd)
 {
@@ -18,6 +25,11 @@ FunctionBrokerParent::Create(Endpoint<PFunctionBrokerParent>&& aParentEnd)
   if (!thread) {
     return nullptr;
   }
+
+  // We get the FunctionHooks so that they are created here, not on the
+  // message thread.
+  FunctionHook::GetHooks();
+
   return new FunctionBrokerParent(thread, Move(aParentEnd));
 }
 
@@ -80,5 +92,39 @@ FunctionBrokerParent::ActorDestroy(ActorDestroyReason aWhy)
   MOZ_RELEASE_ASSERT(mThread->IsOnThread());
 }
 
+mozilla::ipc::IPCResult
+FunctionBrokerParent::RecvBrokerFunction(const FunctionHookId &aFunctionId,
+                                         const IpdlTuple &aInTuple,
+                                         IpdlTuple *aOutTuple)
+{
+#if defined(XP_WIN)
+  MOZ_ASSERT(mThread->IsOnThread());
+  if (RunBrokeredFunction(OtherPid(), aFunctionId, aInTuple, aOutTuple)) {
+    return IPC_OK();
+  }
+  return IPC_FAIL_NO_REASON(this);
+#else
+  MOZ_ASSERT_UNREACHABLE("BrokerFunction is currently only implemented on Windows.");
+  return IPC_FAIL_NO_REASON(this);
+#endif
+}
+
+// static
+bool
+FunctionBrokerParent::RunBrokeredFunction(base::ProcessId aClientId,
+                                        const FunctionHookId &aFunctionId,
+                                        const IPC::IpdlTuple &aInTuple,
+                                        IPC::IpdlTuple *aOutTuple)
+{
+  if ((size_t)aFunctionId >= FunctionHook::GetHooks()->Length()) {
+    MOZ_ASSERT_UNREACHABLE("Invalid function ID");
+    return false;
+  }
+
+  FunctionHook* hook = FunctionHook::GetHooks()->ElementAt(aFunctionId);
+  MOZ_ASSERT(hook->FunctionId() == aFunctionId);
+  return hook->RunOriginalFunction(aClientId, aInTuple, aOutTuple);
+}
+
 } // namespace plugins
 } // namespace mozilla

dom/plugins/ipc/FunctionBrokerParent.h

@@ -26,12 +26,21 @@ public:
 
   void ActorDestroy(ActorDestroyReason aWhy) override;
 
+  mozilla::ipc::IPCResult
+  RecvBrokerFunction(const FunctionHookId &aFunctionId, const IpdlTuple &aInTuple,
+                   IpdlTuple *aOutTuple) override;
+
 private:
   explicit FunctionBrokerParent(FunctionBrokerThread* aThread,
                                 Endpoint<PFunctionBrokerParent>&& aParentEnd);
   void ShutdownOnBrokerThread();
   void Bind(Endpoint<PFunctionBrokerParent>&& aEnd);
 
+  static bool RunBrokeredFunction(base::ProcessId aClientId,
+                                  const FunctionHookId &aFunctionId,
+                                  const IPC::IpdlTuple &aInTuple,
+                                  IPC::IpdlTuple *aOutTuple);
+
   nsAutoPtr<FunctionBrokerThread> mThread;
   Monitor mMonitor;
   bool mShutdownDone;

dom/plugins/ipc/PluginQuirks.h

@@ -7,6 +7,8 @@
 #ifndef dom_plugins_PluginQuirks_h
 #define dom_plugins_PluginQuirks_h
 
+#include "nsString.h"
+
 namespace mozilla {
 namespace plugins {
 

ipc/glue/IPCMessageUtils.h

@@ -310,6 +310,12 @@ struct ParamTraits<int8_t>
   {
     return aMsg->ReadBytesInto(aIter, aResult, sizeof(*aResult));
   }
+
+  static void Log(const paramType& aParam, std::wstring* aLog)
+  {
+    // Use 0xff to avoid sign extension.
+    aLog->append(StringPrintf(L"0x%02x", aParam & 0xff));
+  }
 };
 
 template<>
@@ -326,6 +332,11 @@ struct ParamTraits<uint8_t>
   {
     return aMsg->ReadBytesInto(aIter, aResult, sizeof(*aResult));
   }
+
+  static void Log(const paramType& aParam, std::wstring* aLog)
+  {
+    aLog->append(StringPrintf(L"0x%02x", aParam));
+  }
 };
 
 #if !defined(OS_POSIX)