Bug 1429643: Limit Win32 NPAPI SSL brokering to 64-bit builds; r=jimm

Uses PluginQuirks to avoid brokering in 32-bit, as Flash has its own 32-bit sandbox.

dom/plugins/ipc/FunctionBroker.cpp

@@ -134,11 +134,19 @@ struct GetOpenFileNameWFB::Response::Info::ShouldMarshal<0> { static const bool
 typedef FunctionBroker<ID_InternetOpenA,
                        decltype(InternetOpenA)> InternetOpenAFB;
 
+template<>
+ShouldHookFunc* const
+InternetOpenAFB::BaseType::mShouldHook = &CheckQuirks<QUIRK_FLASH_HOOK_SSL>;
+
 /* InternetConnectA */
 
 typedef FunctionBroker<ID_InternetConnectA,
                              decltype(InternetConnectA)> InternetConnectAFB;
 
+template<>
+ShouldHookFunc* const
+InternetConnectAFB::BaseType::mShouldHook = &CheckQuirks<QUIRK_FLASH_HOOK_SSL>;
+
 typedef InternetConnectAFB::Request ICAReqHandler;
 
 template<>
@@ -156,6 +164,10 @@ bool ICAReqHandler::ShouldBroker(Endpoint endpoint, const HINTERNET& h,
 typedef FunctionBroker<ID_InternetCloseHandle,
                        decltype(InternetCloseHandle)> InternetCloseHandleFB;
 
+template<>
+ShouldHookFunc* const
+InternetCloseHandleFB::BaseType::mShouldHook = &CheckQuirks<QUIRK_FLASH_HOOK_SSL>;
+
 typedef InternetCloseHandleFB::Request ICHReqHandler;
 
 template<>
@@ -172,6 +184,10 @@ bool ICHReqHandler::ShouldBroker(Endpoint endpoint, const HINTERNET& h)
 typedef FunctionBroker<ID_InternetQueryDataAvailable,
                        decltype(InternetQueryDataAvailable)> InternetQueryDataAvailableFB;
 
+template<>
+ShouldHookFunc* const
+InternetQueryDataAvailableFB::BaseType::mShouldHook = &CheckQuirks<QUIRK_FLASH_HOOK_SSL>;
+
 typedef InternetQueryDataAvailableFB::Request IQDAReq;
 
 typedef struct RequestHandler<ID_InternetQueryDataAvailable,
@@ -220,6 +236,10 @@ struct InternetQueryDataAvailableFB::Response::Info::ShouldMarshal<1> { static c
 typedef FunctionBroker<ID_InternetReadFile,
                        decltype(InternetReadFile)> InternetReadFileFB;
 
+template<>
+ShouldHookFunc* const
+InternetReadFileFB::BaseType::mShouldHook = &CheckQuirks<QUIRK_FLASH_HOOK_SSL>;
+
 typedef InternetReadFileFB::Request IRFRequestHandler;
 typedef struct RequestHandler<ID_InternetReadFile,
                               BOOL HOOK_CALL (HINTERNET, DWORD)> IRFDelegateReq;
@@ -304,6 +324,10 @@ bool IRFResponseHandler::Unmarshal(const IpdlTuple& aTuple, BOOL& ret, HINTERNET
 typedef FunctionBroker<ID_InternetWriteFile,
                        decltype(InternetWriteFile)> InternetWriteFileFB;
 
+template<>
+ShouldHookFunc* const
+InternetWriteFileFB::BaseType::mShouldHook = &CheckQuirks<QUIRK_FLASH_HOOK_SSL>;
+
 typedef InternetWriteFileFB::Request IWFReqHandler;
 typedef RequestHandler<ID_InternetWriteFile,
                        int HOOK_CALL (HINTERNET, nsDependentCSubstring)> IWFDelegateReqHandler;
@@ -351,6 +375,10 @@ struct InternetWriteFileFB::Response::Info::ShouldMarshal<3> { static const bool
 typedef FunctionBroker<ID_InternetSetOptionA,
                        decltype(InternetSetOptionA)> InternetSetOptionAFB;
 
+template<>
+ShouldHookFunc* const
+InternetSetOptionAFB::BaseType::mShouldHook = &CheckQuirks<QUIRK_FLASH_HOOK_SSL>;
+
 typedef InternetSetOptionAFB::Request ISOAReqHandler;
 typedef RequestHandler<ID_InternetSetOptionA,
                        BOOL HOOK_CALL (HINTERNET, DWORD, nsDependentCSubstring)> ISOADelegateReqHandler;
@@ -392,6 +420,10 @@ bool ISOAReqHandler::ShouldBroker(Endpoint endpoint, const HINTERNET& h, const D
 typedef FunctionBroker<ID_HttpAddRequestHeadersA,
                        decltype(HttpAddRequestHeadersA)> HttpAddRequestHeadersAFB;
 
+template<>
+ShouldHookFunc* const
+HttpAddRequestHeadersAFB::BaseType::mShouldHook = &CheckQuirks<QUIRK_FLASH_HOOK_SSL>;
+
 typedef HttpAddRequestHeadersAFB::Request HARHAReqHandler;
 
 template<>
@@ -410,6 +442,10 @@ bool HARHAReqHandler::ShouldBroker(Endpoint endpoint, const HINTERNET& h,
 typedef FunctionBroker<ID_HttpOpenRequestA,
                        decltype(HttpOpenRequestA)> HttpOpenRequestAFB;
 
+template<>
+ShouldHookFunc* const
+HttpOpenRequestAFB::BaseType::mShouldHook = &CheckQuirks<QUIRK_FLASH_HOOK_SSL>;
+
 typedef HttpOpenRequestAFB::Request HORAReqHandler;
 typedef RequestHandler<ID_HttpOpenRequestA,
                        HINTERNET HOOK_CALL (HINTERNET, LPCSTR, LPCSTR, LPCSTR, LPCSTR,
@@ -474,6 +510,10 @@ bool HORAReqHandler::ShouldBroker(Endpoint endpoint, const HINTERNET& h,
 typedef FunctionBroker<ID_HttpQueryInfoA,
                        decltype(HttpQueryInfoA)> HttpQueryInfoAFB;
 
+template<>
+ShouldHookFunc* const
+HttpQueryInfoAFB::BaseType::mShouldHook = &CheckQuirks<QUIRK_FLASH_HOOK_SSL>;
+
 typedef HttpQueryInfoAFB::Request HQIARequestHandler;
 typedef RequestHandler<ID_HttpQueryInfoA,
                        BOOL HOOK_CALL (HINTERNET, DWORD, BOOL, DWORD, BOOL,
@@ -594,6 +634,10 @@ bool HQIAResponseHandler::Unmarshal(const IpdlTuple& aTuple, BOOL& ret, HINTERNE
 typedef FunctionBroker<ID_HttpSendRequestA,
                        decltype(HttpSendRequestA)> HttpSendRequestAFB;
 
+template<>
+ShouldHookFunc* const
+HttpSendRequestAFB::BaseType::mShouldHook = &CheckQuirks<QUIRK_FLASH_HOOK_SSL>;
+
 typedef HttpSendRequestAFB::Request HSRARequestHandler;
 typedef RequestHandler<ID_HttpSendRequestA,
                        BOOL HOOK_CALL (HINTERNET, nsDependentCSubstring,
@@ -668,6 +712,10 @@ bool HSRARequestHandler::ShouldBroker(Endpoint endpoint, const HINTERNET& h,
 typedef FunctionBroker<ID_HttpSendRequestExA,
                        decltype(HttpSendRequestExA)> HttpSendRequestExAFB;
 
+template<>
+ShouldHookFunc* const
+HttpSendRequestExAFB::BaseType::mShouldHook = &CheckQuirks<QUIRK_FLASH_HOOK_SSL>;
+
 typedef RequestInfo<ID_HttpSendRequestExA> HSRExAReqInfo;
 
 template<> template<>
@@ -688,6 +736,10 @@ const DWORD_PTR HSRExAReqInfo::FixedValue<4>::value = 0;
 typedef FunctionBroker<ID_InternetQueryOptionA,
                        decltype(InternetQueryOptionA)> InternetQueryOptionAFB;
 
+template<>
+ShouldHookFunc* const
+InternetQueryOptionAFB::BaseType::mShouldHook = &CheckQuirks<QUIRK_FLASH_HOOK_SSL>;
+
 typedef InternetQueryOptionAFB::Request IQOARequestHandler;
 typedef RequestHandler<ID_InternetQueryOptionA,
                        BOOL HOOK_CALL (HINTERNET, DWORD, DWORD)> IQOADelegateRequestHandler;
@@ -771,6 +823,10 @@ bool IQOAResponseHandler::Unmarshal(const IpdlTuple& aTuple, BOOL& ret, HINTERNE
 typedef FunctionBroker<ID_InternetErrorDlg,
                        decltype(InternetErrorDlg)> InternetErrorDlgFB;
 
+template<>
+ShouldHookFunc* const
+InternetErrorDlgFB::BaseType::mShouldHook = &CheckQuirks<QUIRK_FLASH_HOOK_SSL>;
+
 typedef RequestInfo<ID_InternetErrorDlg> IEDReqInfo;
 
 template<> template<>
@@ -800,6 +856,10 @@ bool IEDReqHandler::ShouldBroker(Endpoint endpoint, const HWND& hwnd,
 typedef FunctionBroker<ID_AcquireCredentialsHandleA,
                        decltype(AcquireCredentialsHandleA)> AcquireCredentialsHandleAFB;
 
+template<>
+ShouldHookFunc* const
+AcquireCredentialsHandleAFB::BaseType::mShouldHook = &CheckQuirks<QUIRK_FLASH_HOOK_SSL>;
+
 typedef RequestInfo<ID_AcquireCredentialsHandleA> ACHAReqInfo;
 
 template<> template<>
@@ -875,11 +935,19 @@ struct ACHARspInfo::ShouldMarshal<8> { static const bool value = true; };
 typedef FunctionBroker<ID_QueryCredentialsAttributesA,
                        decltype(QueryCredentialsAttributesA)> QueryCredentialsAttributesAFB;
 
+template<>
+ShouldHookFunc* const
+QueryCredentialsAttributesAFB::BaseType::mShouldHook = &CheckQuirks<QUIRK_FLASH_HOOK_SSL>;
+
 /* FreeCredentialsHandle */
 
 typedef FunctionBroker<ID_FreeCredentialsHandle,
                        decltype(FreeCredentialsHandle)> FreeCredentialsHandleFB;
 
+template<>
+ShouldHookFunc* const
+FreeCredentialsHandleFB::BaseType::mShouldHook = &CheckQuirks<QUIRK_FLASH_HOOK_SSL>;
+
 typedef FreeCredentialsHandleFB::Request FCHReq;
 
 template<>

dom/plugins/ipc/PluginQuirks.cpp

@@ -30,6 +30,7 @@ int GetQuirksFromMimeTypeAndFilename(const nsCString& aMimeType,
 #if defined(_M_X64) || defined(__x86_64__)
         quirks |= QUIRK_FLASH_HOOK_GETKEYSTATE;
         quirks |= QUIRK_FLASH_HOOK_PRINTDLGW;
+        quirks |= QUIRK_FLASH_HOOK_SSL;
 #endif
 #endif
     }

dom/plugins/ipc/PluginQuirks.h

@@ -49,6 +49,8 @@ enum PluginQuirks {
   QUIRK_FLASH_HOOK_GETKEYSTATE                    = 1 << 13,
   // Win: Hook PrintDlgW to show print settings dialog on sandbox process
   QUIRK_FLASH_HOOK_PRINTDLGW                      = 1 << 14,
+  // Win: Broker Win32 SSL operations
+  QUIRK_FLASH_HOOK_SSL                            = 1 << 15,
 };
 
 int GetQuirksFromMimeTypeAndFilename(const nsCString& aMimeType,