Bug 1184333 - Handle UTF8 paths as input for GMP loading. r=bobowen

dom/media/gmp/GMPChild.cpp

@@ -51,6 +51,13 @@ extern PRLogModuleInfo* GetGMPLog();
 
 namespace gmp {
 
+static bool
+FileExists(nsIFile* aFile)
+{
+  bool exists = false;
+  return aFile && NS_SUCCEEDED(aFile->Exists(&exists)) && exists;
+}
+
 GMPChild::GMPChild()
   : mAsyncShutdown(nullptr)
   , mGMPMessageLoop(MessageLoop::current())
@@ -269,7 +276,7 @@ GMPChild::Init(const std::string& aPluginPath,
 #endif
 
   mPluginPath = aPluginPath;
-  mVoucherPath = aVoucherPath;
+  mSandboxVoucherPath = aVoucherPath;
   return true;
 }
 
@@ -323,6 +330,7 @@ GMPChild::PreLoadLibraries(const std::string& aPluginPath)
 
   std::ifstream stream;
 #ifdef _MSC_VER
+  // Must use UTF16 for Windows for paths for non-Latin characters.
   stream.open(static_cast<const wchar_t*>(path.get()));
 #else
   stream.open(NS_ConvertUTF16toUTF8(path).get());
@@ -363,7 +371,7 @@ GMPChild::PreLoadLibraries(const std::string& aPluginPath)
 #endif
 
 bool
-GMPChild::GetLibPath(nsACString& aOutLibPath)
+GMPChild::GetUTF8LibPath(nsACString& aOutLibPath)
 {
 #if defined(XP_MACOSX) && defined(MOZ_GMP_SANDBOX)
   nsAutoCString pluginDirectoryPath, pluginFilePath;
@@ -377,7 +385,17 @@ GMPChild::GetLibPath(nsACString& aOutLibPath)
   if (!GetPluginFile(mPluginPath, libFile)) {
     return false;
   }
-  return NS_SUCCEEDED(libFile->GetNativePath(aOutLibPath));
+
+  if (!FileExists(libFile)) {
+    NS_WARNING("Can't find GMP library file!");
+    return false;
+  }
+
+  nsAutoString path;
+  libFile->GetPath(path);
+  aOutLibPath = NS_ConvertUTF16toUTF8(path);
+
+  return true;
 #endif
 }
 
@@ -389,11 +407,14 @@ GMPChild::RecvStartPlugin()
 #if defined(XP_WIN)
   PreLoadLibraries(mPluginPath);
 #endif
-  PreLoadPluginVoucher(mPluginPath);
+  if (!PreLoadPluginVoucher(mPluginPath)) {
+    NS_WARNING("Plugin voucher failed to load!");
+    return false;
+  }
   PreLoadSandboxVoucher();
 
   nsCString libPath;
-  if (!GetLibPath(libPath)) {
+  if (!GetUTF8LibPath(libPath)) {
     return false;
   }
 
@@ -613,11 +634,20 @@ GMPChild::PreLoadPluginVoucher(const std::string& aPluginPath)
   nsCOMPtr<nsIFile> voucherFile;
   GetPluginVoucherFile(aPluginPath, voucherFile);
 
+  if (!FileExists(voucherFile)) {
+    return true;
+  }
+
   nsString path;
   voucherFile->GetPath(path);
 
   std::ifstream stream;
+  #ifdef _MSC_VER
+  // Must use UTF16 for Windows for paths for non-Latin characters.
+  stream.open(static_cast<const wchar_t*>(path.get()), std::ios::binary);
+  #else
   stream.open(NS_ConvertUTF16toUTF8(path).get(), std::ios::binary);
+  #endif
   if (!stream.good()) {
     return false;
   }
@@ -646,7 +676,15 @@ void
 GMPChild::PreLoadSandboxVoucher()
 {
   std::ifstream stream;
-  stream.open(mVoucherPath.c_str(), std::ios::binary);
+  #ifdef _MSC_VER
+  // Must use UTF16 for Windows for paths for non-Latin characters.
+  nsDependentCString utf8Path(mSandboxVoucherPath.c_str(),
+                              mSandboxVoucherPath.size());
+  NS_ConvertUTF8toUTF16 utf16Path(utf8Path);
+  stream.open(static_cast<const wchar_t*>(utf16Path.get()), std::ios::binary);
+  #else
+  stream.open(mSandboxVoucherPath.c_str(), std::ios::binary);
+  #endif
   if (!stream.good()) {
     NS_WARNING("PreLoadSandboxVoucher can't find sandbox voucher file!");
     return;

dom/media/gmp/GMPChild.h

@@ -53,7 +53,7 @@ private:
   bool PreLoadPluginVoucher(const std::string& aPluginPath);
   void PreLoadSandboxVoucher();
 
-  bool GetLibPath(nsACString& aOutLibPath);
+  bool GetUTF8LibPath(nsACString& aOutLibPath);
 
   virtual bool RecvSetNodeId(const nsCString& aNodeId) override;
   virtual bool RecvStartPlugin() override;
@@ -88,7 +88,7 @@ private:
 
   MessageLoop* mGMPMessageLoop;
   std::string mPluginPath;
-  std::string mVoucherPath;
+  std::string mSandboxVoucherPath;
   std::string mNodeId;
   GMPLoader* mGMPLoader;
   nsTArray<uint8_t> mPluginVoucher;

dom/media/gmp/GMPLoader.cpp

@@ -10,6 +10,7 @@
 #include "gmp-entrypoints.h"
 #include "prlink.h"
 #include "prenv.h"
+#include "nsAutoPtr.h"
 
 #include <string>
 
@@ -69,8 +70,8 @@ public:
   {}
   virtual ~GMPLoaderImpl() {}
 
-  virtual bool Load(const char* aLibPath,
-                    uint32_t aLibPathLen,
+  virtual bool Load(const char* aUTF8LibPath,
+                    uint32_t aUTF8LibPathLen,
                     char* aOriginSalt,
                     uint32_t aOriginSaltLen,
                     const GMPPlatformAPI* aPlatformAPI) override;
@@ -133,8 +134,8 @@ GetStackAfterCurrentFrame(uint8_t** aOutTop, uint8_t** aOutBottom)
 #endif
 
 bool
-GMPLoaderImpl::Load(const char* aLibPath,
-                    uint32_t aLibPathLen,
+GMPLoaderImpl::Load(const char* aUTF8LibPath,
+                    uint32_t aUTF8LibPathLen,
                     char* aOriginSalt,
                     uint32_t aOriginSaltLen,
                     const GMPPlatformAPI* aPlatformAPI)
@@ -200,14 +201,13 @@ GMPLoaderImpl::Load(const char* aLibPath,
   // loader will attempt to create an activation context which will fail because
   // of the sandbox. If we create an activation context before we start the
   // sandbox then this one will get picked up by the DLL loader.
-  int pathLen = MultiByteToWideChar(CP_ACP, 0, aLibPath, -1, nullptr, 0);
+  int pathLen = MultiByteToWideChar(CP_UTF8, 0, aUTF8LibPath, -1, nullptr, 0);
   if (pathLen == 0) {
     return false;
   }
 
-  wchar_t* widePath = new wchar_t[pathLen];
-  if (MultiByteToWideChar(CP_ACP, 0, aLibPath, -1, widePath, pathLen) == 0) {
-    delete[] widePath;
+  nsAutoArrayPtr<wchar_t> widePath(new wchar_t[pathLen]);
+  if (MultiByteToWideChar(CP_UTF8, 0, aUTF8LibPath, -1, widePath, pathLen) == 0) {
     return false;
   }
 
@@ -216,20 +216,24 @@ GMPLoaderImpl::Load(const char* aLibPath,
   actCtx.lpSource = widePath;
   actCtx.lpResourceName = ISOLATIONAWARE_MANIFEST_RESOURCE_ID;
   ScopedActCtxHandle actCtxHandle(CreateActCtx(&actCtx));
-  delete[] widePath;
 #endif
 
   // Start the sandbox now that we've generated the device bound node id.
   // This must happen after the node id is bound to the device id, as
   // generating the device id requires privileges.
-  if (mSandboxStarter && !mSandboxStarter->Start(aLibPath)) {
+  if (mSandboxStarter && !mSandboxStarter->Start(aUTF8LibPath)) {
     return false;
   }
 
   // Load the GMP.
   PRLibSpec libSpec;
-  libSpec.value.pathname = aLibPath;
+#ifdef XP_WIN
+  libSpec.value.pathname_u = widePath;
+  libSpec.type = PR_LibSpec_PathnameU;
+#else
+  libSpec.value.pathname = aUTF8LibPath;
   libSpec.type = PR_LibSpec_Pathname;
+#endif
   mLib = PR_LoadLibraryWithFlags(libSpec, 0);
   if (!mLib) {
     return false;

dom/media/gmp/GMPLoader.h

@@ -52,7 +52,7 @@ public:
   // Calculates the device-bound node id, then activates the sandbox,
   // then loads the GMP library and (if applicable) passes the bound node id
   // to the GMP.
-  virtual bool Load(const char* aLibPath,
+  virtual bool Load(const char* aUTF8LibPath,
                     uint32_t aLibPathLen,
                     char* aOriginSalt,
                     uint32_t aOriginSaltLen,