Bug 1528661 - Pass paths to i686 xul and plugin-container binaries to CDM in host verification paths. r=bryce

On Windows on ARM64 we will run the x86 Widevine DLL in an x86
plugin-container.exe with an x86 xul.dll. We therefore should also pass the
paths to these binaries to the CDM in the host files instead of the aarch64
plugin-container.exe. We should still pass the aarch64 xul.dll to the CDM,
as that's in use by the aarch64 firefox.exe.

Differential Revision: https://phabricator.services.mozilla.com/D21507

--HG--
extra : source : 4c22ddbf15b579a57d1790c1b4f6290e661da346
extra : histedit_source : 8f1f76f36c3a18681e8aa373e7cd117c41c11815

dom/media/gmp/GMPChild.cpp

@@ -331,6 +331,33 @@ bool GMPChild::GetUTF8LibPath(nsACString& aOutLibPath) {
 #endif
 }
 
+static nsCOMPtr<nsIFile> AppendFile(nsCOMPtr<nsIFile>&& aFile,
+                                    const nsString& aStr) {
+  return (aFile && NS_SUCCEEDED(aFile->Append(aStr))) ? aFile : nullptr;
+}
+
+static nsCOMPtr<nsIFile> CloneFile(const nsCOMPtr<nsIFile>& aFile) {
+  nsCOMPtr<nsIFile> clone;
+  return (aFile && NS_SUCCEEDED(aFile->Clone(getter_AddRefs(clone)))) ? clone
+                                                                      : nullptr;
+}
+
+static nsCOMPtr<nsIFile> GetParentFile(const nsCOMPtr<nsIFile>& aFile) {
+  nsCOMPtr<nsIFile> parent;
+  return (aFile && NS_SUCCEEDED(aFile->GetParent(getter_AddRefs(parent))))
+             ? parent
+             : nullptr;
+}
+
+#if defined(XP_WIN)
+static bool IsFileLeafEqualToASCII(const nsCOMPtr<nsIFile>& aFile,
+                                   const char* aStr) {
+  nsAutoString leafName;
+  return aFile && NS_SUCCEEDED(aFile->GetLeafName(leafName)) &&
+         leafName.EqualsASCII(aStr);
+}
+#endif
+
 #if defined(XP_WIN)
 #  define FIREFOX_FILE NS_LITERAL_STRING("firefox.exe")
 #  define XUL_LIB_FILE NS_LITERAL_STRING("xul.dll")
@@ -342,25 +369,38 @@ bool GMPChild::GetUTF8LibPath(nsACString& aOutLibPath) {
 #  define XUL_LIB_FILE NS_LITERAL_STRING("libxul.so")
 #endif
 
+static nsCOMPtr<nsIFile> GetFirefoxAppPath(
+    nsCOMPtr<nsIFile> aPluginContainerPath) {
+  MOZ_ASSERT(aPluginContainerPath);
 #if defined(XP_MACOSX)
-static bool GetFirefoxAppPath(nsCOMPtr<nsIFile> aPluginContainerPath,
-                              nsCOMPtr<nsIFile>& aOutFirefoxAppPath) {
+  // On MacOS the firefox binary is a few parent directories up from
+  // plugin-container.
   // aPluginContainerPath will end with something like:
   // xxxx/NightlyDebug.app/Contents/MacOS/plugin-container.app/Contents/MacOS/plugin-container
-  MOZ_ASSERT(aPluginContainerPath);
   nsCOMPtr<nsIFile> path = aPluginContainerPath;
   for (int i = 0; i < 4; i++) {
     nsCOMPtr<nsIFile> parent;
     if (NS_WARN_IF(NS_FAILED(path->GetParent(getter_AddRefs(parent))))) {
-      return false;
+      return nullptr;
     }
     path = parent;
   }
   MOZ_ASSERT(path);
-  aOutFirefoxAppPath = path;
-  return true;
+  return aOutFirefoxAppPath
+#else
+  nsCOMPtr<nsIFile> parent = GetParentFile(aPluginContainerPath);
+#  if XP_WIN
+  if (IsFileLeafEqualToASCII(parent, "i686")) {
+    // We must be on Windows on ARM64, where the plugin-container path will
+    // be in the 'i686' subdir. The firefox.exe is in the parent directory.
+    parent = GetParentFile(parent);
+  }
+#  endif
+  return parent;
+#endif
 }
 
+#if defined(XP_MACOSX)
 static bool GetSigPath(const int aRelativeLayers,
                        const nsString& aTargetSigFileName,
                        nsCOMPtr<nsIFile> aExecutablePath,
@@ -438,15 +478,30 @@ nsTArray<Pair<nsCString, nsCString>> GMPChild::MakeCDMHostVerificationPaths() {
     return paths;
   }
 
+#if defined(XP_WIN)
+  // On Windows on ARM64, we should also append the x86 plugin-container's
+  // xul.dll.
+  const bool isWindowsOnARM64 =
+      IsFileLeafEqualToASCII(GetParentFile(path), "i686");
+  if (isWindowsOnARM64) {
+    nsCOMPtr<nsIFile> x86XulPath =
+        AppendFile(GetParentFile(path), XUL_LIB_FILE);
+    if (FileExists(x86XulPath) && ResolveLinks(x86XulPath) &&
+        NS_SUCCEEDED(x86XulPath->GetPath(str))) {
+      nsCString filePath = NS_ConvertUTF16toUTF8(str);
+      nsCString sigFilePath = filePath + NS_LITERAL_CSTRING(".sig");
+      paths.AppendElement(
+          MakePair(std::move(filePath), std::move(sigFilePath)));
+    }
+  }
+#endif
+
   // Firefox application binary path.
-  nsCOMPtr<nsIFile> appDir;
+  nsCOMPtr<nsIFile> appDir = GetFirefoxAppPath(path);
+  path = AppendFile(CloneFile(appDir), FIREFOX_FILE);
+  if (FileExists(path) && ResolveLinks(path) &&
+      NS_SUCCEEDED(path->GetPath(str))) {
 #if defined(XP_MACOSX)
-  // On MacOS the firefox binary is a few parent directories up from
-  // plugin-container.
-  if (GetFirefoxAppPath(path, appDir) &&
-      NS_SUCCEEDED(appDir->Clone(getter_AddRefs(path))) &&
-      NS_SUCCEEDED(path->Append(FIREFOX_FILE)) && FileExists(path) &&
-      ResolveLinks(path) && NS_SUCCEEDED(path->GetPath(str))) {
     nsCString filePath = NS_ConvertUTF16toUTF8(str);
     nsCString sigFilePath;
     nsCOMPtr<nsIFile> sigFile;
@@ -460,25 +515,19 @@ nsTArray<Pair<nsCString, nsCString>> GMPChild::MakeCDMHostVerificationPaths() {
           nsCString(NS_ConvertUTF16toUTF8(str) + NS_LITERAL_CSTRING(".sig"));
     }
     paths.AppendElement(MakePair(std::move(filePath), std::move(sigFilePath)));
-  }
 #else
-  // Note: re-using 'path' var here, as on Windows/Linux we assume Firefox
-  // executable is in the same directory as plugin-container.
-  if (NS_SUCCEEDED(path->GetParent(getter_AddRefs(appDir))) &&
-      NS_SUCCEEDED(appDir->Clone(getter_AddRefs(path))) &&
-      NS_SUCCEEDED(path->Append(FIREFOX_FILE)) && FileExists(path) &&
-      ResolveLinks(path) && NS_SUCCEEDED(path->GetPath(str))) {
     paths.AppendElement(MakePair(
         nsCString(NS_ConvertUTF16toUTF8(str)),
         nsCString(NS_ConvertUTF16toUTF8(str) + NS_LITERAL_CSTRING(".sig"))));
-  }
 #endif
+  }
+
   // Libxul path. Note: re-using 'path' var here, as we assume libxul is in
   // the same directory as Firefox executable.
   appDir->GetPath(str);
-  if (NS_SUCCEEDED(appDir->Clone(getter_AddRefs(path))) &&
-      NS_SUCCEEDED(path->Append(XUL_LIB_FILE)) && FileExists(path) &&
-      ResolveLinks(path) && NS_SUCCEEDED(path->GetPath(str))) {
+  path = AppendFile(CloneFile(appDir), XUL_LIB_FILE);
+  if (FileExists(path) && ResolveLinks(path) &&
+      NS_SUCCEEDED(path->GetPath(str))) {
     nsCString filePath = NS_ConvertUTF16toUTF8(str);
     nsCString sigFilePath;
 #if defined(XP_MACOSX)

media/gmp-clearkey/0.1/gmp-clearkey.cpp

@@ -18,6 +18,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <vector>
+#include <string>
 
 #include "ClearKeyCDM.h"
 #include "ClearKeySessionManager.h"
@@ -102,10 +103,40 @@ void ClosePlatformFile(cdm::PlatformFile aFile) {
 #endif
 }
 
+static uint32_t NumExpectedHostFiles(const cdm::HostFile* aHostFiles,
+                                     uint32_t aNumFiles) {
+#if !defined(XP_WIN)
+  // We expect 4 binaries: clearkey, libxul, plugin-container, and Firefox.
+  return 4;
+#else
+  // Windows running x64 or x86 natively should also have 4 as above.
+  // For Windows on ARM64, we run an x86 plugin-contianer process under
+  // emulation, and so we expect one additional binary; the x86
+  // xul.dll used by plugin-container.exe.
+  bool i686underAArch64 = false;
+  // Assume that we're running under x86 emulation on an aarch64 host if
+  // one of the paths ends with the x86 plugin-container path we'd expect.
+  const std::wstring plugincontainer = L"i686\\plugin-container.exe";
+  for (uint32_t i = 0; i < aNumFiles; i++) {
+    const cdm::HostFile& hostFile = aHostFiles[i];
+    if (hostFile.file != cdm::kInvalidPlatformFile) {
+      std::wstring path = hostFile.file_path;
+      auto offset = path.find(plugincontainer);
+      if (offset != std::string::npos &&
+          offset == path.size() - plugincontainer.size()) {
+        i686underAArch64 = true;
+        break;
+      }
+    }
+  }
+  return i686underAArch64 ? 5 : 4;
+#endif
+}
+
 CDM_API
 bool VerifyCdmHost_0(const cdm::HostFile* aHostFiles, uint32_t aNumFiles) {
-  // We expect 4 binaries: clearkey, libxul, plugin-container, and Firefox.
-  bool rv = (aNumFiles == 4);
+  // Check that we've received the expected number of host files.
+  bool rv = (aNumFiles == NumExpectedHostFiles(aHostFiles, aNumFiles));
   // Verify that each binary is readable inside the sandbox,
   // and close the handle.
   for (uint32_t i = 0; i < aNumFiles; i++) {