Bug 1773207 - "libgmpopenh264.dylib" can't be opened because Apple cannot check it for malicious software r=media-playback-reviewers,kinetik

Use the correct function name (delMacXAttr vs removeMacXAttr) to remove the com.apple.quarantine extended attribute from downloaded plugin files.

Add test code to ensure extracted plugin files do not have the quarantine attribute.

Differential Revision: https://phabricator.services.mozilla.com/D148739

toolkit/modules/GMPExtractorWorker.js

@@ -55,12 +55,12 @@ onmessage = async function(msg) {
       // Ensure files are writable and executable. Otherwise, we may be
       // unable to execute or uninstall them.
       await IOUtils.setPermissions(destPath, 0o700);
-      if (IOUtils.removeMacXAttr) {
+      if (IOUtils.delMacXAttr) {
         // If we're on MacOS Firefox will add the quarantine xattr to files it
         // downloads. In this case we want to clear that xattr so we can load
         // the CDM.
         try {
-          await IOUtils.removeMacXAttr(destPath, "com.apple.quarantine");
+          await IOUtils.delMacXAttr(destPath, "com.apple.quarantine");
         } catch (e) {
           // Failed to remove the attribute. This could be because the profile
           // exists on a file system without xattr support.

toolkit/modules/tests/xpcshell/test_GMPInstallManager.js

@@ -28,6 +28,9 @@ const { GMPPrefs, OPEN_H264_ID } = ChromeUtils.import(
 const { ProductAddonCheckerTestUtils } = ChromeUtils.import(
   "resource://gre/modules/addons/ProductAddonChecker.jsm"
 );
+const { AppConstants } = ChromeUtils.import(
+  "resource://gre/modules/AppConstants.jsm"
+);
 
 Services.prefs.setBoolPref("security.allow_eval_with_system_principal", true);
 Services.prefs.setBoolPref("media.gmp-manager.updateEnabled", true);
@@ -1148,6 +1151,9 @@ add_test(function test_installAddon_noServer() {
 /***
  * Tests GMPExtractor (an internal component of GMPInstallManager) to ensure
  * it handles paths with certain characters.
+ *
+ * On Mac, test that the com.apple.quarantine extended attribute is removed
+ * from installed plugin files.
  */
 
 add_task(async function test_GMPExtractor_paths() {
@@ -1204,6 +1210,13 @@ add_task(async function test_GMPExtractor_paths() {
     await IOUtils.exists(extractedFile),
     "Extraction should have created dummy_file.txt"
   );
+  if (AppConstants.platform == "macosx") {
+    await Assert.rejects(
+      IOUtils.getMacXAttr(extractedFile, "com.apple.quarantine"),
+      /NotFoundError: The file `.+' does not have an extended attribute `com.apple.quarantine'/,
+      "The 'com.apple.quarantine' attribute should not be present"
+    );
+  }
   let unextractedFile = PathUtils.join(
     PathUtils.profileDir,
     relativeExtractPath,