Bug 1525076 - Part 1: Fix macOS native `getReferrerUrl` by invoking system sqlite binary (from JS). r=mixedpuppy

The issue this is addresses is that
[`CFURLCopyResourcePropertyForKey`](https://searchfox.org/mozilla-central/rev/ea7f70dac1c5fd18400f6d2a92679777d4b21492/xpcom/io/CocoaFileUtils.mm#212)
does not return quarantine data when launched as a GUI App.  What
happens is that launching via the GUI requires the user to override
GateKeeper by going to Security & Privacy > Open Anyway.  Doing that
updates the GateKeeper flags, and then the macOS API denies access:
once the GK flags reach some state, quarantine information is not
returned.  This is not documented (as far as I can see) but moons ago,
[somebody else on the internet witnessed the same
thing](https://cocoa-dev.apple.narkive.com/kkYeAC8o/is-it-possible-to-read-your-own-quarantine-info-after-launch).

To work around, we run the system SQLite binary, to fish the relevant
information out of the per-user quarantine database.  (SQLite is
installed by default on all relevant macOS versions.)

The most significant security concern I see is whether we can trust
this binary (in /usr/bin/sqlite3). Some discussion within the
Install/Update team suggested that an attacker who could corrupt or
modify that binary already had write access to the disk, which is an
attack vector equal to a totally compromised Firefox. If we determine
that we can't use the system SQLite binary, then we could use
Firefox's compiled copy of SQLite, but we might see versioning
issues. The system SQLite binary feels more robust.

This is implemented as a JS component for convenience, mostly: there
is no API for capturing output from `nsIProcess`.  It would be
possible to maintain the existing XPCOM contract by renaming the
existing contract and adding a contract with a JS implementation that
passes through to the renamed implementation, but it doesn't seem
worth the effort.

In the next commits, we will generalize the existing caching mechanism
form Windows to also apply to macOS. This is mostly a performance
optimization, so that we sniff a single well-known location rather
than launching a process at each startup, although there is a
correctness argument here as well, since the quarantine database is
dynamic and the attribution URL could expire.

Differential Revision: https://phabricator.services.mozilla.com/D92693

browser/components/attribution/AttributionCode.jsm

@@ -133,13 +133,11 @@ var AttributionCode = {
         }
       }
     } else if (AppConstants.platform == "macosx") {
+      const { MacAttribution } = ChromeUtils.import(
+        "resource:///modules/MacAttribution.jsm"
+      );
       try {
-        let appPath = Services.dirsvc.get("GreD", Ci.nsIFile).parent.parent
-          .path;
-        let attributionSvc = Cc["@mozilla.org/mac-attribution;1"].getService(
-          Ci.nsIMacAttributionService
-        );
-        let referrer = attributionSvc.getReferrerUrl(appPath);
+        let referrer = await MacAttribution.getReferrerUrl();
         let params = new URL(referrer).searchParams;
         for (let key of ATTR_CODE_KEYS) {
           // We support the key prefixed with utm_ or not, but intentionally

browser/components/attribution/MacAttribution.jsm

@@ -0,0 +1,186 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+var EXPORTED_SYMBOLS = ["MacAttribution"];
+
+const { XPCOMUtils } = ChromeUtils.import(
+  "resource://gre/modules/XPCOMUtils.jsm"
+);
+XPCOMUtils.defineLazyGetter(this, "log", () => {
+  let ConsoleAPI = ChromeUtils.import("resource://gre/modules/Console.jsm", {})
+    .ConsoleAPI;
+  let consoleOptions = {
+    // tip: set maxLogLevel to "debug" and use log.debug() to create detailed
+    // messages during development. See LOG_LEVELS in Console.jsm for details.
+    maxLogLevel: "error",
+    maxLogLevelPref: "browser.attribution.mac.loglevel",
+    prefix: "MacAttribution",
+  };
+  return new ConsoleAPI(consoleOptions);
+});
+ChromeUtils.defineModuleGetter(this, "OS", "resource://gre/modules/osfile.jsm");
+ChromeUtils.defineModuleGetter(
+  this,
+  "Services",
+  "resource://gre/modules/Services.jsm"
+);
+
+ChromeUtils.defineModuleGetter(
+  this,
+  "Subprocess",
+  "resource://gre/modules/Subprocess.jsm"
+);
+
+/**
+ * Get the location of the user's macOS quarantine database.
+ * @return {String} path.
+ */
+function getQuarantineDatabasePath() {
+  let file = Services.dirsvc.get("Home", Ci.nsIFile);
+  file.append("Library");
+  file.append("Preferences");
+  file.append("com.apple.LaunchServices.QuarantineEventsV2");
+  return file.path;
+}
+
+/**
+ * Query given path for quarantine extended attributes.
+ * @param {String} path of the file to query.
+ * @return {[String, String]} pair of the quarantine data GUID and remaining
+ *                            quarantine data (usually, Gatekeeper flags).
+ * @throws NS_ERROR_NOT_AVAILABLE if there is no quarantine GUID for the given path.
+ * @throws NS_ERROR_UNEXPECTED if there is a quarantine GUID, but it is malformed.
+ */
+async function getQuarantineAttributes(path) {
+  let bytes = await OS.File.macGetXAttr(path, "com.apple.quarantine");
+  if (!bytes) {
+    throw new Components.Exception(
+      `No macOS quarantine xattrs found for ${path}`,
+      Cr.NS_ERROR_NOT_AVAILABLE
+    );
+  }
+
+  let string = new TextDecoder("utf-8").decode(bytes);
+  let parts = string.split(";");
+  if (!parts.length) {
+    throw new Components.Exception(
+      `macOS quarantine data is not ; separated`,
+      Cr.NS_ERROR_UNEXPECTED
+    );
+  }
+  let guid = parts[parts.length - 1];
+  if (guid.length != 36) {
+    // Like "12345678-90AB-CDEF-1234-567890ABCDEF".
+    throw new Components.Exception(
+      `macOS quarantine data guid is not length 36: ${guid.length}`,
+      Cr.NS_ERROR_UNEXPECTED
+    );
+  }
+
+  return { guid, parts };
+}
+
+/**
+ * Invoke system SQLite binary to extract the referrer URL corresponding to
+ * the given GUID from the given macOS quarantine database.
+ * @param {String} path of the user's macOS quarantine database.
+ * @param {String} guid to query.
+ * @return {String} referrer URL.
+ */
+async function queryQuarantineDatabase(
+  guid,
+  path = getQuarantineDatabasePath()
+) {
+  let query = `SELECT COUNT(*), LSQuarantineOriginURLString
+       FROM LSQuarantineEvent
+       WHERE LSQuarantineEventIdentifier = '${guid}'
+       ORDER BY LSQuarantineTimeStamp DESC LIMIT 1`;
+
+  let proc = await Subprocess.call({
+    command: "/usr/bin/sqlite3",
+    arguments: [path, query],
+    environment: {},
+    stderr: "stdout",
+  });
+
+  let stdout = await proc.stdout.readString();
+
+  let { exitCode } = await proc.wait();
+  if (exitCode != 0) {
+    throw new Components.Exception(
+      "Failed to run sqlite3",
+      Cr.NS_ERROR_UNEXPECTED
+    );
+  }
+
+  // Output is like "integer|url".
+  let parts = stdout.split("|", 2);
+  if (parts.length != 2) {
+    throw new Components.Exception(
+      "Failed to parse sqlite3 output",
+      Cr.NS_ERROR_UNEXPECTED
+    );
+  }
+
+  if (parts[0].trim() == "0") {
+    throw new Components.Exception(
+      `Quarantine database does not contain URL for guid ${guid}`,
+      Cr.NS_ERROR_UNEXPECTED
+    );
+  }
+
+  return parts[1].trim();
+}
+
+var MacAttribution = {
+  /**
+   * The file path to the `.app` directory.
+   */
+  get applicationPath() {
+    // On macOS, `GreD` is like "App.app/Contents/macOS".  Return "App.app".
+    return Services.dirsvc.get("GreD", Ci.nsIFile).parent.parent.path;
+  },
+
+  /**
+   * Used by the Attributions system to get the download referrer.
+   *
+   * @param {String} path to get the quarantine data from.
+   *             Usually this is a `.app` directory but can be any
+   *             (existing) file or directory.  Default: `this.applicationPath`.
+   * @return {String} referrer URL.
+   * @throws NS_ERROR_NOT_AVAILABLE if there is no quarantine GUID for the given path.
+   * @throws NS_ERROR_UNEXPECTED if there is a quarantine GUID, but no corresponding referrer URL is known.
+   */
+  async getReferrerUrl(path = this.applicationPath) {
+    log.debug(`getReferrerUrl(${JSON.stringify(path)})`);
+
+    // First, determine the quarantine GUID assigned by macOS to the given path.
+    let guid;
+    try {
+      guid = (await getQuarantineAttributes(path)).guid;
+    } catch (ex) {
+      throw new Components.Exception(
+        `No macOS quarantine GUID found for ${path}`,
+        Cr.NS_ERROR_NOT_AVAILABLE
+      );
+    }
+    log.debug(`getReferrerUrl: guid: ${guid}`);
+
+    // Second, fish the relevant record from the quarantine database.
+    let url = "";
+    try {
+      url = await queryQuarantineDatabase(guid);
+      log.debug(`getReferrerUrl: url: ${url}`);
+    } catch (ex) {
+      // This path is known to macOS but we failed to extract a referrer -- be noisy.
+      throw new Components.Exception(
+        `No macOS quarantine referrer URL found for ${path} with GUID ${guid}`,
+        Cr.NS_ERROR_UNEXPECTED
+      );
+    }
+
+    return url;
+  },
+};

browser/components/attribution/moz.build

@@ -31,3 +31,7 @@ if CONFIG['MOZ_WIDGET_TOOLKIT'] == 'cocoa':
     ]
 
     FINAL_LIBRARY = 'browsercomps'
+
+    EXTRA_JS_MODULES += [
+        'MacAttribution.jsm',
+    ]

browser/components/attribution/nsIMacAttribution.idl

@@ -9,15 +9,7 @@
 interface nsIMacAttributionService : nsISupports
 {
   /**
-   * Used by the Attributions system to get the download referrer.
-   *
-   * @param aFilePath A path to the file to get the quarantine data from.
-   * @returns referrerUrl
-   */
-  AString getReferrerUrl(in AUTF8String aFilePath);
-
-  /**
-   * Used by the tests.
+   * Set the referrer URL on a given path.
    *
    * @param aFilePath A path to the file to set the quarantine data on.
    * @param aReferrer A url to set as the referrer for the download.

browser/components/attribution/nsMacAttribution.cpp

@@ -16,19 +16,6 @@ using namespace mozilla;
 
 NS_IMPL_ISUPPORTS(nsMacAttributionService, nsIMacAttributionService)
 
-NS_IMETHODIMP
-nsMacAttributionService::GetReferrerUrl(const nsACString& aFilePath,
-                                        nsAString& aReferrer) {
-  const nsCString& flat = PromiseFlatCString(aFilePath);
-  CFStringRef filePath = ::CFStringCreateWithCString(
-      kCFAllocatorDefault, flat.get(), kCFStringEncodingUTF8);
-
-  CocoaFileUtils::CopyQuarantineReferrerUrl(filePath, aReferrer);
-
-  ::CFRelease(filePath);
-  return NS_OK;
-}
-
 NS_IMETHODIMP
 nsMacAttributionService::SetReferrerUrl(const nsACString& aFilePath,
                                         const nsACString& aReferrerUrl,

browser/components/attribution/test/xpcshell/test_MacAttribution.js

@@ -4,10 +4,12 @@
 
 "use strict";
 
-const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
+const { MacAttribution } = ChromeUtils.import(
+  "resource:///modules/MacAttribution.jsm"
+);
 
 add_task(async function testValidAttrCodes() {
-  let appPath = Services.dirsvc.get("GreD", Ci.nsIFile).parent.parent.path;
+  let appPath = MacAttribution.applicationPath;
   let attributionSvc = Cc["@mozilla.org/mac-attribution;1"].getService(
     Ci.nsIMacAttributionService
   );
@@ -20,7 +22,7 @@ add_task(async function testValidAttrCodes() {
     // URI-encoded.
     let url = `http://example.com?${encodeURI(decodeURIComponent(entry.code))}`;
     attributionSvc.setReferrerUrl(appPath, url, true);
-    let referrer = attributionSvc.getReferrerUrl(appPath);
+    let referrer = await MacAttribution.getReferrerUrl(appPath);
     equal(referrer, url, "overwrite referrer url");
 
     // Read attribution code from referrer to ensure cache is fresh.
@@ -34,7 +36,7 @@ add_task(async function testValidAttrCodes() {
 
     // Does not overwrite cached existing attribution code.
     attributionSvc.setReferrerUrl(appPath, "http://test.com", false);
-    referrer = attributionSvc.getReferrerUrl(appPath);
+    referrer = await MacAttribution.getReferrerUrl(appPath);
     equal(referrer, url, "update referrer url");
 
     result = await AttributionCode.getAttrDataAsync();
@@ -47,7 +49,7 @@ add_task(async function testValidAttrCodes() {
 });
 
 add_task(async function testInvalidAttrCodes() {
-  let appPath = Services.dirsvc.get("GreD", Ci.nsIFile).parent.parent.path;
+  let appPath = MacAttribution.applicationPath;
   let attributionSvc = Cc["@mozilla.org/mac-attribution;1"].getService(
     Ci.nsIMacAttributionService
   );
@@ -59,7 +61,7 @@ add_task(async function testInvalidAttrCodes() {
     let referrer;
     try {
       attributionSvc.setReferrerUrl(appPath, url, true);
-      referrer = attributionSvc.getReferrerUrl(appPath);
+      referrer = await MacAttribution.getReferrerUrl(appPath);
     } catch (ex) {
       continue;
     }

browser/components/newtab/test/xpcshell/test_ASRouterTargeting_attribution.js

@@ -4,17 +4,19 @@
 
 "use strict";
 
-const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
 const { AttributionCode } = ChromeUtils.import(
   "resource:///modules/AttributionCode.jsm"
 );
 const { ASRouterTargeting } = ChromeUtils.import(
   "resource://activity-stream/lib/ASRouterTargeting.jsm"
 );
+const { MacAttribution } = ChromeUtils.import(
+  "resource:///modules/MacAttribution.jsm"
+);
 
 add_task(async function check_attribution_data() {
   // Some setup to fake the correct attribution data
-  const appPath = Services.dirsvc.get("GreD", Ci.nsIFile).parent.parent.path;
+  const appPath = MacAttribution.applicationPath;
   const attributionSvc = Cc["@mozilla.org/mac-attribution;1"].getService(
     Ci.nsIMacAttributionService
   );
@@ -23,7 +25,7 @@ add_task(async function check_attribution_data() {
   const referrer = `https://allizom.org/anything/?utm_campaign=${campaign}&utm_source=${source}`;
   attributionSvc.setReferrerUrl(appPath, referrer, true);
   AttributionCode._clearCache();
-  AttributionCode.getAttrDataAsync();
+  await AttributionCode.getAttrDataAsync();
 
   const {
     campaign: attributionCampain,