Bug 1637402 - Add pref to compare MLS results r=chutten,mikedeboer

Differential Revision: https://phabricator.services.mozilla.com/D74953

dom/base/LocationHelper.jsm

@@ -32,4 +32,27 @@ class LocationHelper {
       .sort(sort)
       .map(encode);
   }
+
+  /**
+   * Calculate the distance between 2 points using the Haversine formula.
+   * https://en.wikipedia.org/wiki/Haversine_formula
+   */
+  static distance(p1, p2) {
+    let rad = x => (x * Math.PI) / 180;
+    // Radius of the earth.
+    let R = 6371e3;
+    let lat = rad(p2.lat - p1.lat);
+    let lng = rad(p2.lng - p1.lng);
+
+    let a =
+      Math.sin(lat / 2) * Math.sin(lat / 2) +
+      Math.cos(rad(p1.lat)) *
+        Math.cos(rad(p2.lat)) *
+        Math.sin(lng / 2) *
+        Math.sin(lng / 2);
+
+    let c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
+
+    return R * c;
+  }
 }

dom/system/NetworkGeolocationProvider.jsm

@@ -10,13 +10,16 @@ const { XPCOMUtils } = ChromeUtils.import(
 const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
 
 XPCOMUtils.defineLazyModuleGetters(this, {
+  clearTimeout: "resource://gre/modules/Timer.jsm",
   LocationHelper: "resource://gre/modules/LocationHelper.jsm",
+  setTimeout: "resource://gre/modules/Timer.jsm",
 });
 
-XPCOMUtils.defineLazyGlobalGetters(this, ["XMLHttpRequest"]);
+XPCOMUtils.defineLazyGlobalGetters(this, ["fetch"]);
 
 // GeolocationPositionError has no interface object, so we can't use that here.
 const POSITION_UNAVAILABLE = 2;
+const TELEMETRY_KEY = "REGION_LOCATION_SERVICES_DIFFERENCE";
 
 XPCOMUtils.defineLazyPreferenceGetter(
   this,
@@ -274,6 +277,13 @@ function NetworkGeolocationProvider() {
     true
   );
 
+  XPCOMUtils.defineLazyPreferenceGetter(
+    this,
+    "_wifiCompareURL",
+    "geo.provider.network.compare.url",
+    null
+  );
+
   this.wifiService = null;
   this.timer = null;
   this.started = false;
@@ -418,7 +428,7 @@ NetworkGeolocationProvider.prototype = {
    *                          ]
    *                          </code>
    */
-  sendLocationRequest(wifiData) {
+  async sendLocationRequest(wifiData) {
     let data = { cellTowers: undefined, wifiAccessPoints: undefined };
     if (wifiData && wifiData.length >= 2) {
       data.wifiAccessPoints = wifiData;
@@ -443,47 +453,16 @@ NetworkGeolocationProvider.prototype = {
     let url = Services.urlFormatter.formatURLPref("geo.provider.network.url");
     LOG("Sending request");
 
-    let xhr = new XMLHttpRequest();
-    this.onStatus(false, "xhr-start");
+    let result;
     try {
-      xhr.open("POST", url, true);
-      xhr.channel.loadFlags = Ci.nsIChannel.LOAD_ANONYMOUS;
-    } catch (e) {
-      this.onStatus(true, "xhr-error");
-      return;
-    }
-    xhr.setRequestHeader("Content-Type", "application/json; charset=UTF-8");
-    xhr.responseType = "json";
-    xhr.mozBackgroundRequest = true;
-    // Allow deprecated HTTP request from SystemPrincipal
-    xhr.channel.loadInfo.allowDeprecatedSystemRequests = true;
-    xhr.timeout = Services.prefs.getIntPref("geo.provider.network.timeout");
-    xhr.ontimeout = () => {
-      LOG("Location request XHR timed out.");
-      this.onStatus(true, "xhr-timeout");
-    };
-    xhr.onerror = () => {
-      this.onStatus(true, "xhr-error");
-    };
-    xhr.onload = () => {
+      result = await this.makeRequest(url, wifiData);
       LOG(
-        "server returned status: " +
-          xhr.status +
-          " --> " +
-          JSON.stringify(xhr.response)
+        `geo provider reported: ${result.location.lng}:${result.location.lat}`
       );
-      if (
-        (xhr.channel instanceof Ci.nsIHttpChannel && xhr.status != 200) ||
-        !xhr.response
-      ) {
-        this.onStatus(true, !xhr.response ? "xhr-empty" : "xhr-error");
-        return;
-      }
-
       let newLocation = new NetworkGeoPositionObject(
-        xhr.response.location.lat,
-        xhr.response.location.lng,
-        xhr.response.accuracy
+        result.location.lat,
+        result.location.lng,
+        result.accuracy
       );
 
       if (this.listener) {
@@ -495,11 +474,56 @@ NetworkGeolocationProvider.prototype = {
         data.cellTowers,
         data.wifiAccessPoints
       );
+    } catch (err) {
+      LOG("Location request hit error: " + err.name);
+      Cu.reportError(err);
+      if (err.name == "AbortError") {
+        this.onStatus(true, "xhr-timeout");
+      } else {
+        this.onStatus(true, "xhr-error");
+      }
+    }
+
+    if (!this._wifiCompareURL) {
+      return;
+    }
+
+    let compareUrl = Services.urlFormatter.formatURL(this._wifiCompareURL);
+    let compare = await this.makeRequest(compareUrl, wifiData);
+    let distance = LocationHelper.distance(result.location, compare.location);
+    LOG(
+      `compare reported reported: ${compare.location.lng}:${compare.location.lat}`
+    );
+    LOG(`distance between results: ${distance}`);
+    if (!isNaN(distance)) {
+      Services.telemetry.getHistogramById(TELEMETRY_KEY).add(distance);
+    }
+  },
+
+  async makeRequest(url, wifiData) {
+    this.onStatus(false, "xhr-start");
+
+    let fetchController = new AbortController();
+    let fetchOpts = {
+      method: "POST",
+      headers: { "Content-Type": "application/json; charset=UTF-8" },
+      credentials: "omit",
+      signal: fetchController.signal,
     };
 
-    var requestData = JSON.stringify(data);
-    LOG("sending " + requestData);
-    xhr.send(requestData);
+    if (wifiData) {
+      fetchOpts.body = JSON.stringify({ wifiAccessPoints: wifiData });
+    }
+
+    let timeoutId = setTimeout(
+      () => fetchController.abort(),
+      Services.prefs.getIntPref("geo.provider.network.timeout")
+    );
+
+    let req = await fetch(url, fetchOpts);
+    clearTimeout(timeoutId);
+    let result = req.json();
+    return result;
   },
 };
 

dom/system/tests/location_service.sjs

@@ -0,0 +1,39 @@
+function parseQueryString(str) {
+  if (str == "") {
+    return {};
+  }
+
+  var paramArray = str.split("&");
+  var regex = /^([^=]+)=(.*)$/;
+  var params = {};
+  for (var i = 0, sz = paramArray.length; i < sz; i++) {
+    var match = regex.exec(paramArray[i]);
+    if (!match) {
+      throw new Error("Bad parameter in queryString!  '" + paramArray[i] + "'");
+    }
+    params[decodeURIComponent(match[1])] = decodeURIComponent(match[2]);
+  }
+
+  return params;
+}
+
+function getPosition(params) {
+  var response = {
+    status: "OK",
+    accuracy: 100,
+    location: {
+      lat: params.lat,
+      lng: params.lng,
+    },
+  };
+
+  return JSON.stringify(response);
+}
+
+function handleRequest(request, response) {
+  let params = parseQueryString(request.queryString);
+  response.setStatusLine("1.0", 200, "OK");
+  response.setHeader("Cache-Control", "no-cache", false);
+  response.setHeader("Content-Type", "application/x-javascript", false);
+  response.write(getPosition(params));
+}

dom/system/tests/location_services_parent.js

@@ -0,0 +1,20 @@
+/**
+ * Loaded as a frame script fetch telemetry for
+ * test_location_services_telemetry.html
+ */
+
+/* global addMessageListener, sendAsyncMessage */
+
+"use strict";
+
+const HISTOGRAM_KEY = "REGION_LOCATION_SERVICES_DIFFERENCE";
+
+addMessageListener("getTelemetryEvents", options => {
+  let result = Services.telemetry.getHistogramById(HISTOGRAM_KEY).snapshot();
+  sendAsyncMessage("getTelemetryEvents", result);
+});
+
+addMessageListener("clear", options => {
+  Services.telemetry.getHistogramById(HISTOGRAM_KEY).clear();
+  sendAsyncMessage("clear", true);
+});

dom/system/tests/mochitest.ini

@@ -1,5 +1,11 @@
 [DEFAULT]
+scheme = https
+
 support-files =
   file_bug1197901.html
+  location_services_parent.js
+  location_service.sjs
 
 [test_bug1197901.html]
+[test_location_services_telemetry.html]
+skip-if = os == "android"

dom/system/tests/test_location_services_telemetry.html

@@ -0,0 +1,143 @@
+<!DOCTYPE HTML>
+<html>
+<!--
+https://bugzilla.mozilla.org/show_bug.cgi?id=1637402
+-->
+<head>
+  <meta charset="utf-8">
+  <title>Test for Bug 1637402</title>
+  <script src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+  <script type="application/javascript">
+
+  SimpleTest.requestLongerTimeout(2);
+
+  const BASE_GEO_URL = "http://mochi.test:8888/tests/dom/system/tests/location_service.sjs";
+
+  const GEO_PREF = "geo.provider.network.url";
+  const BACKUP_PREF = "geo.provider.network.compare.url";
+
+  const PARENT = SpecialPowers.loadChromeScript(SimpleTest.getTestFileURL("location_services_parent.js"));
+
+  function sendToParent(msg, options) {
+    return new Promise(resolve => {
+      PARENT.addMessageListener(msg, events => {
+        PARENT.removeMessageListener(msg);
+        resolve(events);
+      });
+      PARENT.sendAsyncMessage(msg, options);
+    });
+  }
+
+  function getCurrentPosition() {
+    return new Promise(function(resolve, reject) {
+      navigator.geolocation.getCurrentPosition(resolve, reject);
+    });
+  }
+
+  let tries = 0;
+  let MAX_RETRIES = 500;
+  async function waitFor(fun) {
+    let passing = false;
+    while (!passing && ++tries < MAX_RETRIES) {
+      passing = await fun();
+    }
+    tries = 0;
+    if (!passing) {
+      ok(false, "waitFor condition never passed");
+    }
+  }
+
+  // Keeps track of how many telemetry results we have
+  // seen so we can wait for new ones.
+  let telemetryResultCount = 0;
+  async function newTelemetryResult() {
+    let results = await sendToParent("getTelemetryEvents");
+    let total = Object.values(results.values)
+      .reduce((val, acc) => acc + val, 0);
+    if (total <= telemetryResultCount) {
+      return false;
+    }
+    telemetryResultCount++;
+    return true;
+  }
+
+  SimpleTest.waitForExplicitFinish();
+  window.onload = () => {
+    SimpleTest.waitForFocus(() => {
+      SpecialPowers.pushPrefEnv({"set":
+        [
+          ["geo.prompt.testing", true],
+          ["geo.prompt.testing.allow", true],
+          ["geo.provider.network.logging.enabled", true],
+          ["geo.provider.network.debug.requestCache.enabled", false]
+        ],
+      }, doTest);
+    }, window);
+  };
+
+  const BASE_LOCATION = {lat: 55.867055, lng: -4.271041};
+  const LOCATIONS = [
+    {lat: "foo", lng: "bar", skipWait: true}, // Nan
+    {lat: 55.867055, lng: -4.271041},   // 0M
+    {lat: 50.8251639, lng: -0.1622551}, // 623KM
+    {lat: 55.9438948, lng: -3.1845417}, // 68KM
+    {lat: 39.4780911, lng: -0.3821706}, // 1844KM
+    {lat: 55.867160, lng: -4.271041},   // 10M
+    {lat: 41.8769913, lng: 12.4835351}, // 1969KM
+    {lat: 55.867055, lng: -4.271041},   // 0M
+  ]
+
+  async function setLocations(main, backup) {
+    await SpecialPowers.setCharPref(
+      GEO_PREF,
+      `${BASE_GEO_URL}?lat=${main.lat}&lng=${main.lng}`
+    );
+    await SpecialPowers.setCharPref(
+      BACKUP_PREF,
+      `${BASE_GEO_URL}?lat=${backup.lat}&lng=${backup.lng}`
+    );
+  }
+
+  async function doTest() {
+    // Not all treeherder builds can collect telemetry.
+    if (!SpecialPowers.Services.telemetry.canRecordPrereleaseData) {
+      ok(true, "Cant run any tests without telemetry");
+      SimpleTest.finish();
+      return;
+    }
+    await sendToParent("clear");
+
+    for (let location of LOCATIONS) {
+      await setLocations(BASE_LOCATION, location);
+      await getCurrentPosition();
+      // Not all requests (NaN) will report telemetry.
+      if (!location.skipWait) {
+        await waitFor(newTelemetryResult, "");
+      }
+    }
+
+    let res = await sendToParent("getTelemetryEvents");
+    let total = Object.values(res.values)
+      .reduce((val, acc) => acc + val, 0);
+
+    is(total, 7, "Should have correct number of results");
+    is(res.values["0"], 2, "Two results were same location");
+    // Telemetry could change how exact bucketing
+    // implementation, so check the low bucket
+    // and that the rest are spead out.
+    is(
+      Object.keys(res.values).length,
+      6,
+      "Split the rest of the results across buckets"
+    );
+
+    SimpleTest.finish();
+  }
+</script>
+</head>
+<body>
+<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=1637402">Mozilla Bug </a>
+<pre id="test"></pre>
+</body>
+</html>

modules/libpref/init/all.js

@@ -3954,6 +3954,9 @@ pref("network.psl.onUpdate_notify", false);
 #else
   // Use MLS on Nightly and early Beta.
   pref("geo.provider.network.url", "https://location.services.mozilla.com/v1/geolocate?key=%MOZILLA_API_KEY%");
+  // On Nightly and early Beta, make duplicate location services requests
+  // to google so we can compare results.
+  pref("geo.provider.network.compare.url", "https://www.googleapis.com/geolocation/v1/geolocate?key=%GOOGLE_LOCATION_SERVICE_API_KEY%");
 #endif
 
 // Timeout to wait before sending the location request.

testing/profiles/common/user.js

@@ -67,7 +67,7 @@ user_pref("media.block-autoplay-until-in-foreground", false);
 user_pref("toolkit.telemetry.coverage.endpoint.base", "http://localhost");
 // Don't ask for a request in testing unless explicitly set this as true.
 user_pref("media.geckoview.autoplay.request", false);
-// user_pref("geo.provider.network.url", "http://localhost/geoip-dummy");
+user_pref("geo.provider.network.compare.url", "");
 user_pref("browser.region.network.url", "http://localhost/geoip-dummy");
 // Do not unload tabs on low memory when testing
 user_pref("browser.tabs.unloadOnLowMemory", false);

testing/profiles/xpcshell/user.js

@@ -10,6 +10,7 @@ user_pref("extensions.webextensions.warnings-as-errors", true);
 // Always use network provider for geolocation tests
 // so we bypass the OSX dialog raised by the corelocation provider
 user_pref("geo.provider.testing", true);
+user_pref("geo.provider.network.compare.url", "");
 user_pref("media.gmp-manager.updateEnabled", false);
 user_pref("media.gmp-manager.url.override", "http://%(server)s/dummy-gmp-manager.xml");
 user_pref("toolkit.telemetry.server", "https://%(server)s/telemetry-dummy");

toolkit/components/telemetry/Histograms.json

@@ -9109,6 +9109,17 @@
     "kind": "boolean",
     "description": "If we are on Windows and neither the Windows countryCode nor the geoip countryCode indicates we are in the US, set to false if they both agree on the value or true otherwise"
   },
+  "REGION_LOCATION_SERVICES_DIFFERENCE": {
+    "record_in_processes": ["main", "content"],
+    "products": ["firefox"],
+    "expires_in_version": "84",
+    "kind": "exponential",
+    "n_buckets": 20,
+    "high": 12742000,
+    "bug_numbers": [1637402],
+    "alert_emails": ["dharvey@mozilla.com"],
+    "description": "The distance(m) between the the result of 2 services that detect the users location"
+  },
   "TOUCH_ENABLED_DEVICE": {
     "record_in_processes": ["main"],
     "products": ["firefox", "fennec", "geckoview"],