Add Disconnect Entity List (AKA whitelist). Fixes #98.

This PR includes the following changes:
Add [Shavar-Prod-Lists](https://github.com/mozilla-services/shavar-prod-lists) repo as a submodule to use their maintained `disconnect-entitylist.json` whitelist. The submodule is located in the `shavar-prod-lists` subfolder in this repo's root directory.
Update `README.md` with new instructions for cloning the repo with the submodule and updating the submodule
Import the `disconnect-entitylist.json` object, parsed, into `capture.js`
Reformat the object to more easily check if a first and third party pair are whitelisted
Update `capture.shouldStore` method to return `false` if the first and third party from a request are on the whitelist
Add `.eslintignore` to ignore submodule

.eslintignore

@@ -0,0 +1 @@
+/shavar-prod-lists/*

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "shavar-prod-lists"]
+	path = shavar-prod-lists
+	url = https://github.com/mozilla-services/shavar-prod-lists.git

README.md

@@ -5,11 +5,16 @@ The Mozilla Lightbeam extension is a key tool for Mozilla to educate the public
 
 ## Quick Start
 
-* Clone the repository
-`git clone https://github.com/electrolyfish/lightbeam-we.git`
+### Clone the repository
 
-* There are a couple ways to try out this web extension:
-  1. Open Firefox and load `about:debugging` in the URL bar.
+**Note** This repository uses a [submodule](https://github.com/mozilla-services/shavar-prod-lists) to whitelist some third party requests. To ensure the submodule is cloned along with this repository, use a modified `clone` command:
+`git clone --recursive https://github.com/electrolyfish/lightbeam-we.git`
+
+### Run the web extension
+
+There are a couple ways to try out this web extension:
+
+1. Open Firefox and load `about:debugging` in the URL bar.
     - Click the [Load Temporary Add-on](https://developer.mozilla.org/en-US/Add-ons/WebExtensions/Temporary_Installation_in_Firefox) button and select the `manifest.json` file within the directory of this repository.
     - You should now see the Lightbeam icon on the top right bar of the browser.
     - Click the Lightbeam icon to launch the web extension.
@@ -17,21 +22,26 @@ The Mozilla Lightbeam extension is a key tool for Mozilla to educate the public
 
     ![lightbeam-screenshot](/docs/images/lightbeam-launch.gif)
 
-  2. Install the [web-ext](https://developer.mozilla.org/en-US/Add-ons/WebExtensions/Getting_started_with_web-ext) tool, change into the directory of this repository, and type `web-ext run`.
+2. Install the [web-ext](https://developer.mozilla.org/en-US/Add-ons/WebExtensions/Getting_started_with_web-ext) tool, change into the directory of this repository, and type `web-ext run`.
     - This will launch Firefox and install the extension automatically.
     - This tool gives you some additional development features such as [automatic reloading](https://developer.mozilla.org/en-US/Add-ons/WebExtensions/Getting_started_with_web-ext#Automatic_extension_reloading).
 
 ## Development Guide
 
-* Run `npm install`.
-* Run `npm run test` to check that everything is OK.
-  * If you have installed `eslint` globally, you will have to install globally the following `eslint` plugins too:
+### Download dependencies
+Run `npm install`.
+
+### Update the submodule
+To manually update the submodule at any time during development, run `git submodule update`.
+
+### Testing
+Run `npm run test` to check that everything is OK.
+
+* If you have installed `eslint` globally, you will have to install globally the following `eslint` plugins too:
     - `eslint-plugin-json`
     - `eslint-plugin-mocha`
-* Test suites include lint and unit testing.
-  * You can individually run lint or unit tests using the following commands:
+* Test suites include lint and unit testing. You can individually run lint or unit tests using the following commands:
     * `npm run lint:eslint`
     * `npm run test:karma`
-* Eslint is used for linting.
-* Karma, Mocha & Chai are used for unit testing.
-* Additionally the test suites are run on the Travis service providing continuous integration support.
+
+Eslint is used for linting. Karma, Mocha & Chai are used for unit testing. Additionally the test suites are run on the Travis service providing continuous integration support.

js/capture.js

@@ -3,8 +3,16 @@
 * third-party requests to storage.
 */
 const capture = {
-  init() {
+  async init() {
     this.addListeners();
+    // get Disconnect Entity List from shavar-prod-lists submodule
+    let whiteList
+      = await fetch('/shavar-prod-lists/disconnect-entitylist.json');
+    whiteList = await whiteList.json();
+    const { firstPartyWhiteList, thirdPartyWhiteList }
+      = this.reformatList(whiteList);
+    this.firstPartyWhiteList = firstPartyWhiteList;
+    this.thirdPartyWhiteList = thirdPartyWhiteList;
   },
 
   addListeners() {
@@ -19,18 +27,127 @@ const capture = {
       });
   },
 
+/*
+  disconnect-entitylist.json is expected to match this format, where:
+    - 'properties' keys are first parties
+    - 'resources' keys are third parties
+
+  {
+    "Facebook" : {
+      "properties": [
+        "facebook.com",
+        "facebook.de",
+        ...
+        "messenger.com"
+      ],
+      "resources": [
+        "facebook.com",
+        "facebook.de",
+        ...
+        "akamaihd.net"
+      ]
+    }
+
+    "Google" : {
+      ...
+    }
+  }
+
+  this.firstPartyWhiteList is expected to match this format:
+  {
+    "google.com": 1,
+    "abc.xyz": 1
+    ....
+    "facebook.com": 2,
+    ...
+  }
+
+  this.thirdPartyWhiteList is expected to match this format:
+  {
+    1: [
+      "google.com",
+      "googleanalytics.com",
+      "weloveevilstuff.com"
+    ]
+  }
+*/
+
+  reformatList(whiteList) {
+    const firstPartyWhiteList = {};
+    const thirdPartyWhiteList = {};
+    let counter = 0;
+    for (const siteOwner in whiteList) {
+      const firstParties = whiteList[siteOwner].properties;
+      for (let i = 0; i < firstParties.length; i++) {
+        firstPartyWhiteList[firstParties[i]] = counter;
+      }
+      const thirdParties = whiteList[siteOwner].resources;
+      thirdPartyWhiteList[counter] = [];
+      for (let i = 0; i < thirdParties.length; i++) {
+        thirdPartyWhiteList[counter].push(thirdParties[i]);
+      }
+      counter++;
+    }
+
+    return {
+      firstPartyWhiteList,
+      thirdPartyWhiteList
+    };
+  },
+
   // Returns true if the request should be stored, otherwise false.
-  shouldStore(tab) {
+  shouldStore(tab, thirdPartyFromRequest) {
     const documentUrl = new URL(tab.url);
+    const firstPartyFromRequest = documentUrl.hostname;
     // ignore about:*, moz-extension:* & non-visible tabs (like dev tools)
+    // also ignore third parties owned by first parties
     if (documentUrl.protocol !== 'about:'
       && documentUrl.protocol !== 'moz-extension:'
-      && tab.id !== browser.tabs.TAB_ID_NONE) {
+      && tab.id !== browser.tabs.TAB_ID_NONE
+      && !(this.onWhitelist(firstPartyFromRequest, thirdPartyFromRequest))) {
       return true;
     }
     return false;
   },
 
+  // check if third party is on the whitelist (owned by the first party)
+  // returns true if it is and false otherwise
+  onWhitelist(firstPartyFromRequest, thirdPartyFromRequest) {
+    if (thirdPartyFromRequest) {
+      const hostnameVariantsFirstParty
+        = this.getHostnameVariants(firstPartyFromRequest);
+      for (let i = 0; i < hostnameVariantsFirstParty.length; i++) {
+        if (this.firstPartyWhiteList
+          .hasOwnProperty(hostnameVariantsFirstParty[i])) {
+          // first party is in the whitelist
+          const index = this.firstPartyWhiteList[hostnameVariantsFirstParty[i]];
+          const hostnameVariantsThirdParty
+            = this.getHostnameVariants(thirdPartyFromRequest);
+          for (let j = 0; j < hostnameVariantsThirdParty.length; j++) {
+            if (this.thirdPartyWhiteList[index]
+              .includes(hostnameVariantsThirdParty[j])) {
+              return true;
+            }
+          }
+          return false;
+        }
+      }
+    }
+    return false;
+  },
+
+  getHostnameVariants(hostname) {
+    const hostnameVariants = [hostname];
+    const hostnameArr = hostname.split('.');
+    const numDots = hostnameArr.length - 1;
+    for (let i = 0; i < numDots - 1; i++) {
+      hostnameArr.shift();
+      hostname = hostnameArr.join('.');
+      hostnameVariants.push(hostname);
+    }
+    return hostnameVariants;
+  },
+
   // capture third party requests
   async sendThirdParty(response) {
     const tab = await browser.tabs.get(response.tabId);
@@ -39,7 +156,7 @@ const capture = {
     const originUrl = new URL(response.originUrl);
 
     if (targetUrl.hostname !== documentUrl.hostname
-      && this.shouldStore(tab)) {
+      && this.shouldStore(tab, targetUrl.hostname)) {
       const data = {
         document: documentUrl.hostname,
         target: targetUrl.hostname,

shavar-prod-lists

@@ -0,0 +1 @@
+Subproject commit 218c5dffa23b72d23520c9926219b3ac34225ed7