Merge pull request #56 from mozilla/nimbledroid

Nimbledroid changes

README.md

@@ -52,11 +52,13 @@ Once you have it you can start the backend like this:
 ```
 export NIMBLEDROID_API_KEY=<API key>
 export NIMBLEDROID_EMAIL=<your email address>
-yarn fetchNimbledroidData
+export REDIS_URL=redis://localhost:6379
+// Change DEBUG to script:* if you also want debug output
+DEBUG=script:info,script:error yarn fetchNimbledroidData
 yarn start
 ```
 
-Load [this page](http://localhost:3000/api/nimbledroid?product=focus) to verify it works.
+Load [this page](http://localhost:3000/api/android/nimbledroid?product=com.chrome.beta) to verify it works.
 
 ### Redis
 
@@ -91,6 +93,14 @@ Ths add-on expects the Heroku scheduled job to consistently report a successful
 If DMS does not hear back from the script after a couple of hours it will send an email notifying few users.
 You can adjust who the recepients of the alerts are via the add-on on Heroku.
 
+### Nimbledroid data seems old
+
+First check the APKs uploaded dates from [Nimbledroid](https://nimbledroid.com/my_apps). Hover over the apps to verify the match the package IDs (e.g. [org.mozilla.klar](https://nimbledroid.com/my_apps/org.mozilla.klar?a=2ab0db47-8e11-4be3-bd58-cfec06e225e9#summary).
+
+If everything seems fine, try to run the script as described in the section above to see if there's anything broken.
+
+If everything works, load the Nimbledroid APKs directly and inspect the output.
+
 ## Attributions
 
 - heartbeat icon by Creative Stall from the Noun Project

app.json

@@ -1,7 +1,7 @@
 {
   "name": "firefox-health-backend",
   "scripts": {
-    "postdeploy": "yarn fetchNimbledroidData"
+    "postdeploy": "DEBUG=script:info,script:error yarn fetchNimbledroidData"
   },
   "env": {
     "GOOGLE_API_KEY": {

package.json

@@ -26,6 +26,7 @@
   "license": "MPL-2.0",
   "dependencies": {
     "async-redis": "^1.1.4",
+    "debug": "^4.1.1",
     "googleapis": "^27.0.0",
     "isomorphic-fetch": "^2.2.1",
     "koa": "^2.3.0",

src/scripts/fetchNimbledroidData.js

@@ -1,6 +1,11 @@
+import debug from 'debug';
 import asyncRedis from 'async-redis';
 import NimbledroidClient from '../utils/NimbledroidClient';
 
+const debugLog = debug('script:debug');
+const infoLog = debug('script:info');
+const errorLog = debug('script:error');
+
 if (
   !process.env.REDIS_URL ||
   !process.env.NIMBLEDROID_API_KEY ||
@@ -16,7 +21,7 @@ const nimbledroidClient = new NimbledroidClient(
 const redisClient = asyncRedis.createClient(process.env.REDIS_URL);
 
 redisClient.on('error', (err) => {
-  console.error(err);
+  errorLog(err);
 });
 
 // eslint-disable-next-line consistent-return
@@ -33,10 +38,10 @@ const storeProfilingRunIfMissing = async (profilingRunData) => {
   if (status === 'Failed') {
     const cached = await redisClient.get(key);
     if (!cached) {
-      console.log(`Storing ${key}`);
+      debugLog(`Storing ${key}`);
       await redisClient.set(key, JSON.stringify(profilingRunData));
     } else {
-      console.log(`The key is already in the cache (${key})`);
+      debugLog(`The key is already in the cache (${key})`);
     }
   }
 };
@@ -51,22 +56,22 @@ const fetchData = async productName =>
 
 const main = async () => {
   let errorCode = -1;
-  console.log('Fetching each product can take between 20-40 seconds.');
+  debugLog('DEBUG output will be shown.');
+  infoLog('Fetching each product can take between 20-40 seconds.');
   try {
     await Promise.all([
       'org.mozilla.klar',
-      'org.mozilla.focus',
       'org.mozilla.geckoview_example',
       'com.chrome.beta',
     ].map(async (productName) => {
-      console.log(`Fetching ${productName}`);
+      infoLog(`Fetching ${productName}`);
       const productData = await fetchData(productName);
-      console.log(`Storing ${productName}`);
+      infoLog(`Storing ${productName}`);
       await storeDataInRedis(productData);
     }));
     errorCode = 0;
   } catch (e) {
-    console.error(e);
+    errorLog(e);
   } finally {
     process.exit(errorCode);
   }

src/utils/apis/queryNimbledroidData.js

@@ -19,7 +19,7 @@ const latest = (a, b) => {
   return latestVersion;
 };
 
-const moreData = (data, packageId) => {
+const processedData = (data, packageId) => {
   let latestVersion = ['0', '0', '0', '0'];
   const scenarios = {};
   // eslint-disable-next-line camelcase
@@ -45,32 +45,14 @@ const moreData = (data, packageId) => {
   };
 };
 
-const optimizedData = (data, packageId) => {
-  const newData = {};
-  data.forEach(({ added, profiles }) => {
-    // eslint-disable-next-line camelcase
-    profiles.forEach(({ scenario_name, time_in_ms }) => {
-      if (!newData[scenario_name]) {
-        newData[scenario_name] = [];
-      }
-      newData[scenario_name].push({
-        date: added,
-        ms: time_in_ms,
-      });
-    });
-  });
-  return { [packageId]: newData };
-};
-
-const queryNimbledroidData = async (product, version) => {
+const queryNimbledroidData = async (product) => {
   const cachedKeys = await client.keys(`cache:*nimble*${product}/apks/*`);
   const data = await Promise.all(cachedKeys.map(async key =>
     JSON.parse(await client.get(key))));
   if (data.length === 0) {
     throw Error('The script that fetches data should have run before hitting the API.');
   }
-  // Drop support for version 1
-  return version === '3' ? moreData(data, product) : optimizedData(data, product);
+  return processedData(data, product);
 };
 
 export default queryNimbledroidData;

yarn.lock

@@ -2025,6 +2025,13 @@ debug@2.6.9, debug@^2.1.2, debug@^2.2.0, debug@^2.3.3, debug@^2.6.6, debug@^2.6.
   dependencies:
     ms "2.0.0"
 
+debug@^4.1.1:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/debug/-/debug-4.1.1.tgz#3b72260255109c6b589cee050f1d516139664791"
+  integrity sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==
+  dependencies:
+    ms "^2.1.1"
+
 decamelize@^1.0.0, decamelize@^1.1.1, decamelize@^1.1.2:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
@@ -4975,6 +4982,11 @@ ms@2.0.0:
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
   integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=
 
+ms@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a"
+  integrity sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==
+
 multicast-dns-service-types@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/multicast-dns-service-types/-/multicast-dns-service-types-1.1.0.tgz#899f11d9686e5e05cb91b35d5f0e63b773cfc901"