Script to fetch Nimbledroid data

This script will cache all Nimbledroid data for focus and klar in Redis.

The two fetches are only cached for 30 minutes since the number of code
landing for focus/klar is approximately that. If Nimbledroid's profiling runs
throughput at the same speed we should get the most recent data.

In the future, the API will have pagination (about 10 runs) and will give us
the number of the most recent build. The code in this patch is already taken
some of that into account and will make switching to it require less development
work.

Only profiling runs that are completed (marked as "Failed") are cached indefinitively.

.neutrinorc.js

@@ -1,4 +1,10 @@
 module.exports = {
+  options: {
+    mains: {
+      index: 'index',
+      nimbledroid: 'scripts/fetchNimbledroidData.js',
+    }
+  },
   use: [
     [
       '@neutrinojs/airbnb-base',

README.md

@@ -18,7 +18,7 @@ yarn // To get the dependencies installed
 yarn start // To start the server
 ```
 
-### Providing a Google API key
+### Enable access to the Google status spreadsheet
 
 The [notes](http://localhost:3000/api/perf/notes) API requires a `GOOGLE_API_KEY`
 in order to access a Google Spreadsheet. In order for this API to work locally
@@ -39,7 +39,7 @@ GOOGLE_API_KEY=<created API key> yarn start
 ```
 * Visit http://localhost:3000/api/perf/notes to verify it works
 
-### Providing a Nimbledroid API key
+### Enable access to Nimbledroid's data
 Nimbledroid provides us with performance data for various sites on Android.
 If you want to make changes to the Nimbledroid APIs on the backend you will need
 to have access to our corporate Nimbledroid account.
@@ -52,10 +52,15 @@ Once you have it you can start the backend like this:
 ```
 export NIMBLEDROID_API_KEY=<API key>
 export NIMBLEDROID_EMAIL=<your email address>
+yarn fetchNimbledroidData
 yarn start
 ```
 
-Load http://localhost:3000/api/nimbledroid to verify it works.
+Load [this page](http://localhost:3000/api/nimbledroid?product=focus) to verify it works.
+
+### Redis
+
+If you want to test caching with Redis (there's caching with JS as a fallback) make sure to install Redis and set the REDIS_URL env to `redis://localhost:6379` before starting the server.
 
 ## Attributions
 

package.json

@@ -14,7 +14,8 @@
     "start:prod": "neutrino build && node .",
     "start:debugger": "neutrino build && node --inspect .",
     "precommit": "lint-staged",
-    "heroku-postbuild": "npm run build"
+    "heroku-postbuild": "npm run build",
+    "fetchNimbledroidData": "node build/nimbledroid.js"
   },
   "lint-staged": {
     "*.js": [

src/scripts/fetchNimbledroidData.js

@@ -0,0 +1,65 @@
+import asyncRedis from 'async-redis';
+import NimbledroidClient from '../utils/NimbledroidClient';
+
+if (
+  !process.env.REDIS_URL ||
+  !process.env.NIMBLEDROID_API_KEY ||
+  !process.env.NIMBLEDROID_EMAIL
+) {
+  throw Error('You need to set NIMBLEDROID_EMAIL, NIMBLEDROID_API_KEY and REDIS_URL');
+}
+
+const nimbledroidClient = new NimbledroidClient(
+  process.env.NIMBLEDROID_EMAIL,
+  process.env.NIMBLEDROID_API_KEY,
+);
+const redisClient = asyncRedis.createClient(process.env.REDIS_URL);
+
+redisClient.on('error', (err) => {
+  console.error(err);
+});
+
+// eslint-disable-next-line consistent-return
+const storeProfilingRunIfMissing = async (profilingRunData) => {
+  const KNOWN_STATUS = ['Crawling', 'Failed', 'Profiling', 'Profiled'];
+  const { status, url } = profilingRunData;
+  if (!KNOWN_STATUS.includes(status)) {
+    throw Error(`Status: ${status} is new to us; Handle it in the code.`);
+  }
+
+  // e.g. cache:https://nimbledroid.com/api/v2/users/npark@mozilla.com/apps/org.mozilla.klar/apks/103
+  const key = `cache:${url}`;
+  // The status 'Failed' means 'completed' in the Nimbledroid API
+  if (status === 'Failed') {
+    const cached = await redisClient.get(key);
+    if (!cached) {
+      console.log(`Storing ${key}`);
+      await redisClient.set(key, JSON.stringify(profilingRunData));
+    }
+  }
+};
+
+const storeDataInRedis = async (data) => {
+  await Promise.all(Object.keys(data).map(index =>
+    storeProfilingRunIfMissing(data[index])));
+};
+
+const fetchData = async productName =>
+  nimbledroidClient.getNimbledroidData(productName);
+
+const main = async () => {
+  console.log('Fetching each product can take between 20-40 seconds.');
+  try {
+    await Promise.all(['klar', 'focus'].map(async (productName) => {
+      console.log(`Fetching ${productName}`);
+      const productData = await fetchData(productName);
+      await storeDataInRedis(productData);
+    }));
+  } catch (e) {
+    console.error(e);
+  } finally {
+    process.exit();
+  }
+};
+
+main();

src/utils/NimbledroidClient/index.js

@@ -31,7 +31,11 @@ class NimbledroidHandler {
   async fetchData(product) {
     return fetchJson(
       apiUrl(product),
-      { method: 'GET', headers: this.generateAuthHeaders() },
+      {
+        method: 'GET',
+        headers: this.generateAuthHeaders(),
+        ttl: 30 * 60, // 30 minutes
+      },
     );
   }