From bf890ec707911c2e2b1e319f75c92cac4335e7c0 Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Tue, 17 Jul 2018 16:18:39 -0400 Subject: [PATCH] Script to fetch Nimbledroid data This script will cache all Nimbledroid data for focus and klar in Redis. The two fetches are only cached for 30 minutes since the number of code landing for focus/klar is approximately that. If Nimbledroid's profiling runs throughput at the same speed we should get the most recent data. In the future, the API will have pagination (about 10 runs) and will give us the number of the most recent build. The code in this patch is already taken some of that into account and will make switching to it require less development work. Only profiling runs that are completed (marked as "Failed") are cached indefinitively. --- .neutrinorc.js | 6 +++ README.md | 11 +++-- package.json | 3 +- src/scripts/fetchNimbledroidData.js | 65 ++++++++++++++++++++++++++++ src/utils/NimbledroidClient/index.js | 6 ++- 5 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 src/scripts/fetchNimbledroidData.js diff --git a/.neutrinorc.js b/.neutrinorc.js index aec1d34..8e8cb5c 100644 --- a/.neutrinorc.js +++ b/.neutrinorc.js @@ -1,4 +1,10 @@ module.exports = { + options: { + mains: { + index: 'index', + nimbledroid: 'scripts/fetchNimbledroidData.js', + } + }, use: [ [ '@neutrinojs/airbnb-base', diff --git a/README.md b/README.md index dc21209..51fd844 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ yarn // To get the dependencies installed yarn start // To start the server ``` -### Providing a Google API key +### Enable access to the Google status spreadsheet The [notes](http://localhost:3000/api/perf/notes) API requires a `GOOGLE_API_KEY` in order to access a Google Spreadsheet. In order for this API to work locally @@ -39,7 +39,7 @@ GOOGLE_API_KEY= yarn start ``` * Visit http://localhost:3000/api/perf/notes to verify it works -### Providing a Nimbledroid API key +### Enable access to Nimbledroid's data Nimbledroid provides us with performance data for various sites on Android. If you want to make changes to the Nimbledroid APIs on the backend you will need to have access to our corporate Nimbledroid account. @@ -52,10 +52,15 @@ Once you have it you can start the backend like this: ``` export NIMBLEDROID_API_KEY= export NIMBLEDROID_EMAIL= +yarn fetchNimbledroidData yarn start ``` -Load http://localhost:3000/api/nimbledroid to verify it works. +Load [this page](http://localhost:3000/api/nimbledroid?product=focus) to verify it works. + +### Redis + +If you want to test caching with Redis (there's caching with JS as a fallback) make sure to install Redis and set the REDIS_URL env to `redis://localhost:6379` before starting the server. ## Attributions diff --git a/package.json b/package.json index fa76a8a..cd538f8 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,8 @@ "start:prod": "neutrino build && node .", "start:debugger": "neutrino build && node --inspect .", "precommit": "lint-staged", - "heroku-postbuild": "npm run build" + "heroku-postbuild": "npm run build", + "fetchNimbledroidData": "node build/nimbledroid.js" }, "lint-staged": { "*.js": [ diff --git a/src/scripts/fetchNimbledroidData.js b/src/scripts/fetchNimbledroidData.js new file mode 100644 index 0000000..fde5946 --- /dev/null +++ b/src/scripts/fetchNimbledroidData.js @@ -0,0 +1,65 @@ +import asyncRedis from 'async-redis'; +import NimbledroidClient from '../utils/NimbledroidClient'; + +if ( + !process.env.REDIS_URL || + !process.env.NIMBLEDROID_API_KEY || + !process.env.NIMBLEDROID_EMAIL +) { + throw Error('You need to set NIMBLEDROID_EMAIL, NIMBLEDROID_API_KEY and REDIS_URL'); +} + +const nimbledroidClient = new NimbledroidClient( + process.env.NIMBLEDROID_EMAIL, + process.env.NIMBLEDROID_API_KEY, +); +const redisClient = asyncRedis.createClient(process.env.REDIS_URL); + +redisClient.on('error', (err) => { + console.error(err); +}); + +// eslint-disable-next-line consistent-return +const storeProfilingRunIfMissing = async (profilingRunData) => { + const KNOWN_STATUS = ['Crawling', 'Failed', 'Profiling', 'Profiled']; + const { status, url } = profilingRunData; + if (!KNOWN_STATUS.includes(status)) { + throw Error(`Status: ${status} is new to us; Handle it in the code.`); + } + + // e.g. cache:https://nimbledroid.com/api/v2/users/npark@mozilla.com/apps/org.mozilla.klar/apks/103 + const key = `cache:${url}`; + // The status 'Failed' means 'completed' in the Nimbledroid API + if (status === 'Failed') { + const cached = await redisClient.get(key); + if (!cached) { + console.log(`Storing ${key}`); + await redisClient.set(key, JSON.stringify(profilingRunData)); + } + } +}; + +const storeDataInRedis = async (data) => { + await Promise.all(Object.keys(data).map(index => + storeProfilingRunIfMissing(data[index]))); +}; + +const fetchData = async productName => + nimbledroidClient.getNimbledroidData(productName); + +const main = async () => { + console.log('Fetching each product can take between 20-40 seconds.'); + try { + await Promise.all(['klar', 'focus'].map(async (productName) => { + console.log(`Fetching ${productName}`); + const productData = await fetchData(productName); + await storeDataInRedis(productData); + })); + } catch (e) { + console.error(e); + } finally { + process.exit(); + } +}; + +main(); diff --git a/src/utils/NimbledroidClient/index.js b/src/utils/NimbledroidClient/index.js index 6b7f9cd..5b82c86 100644 --- a/src/utils/NimbledroidClient/index.js +++ b/src/utils/NimbledroidClient/index.js @@ -31,7 +31,11 @@ class NimbledroidHandler { async fetchData(product) { return fetchJson( apiUrl(product), - { method: 'GET', headers: this.generateAuthHeaders() }, + { + method: 'GET', + headers: this.generateAuthHeaders(), + ttl: 30 * 60, // 30 minutes + }, ); }