handle benched tokens

lib/githubFetcher.js

@@ -36,7 +36,14 @@ class GitHubFetcher {
     const self = this;
     const etagPromise = checkEtag ? this.store.etag(request.type, request.url) : Q(null);
     return etagPromise.then(etag => {
-      return this._getFromGitHub(request, etag).then(githubResponse => {
+      let options = self._addTokenOption(request, { headers: {} });
+      if (typeof options === 'number') {
+        // if we get back a number, all tokens have been benched so we have to requeue and wait.
+        return self._requeueBenched(request, options);
+      }
+      options = self._addEtagOption(options, etag);
+      options = self._addTypeOptions(request, options);
+      return this._getFromGitHub(request, options).then(githubResponse => {
         const status = githubResponse.statusCode;
         if (status !== 200 && status !== 304) {
           if (status === 409 || status === 204) {
@@ -46,11 +53,7 @@ class GitHubFetcher {
           // and wait a bit before processing more requests
           const remaining = parseInt(githubResponse.headers['x-ratelimit-remaining'], 10) || 0;
           if (status === 403 && remaining === 0) {
-            const delay = self.options.forbiddenDelay || 120000;
-            request.exhaustToken(Date.now() + delay);
-            request.delay(delay);
-            request.addMeta({ forbiddenDelay: delay });
-            return request.markRequeue('Throttled', `GitHub throttled ${request.url}`);
+            return self._requeueThrottled(request);
           }
           throw new Error(`Code ${status} for ${request.url}`);
         }
@@ -74,11 +77,7 @@ class GitHubFetcher {
     });
   }
 
-  _getFromGitHub(request, etag) {
-    const options = this._getOptions(request);
-    if (etag) {
-      options.headers['If-None-Match'] = etag;
-    }
+  _getFromGitHub(request, options) {
     const start = Date.now();
     const deferred = Q.defer();
     this.fetchQueue.push({ url: request.url, options: options }, (error, response) => {
@@ -100,13 +99,29 @@ class GitHubFetcher {
     try {
       this._incrementMetric('fetch');
       this.requestor.get(spec.url, spec.options).then(
-        response => callback(null, response),
-        error => callback(error));
+        response =>
+          callback(null, response),
+        error =>
+          callback(error));
     } catch (e) {
       callback(e);
     }
   }
 
+  _requeueBenched(request, benchTime) {
+    request.delayUntil(benchTime);
+    request.addMeta({ benchDelay: benchTime - Date.now() });
+    return request.markRequeue('Benched', `Wait for token while getting ${request.url}`);
+  }
+
+  _requeueThrottled(request) {
+    const delay = this.options.forbiddenDelay || 120000;
+    request.exhaustToken(Date.now() + delay);
+    request.delay(delay);
+    request.addMeta({ forbiddenDelay: delay });
+    return request.markRequeue('Throttled', `GitHub throttled ${request.url}`);
+  }
+
   _fetchFromStorage(request) {
     const start = Date.now();
     return this.store.get(request.type, request.url).then(
@@ -138,7 +153,7 @@ class GitHubFetcher {
     // This code is not designed to handle the 403 scenarios.  That is handled by the retry logic.
     const remaining = parseInt(response.headers['x-ratelimit-remaining'], 10) || 0;
     request.addMeta({ remaining: remaining });
-    const tokenLowerBound = this.options ? (this.options.tokenLowerBound || 50) : 50;
+    const tokenLowerBound = this.options.tokenLowerBound || 50;
     if (remaining < tokenLowerBound) {
       const reset = parseInt(response.headers['x-ratelimit-reset'], 10) || 0;
       const delay = Math.max(0, reset - Date.now());
@@ -170,23 +185,15 @@ class GitHubFetcher {
     return request;
   }
 
-  _getOptions(request) {
-    const result = { headers: {} };
-    const token = this._getToken(request);
-    if (token) {
-      result.headers.authorization = `token ${token}`;
-      request.exhaustToken = until => this.tokenFactory.exhaust(token, until);
-    } else {
-      throw new Error(`No API tokens available for ${request.toString()}`);
+
+  _addEtagOption(options, etag) {
+    if (etag) {
+      options.headers['If-None-Match'] = etag;
     }
-    const headers = this._getHeaders(request);
-    if (headers) {
-      Object.assign(result.headers, headers);
-    }
-    return result;
+    return options;
   }
 
-  _getToken(request) {
+  _addTokenOption(request, options) {
     const traits = this._getTypeDetails(request.type).tokenTraits || [];
     const additionalTraits = [];
     if (request.context.repoType) {
@@ -196,12 +203,25 @@ class GitHubFetcher {
       // if this is a retry, elevate the token to avoid any permissions issues
       additionalTraits.push('private', 'admin');
     }
-    return this.tokenFactory.getToken(traits.concat(additionalTraits));
+    const token = this.tokenFactory.getToken(traits.concat(additionalTraits));
+    if (!token) {
+      throw new Error(`No API tokens available for ${request.toString()}`);
+    }
+    if (typeof token === 'number') {
+      return token;
+    }
+    options.headers.authorization = `token ${token}`;
+    request.exhaustToken = until => this.tokenFactory.exhaust(token, until);
+    return options;
   }
 
-  _getHeaders(request) {
+  _addTypeOptions(request, options) {
     const typeDetails = this._getTypeDetails(request.type);
-    return typeDetails.headers;
+    const headers = typeDetails.headers;
+    if (headers) {
+      Object.assign(options.headers, headers);
+    }
+    return options;
   }
 
   _incrementMetric(operation) {