diff --git a/lib/githubFetcher.js b/lib/githubFetcher.js index 2743375..fa31de1 100644 --- a/lib/githubFetcher.js +++ b/lib/githubFetcher.js @@ -45,6 +45,8 @@ class GitHubFetcher { // and wait a bit before processing more requests const remaining = parseInt(response.headers['x-ratelimit-remaining'], 10) || 0; if (status === 403 && remaining === 0) { + request.addMeta({ token: `${token.slice(0, 2)}..${token.slice(-2)}` }); + request.addMeta({ requestId: `${response.headers['x-github-request-id']}` }); return self._requeueThrottled(request); } // If response indicates something that may be related to Auth, note the token used and bail if @@ -151,7 +153,7 @@ class GitHubFetcher { request.addMeta({ remaining: remaining }); const tokenLowerBound = this.options.tokenLowerBound || 50; if (remaining < tokenLowerBound) { - const reset = parseInt(response.headers['x-ratelimit-reset'], 10) || 0; + const reset = (parseInt(response.headers['x-ratelimit-reset'], 10) || 0) * 1000; const delay = Math.max(0, reset - Date.now()); if (delay > 0) { request.addMeta({ backoffDelay: delay }); diff --git a/lib/githubProcessor.js b/lib/githubProcessor.js index 79aafb1..000934e 100644 --- a/lib/githubProcessor.js +++ b/lib/githubProcessor.js @@ -61,7 +61,8 @@ class GitHubProcessor { const links = parse(linkHeader); const requests = []; for (let i = 2; i <= links.last.page; i++) { - const url = request.url + `?page=${i}&per_page=100`; + const separator = request.url.includes('?') ? '&' : '?'; + const url = request.url + `${separator}page=${i}&per_page=100`; const newRequest = new Request(request.type, url, request.context, request.relationship); // Carry this request's transitivity forward to the other pages. newRequest.policy = request.policy; @@ -181,7 +182,7 @@ class GitHubProcessor { if (document.subscribers_count) { this._addRelation(request, 'subscribers', 'user'); } - this._addCollection(request, 'issues', 'issue', document.issues_url.replace('{/number}', '')); + this._addCollection(request, 'issues', 'issue', document.issues_url.replace('{/number}', '?state=all')); this._addCollection(request, 'commits', 'commit', document.commits_url.replace('{/sha}', '')); this._addCollection(request, 'events', null);