Fix paths

app.js

@@ -5,7 +5,7 @@ const appInsights = require('applicationinsights');
 const auth = require('./middleware/auth');
 const bodyParser = require('body-parser');
 const config = require('painless-config');
-const CrawlerService = require('ghcrawler').crawlerService;
+const CrawlerService = require('./index').crawlerService;
 const express = require('express');
 const logger = require('morgan');
 const mockInsights = require('./providers/logger/mockInsights');

index.js

@@ -4,9 +4,9 @@
 module.exports.crawler = require('./lib/crawler');
 module.exports.crawlerService = require('./lib/crawlerService');
 module.exports.eventFinder = require('./lib/eventFinder');
-module.exports.githubFetcher = require('./lib/githubFetcher');
-module.exports.githubProcessor = require('./lib/githubProcessor');
+module.exports.githubFetcher = require('./providers/fetcher/githubFetcher');
+module.exports.githubProcessor = require('./providers/fetcher/githubProcessor');
 module.exports.policy = require('./lib/traversalPolicy');
-module.exports.queueSet = require('./lib/queueSet');
+module.exports.queueSet = require('./providers/queuing/queueSet');
 module.exports.request = require('./lib/request');
 module.exports.traversalPolicy = require('./lib/traversalPolicy');

lib/crawlerFactory.js

@@ -22,8 +22,8 @@ const appInsights = require('applicationinsights');
 const aiLogger = require('winston-azure-application-insights').AzureApplicationInsightsLogger;
 const AzureStorage = require('azure-storage');
 const config = require('painless-config');
-const Crawler = require('ghcrawler').crawler;
-const CrawlerService = require('ghcrawler').crawlerService;
+const Crawler = require('../index').crawler;
+const CrawlerService = require('../index').crawlerService;
 const fs = require('fs');
 const GitHubFetcher = require('../index').githubFetcher;
 const GitHubProcessor = require('../index').githubProcessor;

package.json

@@ -43,7 +43,6 @@
     "express": "^4.14.0",
     "express-init": "^1.1.0",
     "express-joi": "^0.3.1",
-    "ghcrawler": "0.2.22",
     "ghrequestor": "^0.1.6",
     "htmlencode": "0.0.4",
     "ip": "^1.1.4",

providers/fetcher/githubProcessor.js

@@ -3,10 +3,10 @@
 
 const moment = require('moment');
 const parse = require('parse-link-header');
-const Request = require('./request');
+const Request = require('../../lib/request');
 const Q = require('q');
 const qlimit = require('qlimit');
-const TraversalPolicy = require('./traversalPolicy');
+const TraversalPolicy = require('../../lib/traversalPolicy');
 const URL = require('url');
 const uuid = require('node-uuid');
 

providers/queuing/inmemorycrawlqueue.js

@@ -3,7 +3,7 @@
 
 const extend = require('extend');
 const Q = require('q');
-const Request = require('ghcrawler').request;
+const Request = require('../../index').request;
 
 class InMemoryCrawlQueue {
   constructor(name, options) {

providers/queuing/rabbitQueueManager.js

@@ -5,7 +5,7 @@ const AmqpQueue = require('./amqpQueue');
 const AttenuatedQueue = require('./attenuatedQueue');
 const Q = require('q');
 const request = require('request');
-const Request = require('ghcrawler').request;
+const Request = require('../../index').request;
 const TrackedQueue = require('./trackedQueue');
 
 class RabbitQueueManager {

providers/queuing/serviceBusQueueManager.js

@@ -7,7 +7,7 @@ const AttenuatedQueue = require('./attenuatedQueue');
 const azureCommon = require('azure-common');
 const InMemoryRateLimiter = require('../limiting/inmemoryRateLimiter');
 const RateLimitedPushQueue = require('./ratelimitedPushQueue');
-const Request = require('ghcrawler').request;
+const Request = require('../../index').request;
 const serviceBus = require('azure-sb');
 const TrackedQueue = require('./trackedQueue');
 const Q = require('q');

routes/deadletters.js

@@ -4,7 +4,7 @@
 const auth = require('../middleware/auth');
 const express = require('express');
 const expressJoi = require('express-joi');
-const Request = require('ghcrawler').request;
+const Request = require('../index').request;
 const wrap = require('../middleware/promiseWrap');
 
 let crawlerService = null;

routes/requests.js

@@ -4,8 +4,8 @@
 const auth = require('../middleware/auth');
 const express = require('express');
 const expressJoi = require('express-joi');
-const Request = require('ghcrawler').request;
-const TraversalPolicy = require('ghcrawler').traversalPolicy;
+const Request = require('../index').request;
+const TraversalPolicy = require('../index').traversalPolicy;
 const wrap = require('../middleware/promiseWrap');
 
 const requestsSchema = {

routes/webhook.js

@@ -4,7 +4,7 @@
 const crypto = require('crypto');
 const express = require('express');
 const moment = require('moment');
-const Request = require('ghcrawler').request;
+const Request = require('../index').request;
 const wrap = require('../middleware/promiseWrap');
 
 let crawlerService = null;
@@ -15,7 +15,7 @@ router.post('/', wrap(function* (request, response, next) {
   if (crawlerService.options.queuing.events.provider !== 'webhook') {
     return warn(request, response, 'Webhooks not enabled');
   }
-  getLogger().verbose('Received', `Webhook event`, {delivery: request.headers['x-github-delivery']});
+  getLogger().verbose('Received', `Webhook event`, { delivery: request.headers['x-github-delivery'] });
   const signature = request.headers['x-hub-signature'];
   const eventType = request.headers['x-github-event'];
 
@@ -39,20 +39,20 @@ router.post('/', wrap(function* (request, response, next) {
     result.context.repoType = 'private';
   }
   yield crawlerService.queue(result, 'events');
-  getLogger().info('Queued', `Webhook event for ${eventsUrl}`, {delivery: request.headers['x-github-delivery']});
+  getLogger().info('Queued', `Webhook event for ${eventsUrl}`, { delivery: request.headers['x-github-delivery'] });
 
   response.status(200).end();
 }));
 
 function warn(request, response, message) {
-  getLogger().warn(fatal, { delivery: request.headers['x-github-delivery']});
+  getLogger().warn(fatal, { delivery: request.headers['x-github-delivery'] });
   response.status(500);
   response.setHeader('content-type', 'text/plain');
   response.end(JSON.stringify(fatal));
 }
 
 function fatal(request, response, error) {
-  getLogger().error(error, { delivery: request.headers['x-github-delivery']});
+  getLogger().error(error, { delivery: request.headers['x-github-delivery'] });
   response.status(400);
   response.setHeader('content-type', 'text/plain');
   response.end(JSON.stringify(error));

test/integration/amqp10QueueTests.js

@@ -6,7 +6,7 @@ const config = require('painless-config');
 const expect = require('chai').expect;
 const CrawlerFactory = require('../../lib/crawlerFactory');
 const Q = require('q');
-const Request = require('ghcrawler').request;
+const Request = require('../../index').request;
 
 const url = config.get('CRAWLER_AMQP10_URL'); // URL should be: amqps://<keyName>:<key>@<host>
 const name = 'test';

test/unit/crawlerTests.js

@@ -3,16 +3,16 @@
 
 const assert = require('chai').assert;
 const chai = require('chai');
-const Crawler = require('../lib/crawler');
+const Crawler = require('../../lib/crawler');
 const expect = require('chai').expect;
 const extend = require('extend');
-const GitHubFetcher = require('../lib/githubFetcher');
-const GitHubProcessor = require('../lib/githubProcessor');
+const GitHubFetcher = require('../../providers/fetcher/githubFetcher');
+const GitHubProcessor = require('../../providers/fetcher/githubProcessor');
 const Q = require('q');
-const QueueSet = require('../lib/queueSet');
-const Request = require('../lib/request');
+const QueueSet = require('../../providers/queuing/queueSet');
+const Request = require('../../lib/request');
 const sinon = require('sinon');
-const TraversalPolicy = require('../lib/traversalPolicy');
+const TraversalPolicy = require('../../lib/traversalPolicy');
 
 describe('Crawler get request', () => {
   it('should get from the priority queue first', () => {
@@ -1074,7 +1074,7 @@ function createErrorResponse(error) {
   };
 }
 
-function createBaseCrawler({queues = createBaseQueues(), store = createBaseStore(), locker = createBaseLocker(), requestor = createBaseRequestor(), fetcher = null, options = createBaseOptions() } = {}) {
+function createBaseCrawler({ queues = createBaseQueues(), store = createBaseStore(), locker = createBaseLocker(), requestor = createBaseRequestor(), fetcher = null, options = createBaseOptions() } = {}) {
   if (!fetcher) {
     fetcher = createBaseFetcher();
   }
@@ -1119,11 +1119,11 @@ function createBaseOptions(logger = createBaseLog()) {
   return result;
 }
 
-function createBaseQueues({ priority = null, normal = null, deadletter = null, options = null} = {}) {
+function createBaseQueues({ priority = null, normal = null, deadletter = null, options = null } = {}) {
   return new QueueSet([priority || createBaseQueue('priority'), normal || createBaseQueue('normal')], deadletter || createBaseQueue('deadletter'), (options || createBaseOptions()).queuing);
 }
 
-function createBaseQueue(name, { pop = null, push = null, done = null, abandon = null} = {}) {
+function createBaseQueue(name, { pop = null, push = null, done = null, abandon = null } = {}) {
   const result = { name: name };
   result.getName = () => { return name; };
   result.pop = pop || (() => assert.fail('should not pop'));
@@ -1133,7 +1133,7 @@ function createBaseQueue(name, { pop = null, push = null, done = null, abandon =
   return result;
 }
 
-function createBaseStore({etag = null, upsert = null, get = null} = {}) {
+function createBaseStore({ etag = null, upsert = null, get = null } = {}) {
   const result = {};
   result.etag = etag || (() => { assert.fail('should not etag'); });
   result.upsert = upsert || (() => { assert.fail('should not upsert'); });
@@ -1141,7 +1141,7 @@ function createBaseStore({etag = null, upsert = null, get = null} = {}) {
   return result;
 }
 
-function createBaseLog({log = null, info = null, warn = null, error = null, verbose = null, silly = null} = {}) {
+function createBaseLog({ log = null, info = null, warn = null, error = null, verbose = null, silly = null } = {}) {
   const result = {};
   result.log = log || (() => { });
   result.info = info || (() => { });
@@ -1153,7 +1153,7 @@ function createBaseLog({log = null, info = null, warn = null, error = null, verb
   return result;
 }
 
-function createBaseLocker({lock = null, unlock = null} = {}) {
+function createBaseLocker({ lock = null, unlock = null } = {}) {
   const result = {};
   result.lock = lock || (() => assert.fail('should not lock'));
   result.unlock = unlock || (() => assert.fail('should not unlock'));

test/unit/eventFinderTests.js

@@ -3,7 +3,7 @@
 
 const expect = require('chai').expect;
 const extend = require('extend');
-const finder = require('../lib/eventFinder.js');
+const finder = require('../../lib/eventFinder.js');
 const Q = require('q');
 
 describe('Event Finder', () => {

test/unit/gitHubProcessorTests.js

@@ -2,11 +2,11 @@
 // Licensed under the MIT License.
 
 const expect = require('chai').expect;
-const GitHubProcessor = require('../lib/githubProcessor.js');
+const GitHubProcessor = require('../../providers/fetcher/githubProcessor.js');
 const Q = require('q');
-const Request = require('../lib/request.js');
+const Request = require('../../lib/request.js');
 const sinon = require('sinon');
-const TraversalPolicy = require('../lib/traversalPolicy');
+const TraversalPolicy = require('../../lib/traversalPolicy');
 
 describe('GitHubProcessor reprocessing', () => {
   it('will skip if at same version', () => {
@@ -435,7 +435,7 @@ describe('Pull Request processing', () => {
       head: { repo: { id: 45, url: 'http://repo/45' } },
       base: { repo: { id: 17, url: 'http://repo/17' } },
       _links: {
-        self: { href: 'http://pull_request/13'},
+        self: { href: 'http://pull_request/13' },
         issue: { href: 'http://issue/13' },
         review_comments: { href: 'http://review_comments' },
         commits: { href: 'http://commits' },
@@ -622,7 +622,7 @@ describe('Pull request/review comment processing', () => {
     // test that the new request got queued and that the doc has the right stuff
 
     const newRequest = queue.pop();
-    newRequest.document = {id: 1, url: 'http://pull_request/1'}
+    newRequest.document = { id: 1, url: 'http://pull_request/1' }
     testPullRequestReviewCommentEvent(newRequest, 'LegacyPullRequestReviewCommentEvent');
   });
 });

test/unit/githubFetcherTests.js

@@ -3,15 +3,15 @@
 
 const assert = require('chai').assert;
 const chai = require('chai');
-const Crawler = require('../lib/crawler');
+const Crawler = require('../../lib/crawler');
 const expect = require('chai').expect;
 const extend = require('extend');
-const GitHubFetcher = require('../lib/githubFetcher');
+const GitHubFetcher = require('../../providers/fetcher/githubFetcher');
 const Q = require('q');
-const QueueSet = require('../lib/queueSet');
-const Request = require('../lib/request');
+const QueueSet = require('../../providers/queuing/queueSet');
+const Request = require('../../lib/request');
 const sinon = require('sinon');
-const TraversalPolicy = require('../lib/traversalPolicy');
+const TraversalPolicy = require('../../lib/traversalPolicy');
 const URL = require('url');
 
 describe('GitHub fetcher', () => {
@@ -250,7 +250,7 @@ function createResponse(body, code = 200, etag = null, remaining = 4000, reset =
   };
 }
 
-function createBaseStore({etag = null, upsert = null, get = null} = {}) {
+function createBaseStore({ etag = null, upsert = null, get = null } = {}) {
   const result = {};
   result.etag = etag || (() => { assert.fail('should not etag'); });
   result.upsert = upsert || (() => { assert.fail('should not upsert'); });
@@ -296,7 +296,7 @@ function createBaseOptions(logger = createBaseLog()) {
   };
 }
 
-function createBaseLog({info = null, warn = null, error = null, verbose = null, silly = null} = {}) {
+function createBaseLog({ info = null, warn = null, error = null, verbose = null, silly = null } = {}) {
   const result = {};
   result.info = info || (() => { });
   result.warn = warn || (() => { });

test/unit/processingTests.js

@@ -4,7 +4,7 @@
 const expect = require('chai').expect;
 const CrawlerFactory = require('../../lib/crawlerFactory');
 const Q = require('q');
-const Request = require('ghcrawler').request;
+const Request = require('../../index').request;
 const sinon = require('sinon');
 
 let crawler = null;

test/unit/queueSetTests.js

@@ -3,9 +3,9 @@
 
 const assert = require('chai').assert;
 const expect = require('chai').expect;
-const Request = require('../lib/request.js');
+const Request = require('../../lib/request.js');
 const Q = require('q');
-const QueueSet = require('../lib/queueSet.js');
+const QueueSet = require('../../providers/queuing/queueSet.js');
 const sinon = require('sinon');
 
 describe('QueueSet construction', () => {
@@ -253,7 +253,7 @@ function createBaseQueues(queues, deadletter, weights = [1]) {
   return new QueueSet(queues, deadletter || createBaseQueue('deadletter'), createOptions(weights));
 }
 
-function createBaseQueue(name, { pop = null, push = null, done = null, abandon = null, subscribe = null, unsubscribe = null} = {}) {
+function createBaseQueue(name, { pop = null, push = null, done = null, abandon = null, subscribe = null, unsubscribe = null } = {}) {
   const result = { name: name };
   result.getName = () => { return name; };
   result.pop = pop || (() => assert.fail('should not pop'));

test/unit/requestTests.js

@@ -3,9 +3,9 @@
 
 const assert = require('chai').assert;
 const expect = require('chai').expect;
-const Request = require('../lib/request.js');
+const Request = require('../../lib/request.js');
 const sinon = require('sinon');
-const TraversalPolicy = require('../lib/traversalPolicy');
+const TraversalPolicy = require('../../lib/traversalPolicy');
 
 describe('Request context/qualifier', () => {
   it('will not queueRoot if none transitivity', () => {

test/unit/requestTrackerTests.js

@@ -7,7 +7,7 @@ const expect = require('chai').expect;
 const extend = require('extend');
 const Q = require('q');
 const redlock = require('redlock');
-const Request = require('ghcrawler').request;
+const Request = require('../../index').request;
 const RequestTracker = require('../../providers/queuing/redisRequestTracker.js');
 const sinon = require('sinon');
 

test/unit/visitorMapTests.js

@@ -4,7 +4,7 @@
 const assert = require('chai').assert;
 const expect = require('chai').expect;
 const sinon = require('sinon');
-const VisitorMap = require('../lib/visitorMap');
+const VisitorMap = require('../../lib/visitorMap');
 
 describe('Visitor Map', () => {