From b6c3aa03ba99ae9fdea55ad9a2c49a84223e0e9e Mon Sep 17 00:00:00 2001 From: William Bartholomew Date: Thu, 9 Mar 2017 16:44:45 -0800 Subject: [PATCH] Fix paths --- app.js | 2 +- index.js | 6 ++--- lib/crawlerFactory.js | 4 ++-- package.json | 1 - {lib => providers/fetcher}/githubFetcher.js | 0 {lib => providers/fetcher}/githubProcessor.js | 4 ++-- providers/queuing/inmemorycrawlqueue.js | 2 +- {lib => providers/queuing}/queueSet.js | 0 providers/queuing/rabbitQueueManager.js | 2 +- providers/queuing/serviceBusQueueManager.js | 2 +- routes/deadletters.js | 2 +- routes/requests.js | 4 ++-- routes/webhook.js | 10 ++++---- test/integration/amqp10QueueTests.js | 2 +- test/unit/crawlerTests.js | 24 +++++++++---------- test/unit/eventFinderTests.js | 2 +- test/unit/gitHubProcessorTests.js | 10 ++++---- test/unit/githubFetcherTests.js | 14 +++++------ test/unit/processingTests.js | 2 +- test/unit/queueSetTests.js | 6 ++--- test/unit/requestTests.js | 4 ++-- test/unit/requestTrackerTests.js | 2 +- test/unit/visitorMapTests.js | 2 +- 23 files changed, 53 insertions(+), 54 deletions(-) rename {lib => providers/fetcher}/githubFetcher.js (100%) rename {lib => providers/fetcher}/githubProcessor.js (99%) rename {lib => providers/queuing}/queueSet.js (100%) diff --git a/app.js b/app.js index 5f26ddb..eea7e2d 100644 --- a/app.js +++ b/app.js @@ -5,7 +5,7 @@ const appInsights = require('applicationinsights'); const auth = require('./middleware/auth'); const bodyParser = require('body-parser'); const config = require('painless-config'); -const CrawlerService = require('ghcrawler').crawlerService; +const CrawlerService = require('./index').crawlerService; const express = require('express'); const logger = require('morgan'); const mockInsights = require('./providers/logger/mockInsights'); diff --git a/index.js b/index.js index ab6f8dc..a0e6dec 100644 --- a/index.js +++ b/index.js @@ -4,9 +4,9 @@ module.exports.crawler = require('./lib/crawler'); module.exports.crawlerService = require('./lib/crawlerService'); module.exports.eventFinder = require('./lib/eventFinder'); -module.exports.githubFetcher = require('./lib/githubFetcher'); -module.exports.githubProcessor = require('./lib/githubProcessor'); +module.exports.githubFetcher = require('./providers/fetcher/githubFetcher'); +module.exports.githubProcessor = require('./providers/fetcher/githubProcessor'); module.exports.policy = require('./lib/traversalPolicy'); -module.exports.queueSet = require('./lib/queueSet'); +module.exports.queueSet = require('./providers/queuing/queueSet'); module.exports.request = require('./lib/request'); module.exports.traversalPolicy = require('./lib/traversalPolicy'); diff --git a/lib/crawlerFactory.js b/lib/crawlerFactory.js index 14265bf..4601023 100644 --- a/lib/crawlerFactory.js +++ b/lib/crawlerFactory.js @@ -22,8 +22,8 @@ const appInsights = require('applicationinsights'); const aiLogger = require('winston-azure-application-insights').AzureApplicationInsightsLogger; const AzureStorage = require('azure-storage'); const config = require('painless-config'); -const Crawler = require('ghcrawler').crawler; -const CrawlerService = require('ghcrawler').crawlerService; +const Crawler = require('../index').crawler; +const CrawlerService = require('../index').crawlerService; const fs = require('fs'); const GitHubFetcher = require('../index').githubFetcher; const GitHubProcessor = require('../index').githubProcessor; diff --git a/package.json b/package.json index 5bc38a2..19e56ec 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,6 @@ "express": "^4.14.0", "express-init": "^1.1.0", "express-joi": "^0.3.1", - "ghcrawler": "0.2.22", "ghrequestor": "^0.1.6", "htmlencode": "0.0.4", "ip": "^1.1.4", diff --git a/lib/githubFetcher.js b/providers/fetcher/githubFetcher.js similarity index 100% rename from lib/githubFetcher.js rename to providers/fetcher/githubFetcher.js diff --git a/lib/githubProcessor.js b/providers/fetcher/githubProcessor.js similarity index 99% rename from lib/githubProcessor.js rename to providers/fetcher/githubProcessor.js index d6a9ece..ea03b9a 100644 --- a/lib/githubProcessor.js +++ b/providers/fetcher/githubProcessor.js @@ -3,10 +3,10 @@ const moment = require('moment'); const parse = require('parse-link-header'); -const Request = require('./request'); +const Request = require('../../lib/request'); const Q = require('q'); const qlimit = require('qlimit'); -const TraversalPolicy = require('./traversalPolicy'); +const TraversalPolicy = require('../../lib/traversalPolicy'); const URL = require('url'); const uuid = require('node-uuid'); diff --git a/providers/queuing/inmemorycrawlqueue.js b/providers/queuing/inmemorycrawlqueue.js index 145c28d..fd4b4e3 100644 --- a/providers/queuing/inmemorycrawlqueue.js +++ b/providers/queuing/inmemorycrawlqueue.js @@ -3,7 +3,7 @@ const extend = require('extend'); const Q = require('q'); -const Request = require('ghcrawler').request; +const Request = require('../../index').request; class InMemoryCrawlQueue { constructor(name, options) { diff --git a/lib/queueSet.js b/providers/queuing/queueSet.js similarity index 100% rename from lib/queueSet.js rename to providers/queuing/queueSet.js diff --git a/providers/queuing/rabbitQueueManager.js b/providers/queuing/rabbitQueueManager.js index 7bb270e..77a82c8 100644 --- a/providers/queuing/rabbitQueueManager.js +++ b/providers/queuing/rabbitQueueManager.js @@ -5,7 +5,7 @@ const AmqpQueue = require('./amqpQueue'); const AttenuatedQueue = require('./attenuatedQueue'); const Q = require('q'); const request = require('request'); -const Request = require('ghcrawler').request; +const Request = require('../../index').request; const TrackedQueue = require('./trackedQueue'); class RabbitQueueManager { diff --git a/providers/queuing/serviceBusQueueManager.js b/providers/queuing/serviceBusQueueManager.js index 36617eb..11da4f6 100644 --- a/providers/queuing/serviceBusQueueManager.js +++ b/providers/queuing/serviceBusQueueManager.js @@ -7,7 +7,7 @@ const AttenuatedQueue = require('./attenuatedQueue'); const azureCommon = require('azure-common'); const InMemoryRateLimiter = require('../limiting/inmemoryRateLimiter'); const RateLimitedPushQueue = require('./ratelimitedPushQueue'); -const Request = require('ghcrawler').request; +const Request = require('../../index').request; const serviceBus = require('azure-sb'); const TrackedQueue = require('./trackedQueue'); const Q = require('q'); diff --git a/routes/deadletters.js b/routes/deadletters.js index 6486bbc..686d80e 100644 --- a/routes/deadletters.js +++ b/routes/deadletters.js @@ -4,7 +4,7 @@ const auth = require('../middleware/auth'); const express = require('express'); const expressJoi = require('express-joi'); -const Request = require('ghcrawler').request; +const Request = require('../index').request; const wrap = require('../middleware/promiseWrap'); let crawlerService = null; diff --git a/routes/requests.js b/routes/requests.js index 34261a3..e269817 100644 --- a/routes/requests.js +++ b/routes/requests.js @@ -4,8 +4,8 @@ const auth = require('../middleware/auth'); const express = require('express'); const expressJoi = require('express-joi'); -const Request = require('ghcrawler').request; -const TraversalPolicy = require('ghcrawler').traversalPolicy; +const Request = require('../index').request; +const TraversalPolicy = require('../index').traversalPolicy; const wrap = require('../middleware/promiseWrap'); const requestsSchema = { diff --git a/routes/webhook.js b/routes/webhook.js index f3ac4bb..a86f4be 100644 --- a/routes/webhook.js +++ b/routes/webhook.js @@ -4,7 +4,7 @@ const crypto = require('crypto'); const express = require('express'); const moment = require('moment'); -const Request = require('ghcrawler').request; +const Request = require('../index').request; const wrap = require('../middleware/promiseWrap'); let crawlerService = null; @@ -15,7 +15,7 @@ router.post('/', wrap(function* (request, response, next) { if (crawlerService.options.queuing.events.provider !== 'webhook') { return warn(request, response, 'Webhooks not enabled'); } - getLogger().verbose('Received', `Webhook event`, {delivery: request.headers['x-github-delivery']}); + getLogger().verbose('Received', `Webhook event`, { delivery: request.headers['x-github-delivery'] }); const signature = request.headers['x-hub-signature']; const eventType = request.headers['x-github-event']; @@ -39,20 +39,20 @@ router.post('/', wrap(function* (request, response, next) { result.context.repoType = 'private'; } yield crawlerService.queue(result, 'events'); - getLogger().info('Queued', `Webhook event for ${eventsUrl}`, {delivery: request.headers['x-github-delivery']}); + getLogger().info('Queued', `Webhook event for ${eventsUrl}`, { delivery: request.headers['x-github-delivery'] }); response.status(200).end(); })); function warn(request, response, message) { - getLogger().warn(fatal, { delivery: request.headers['x-github-delivery']}); + getLogger().warn(fatal, { delivery: request.headers['x-github-delivery'] }); response.status(500); response.setHeader('content-type', 'text/plain'); response.end(JSON.stringify(fatal)); } function fatal(request, response, error) { - getLogger().error(error, { delivery: request.headers['x-github-delivery']}); + getLogger().error(error, { delivery: request.headers['x-github-delivery'] }); response.status(400); response.setHeader('content-type', 'text/plain'); response.end(JSON.stringify(error)); diff --git a/test/integration/amqp10QueueTests.js b/test/integration/amqp10QueueTests.js index a717221..4f361f8 100644 --- a/test/integration/amqp10QueueTests.js +++ b/test/integration/amqp10QueueTests.js @@ -6,7 +6,7 @@ const config = require('painless-config'); const expect = require('chai').expect; const CrawlerFactory = require('../../lib/crawlerFactory'); const Q = require('q'); -const Request = require('ghcrawler').request; +const Request = require('../../index').request; const url = config.get('CRAWLER_AMQP10_URL'); // URL should be: amqps://:@ const name = 'test'; diff --git a/test/unit/crawlerTests.js b/test/unit/crawlerTests.js index 363fa62..e5f2242 100644 --- a/test/unit/crawlerTests.js +++ b/test/unit/crawlerTests.js @@ -3,16 +3,16 @@ const assert = require('chai').assert; const chai = require('chai'); -const Crawler = require('../lib/crawler'); +const Crawler = require('../../lib/crawler'); const expect = require('chai').expect; const extend = require('extend'); -const GitHubFetcher = require('../lib/githubFetcher'); -const GitHubProcessor = require('../lib/githubProcessor'); +const GitHubFetcher = require('../../providers/fetcher/githubFetcher'); +const GitHubProcessor = require('../../providers/fetcher/githubProcessor'); const Q = require('q'); -const QueueSet = require('../lib/queueSet'); -const Request = require('../lib/request'); +const QueueSet = require('../../providers/queuing/queueSet'); +const Request = require('../../lib/request'); const sinon = require('sinon'); -const TraversalPolicy = require('../lib/traversalPolicy'); +const TraversalPolicy = require('../../lib/traversalPolicy'); describe('Crawler get request', () => { it('should get from the priority queue first', () => { @@ -1074,7 +1074,7 @@ function createErrorResponse(error) { }; } -function createBaseCrawler({queues = createBaseQueues(), store = createBaseStore(), locker = createBaseLocker(), requestor = createBaseRequestor(), fetcher = null, options = createBaseOptions() } = {}) { +function createBaseCrawler({ queues = createBaseQueues(), store = createBaseStore(), locker = createBaseLocker(), requestor = createBaseRequestor(), fetcher = null, options = createBaseOptions() } = {}) { if (!fetcher) { fetcher = createBaseFetcher(); } @@ -1119,11 +1119,11 @@ function createBaseOptions(logger = createBaseLog()) { return result; } -function createBaseQueues({ priority = null, normal = null, deadletter = null, options = null} = {}) { +function createBaseQueues({ priority = null, normal = null, deadletter = null, options = null } = {}) { return new QueueSet([priority || createBaseQueue('priority'), normal || createBaseQueue('normal')], deadletter || createBaseQueue('deadletter'), (options || createBaseOptions()).queuing); } -function createBaseQueue(name, { pop = null, push = null, done = null, abandon = null} = {}) { +function createBaseQueue(name, { pop = null, push = null, done = null, abandon = null } = {}) { const result = { name: name }; result.getName = () => { return name; }; result.pop = pop || (() => assert.fail('should not pop')); @@ -1133,7 +1133,7 @@ function createBaseQueue(name, { pop = null, push = null, done = null, abandon = return result; } -function createBaseStore({etag = null, upsert = null, get = null} = {}) { +function createBaseStore({ etag = null, upsert = null, get = null } = {}) { const result = {}; result.etag = etag || (() => { assert.fail('should not etag'); }); result.upsert = upsert || (() => { assert.fail('should not upsert'); }); @@ -1141,7 +1141,7 @@ function createBaseStore({etag = null, upsert = null, get = null} = {}) { return result; } -function createBaseLog({log = null, info = null, warn = null, error = null, verbose = null, silly = null} = {}) { +function createBaseLog({ log = null, info = null, warn = null, error = null, verbose = null, silly = null } = {}) { const result = {}; result.log = log || (() => { }); result.info = info || (() => { }); @@ -1153,7 +1153,7 @@ function createBaseLog({log = null, info = null, warn = null, error = null, verb return result; } -function createBaseLocker({lock = null, unlock = null} = {}) { +function createBaseLocker({ lock = null, unlock = null } = {}) { const result = {}; result.lock = lock || (() => assert.fail('should not lock')); result.unlock = unlock || (() => assert.fail('should not unlock')); diff --git a/test/unit/eventFinderTests.js b/test/unit/eventFinderTests.js index 67665a2..dbeca22 100644 --- a/test/unit/eventFinderTests.js +++ b/test/unit/eventFinderTests.js @@ -3,7 +3,7 @@ const expect = require('chai').expect; const extend = require('extend'); -const finder = require('../lib/eventFinder.js'); +const finder = require('../../lib/eventFinder.js'); const Q = require('q'); describe('Event Finder', () => { diff --git a/test/unit/gitHubProcessorTests.js b/test/unit/gitHubProcessorTests.js index ac246c6..6908407 100644 --- a/test/unit/gitHubProcessorTests.js +++ b/test/unit/gitHubProcessorTests.js @@ -2,11 +2,11 @@ // Licensed under the MIT License. const expect = require('chai').expect; -const GitHubProcessor = require('../lib/githubProcessor.js'); +const GitHubProcessor = require('../../providers/fetcher/githubProcessor.js'); const Q = require('q'); -const Request = require('../lib/request.js'); +const Request = require('../../lib/request.js'); const sinon = require('sinon'); -const TraversalPolicy = require('../lib/traversalPolicy'); +const TraversalPolicy = require('../../lib/traversalPolicy'); describe('GitHubProcessor reprocessing', () => { it('will skip if at same version', () => { @@ -435,7 +435,7 @@ describe('Pull Request processing', () => { head: { repo: { id: 45, url: 'http://repo/45' } }, base: { repo: { id: 17, url: 'http://repo/17' } }, _links: { - self: { href: 'http://pull_request/13'}, + self: { href: 'http://pull_request/13' }, issue: { href: 'http://issue/13' }, review_comments: { href: 'http://review_comments' }, commits: { href: 'http://commits' }, @@ -622,7 +622,7 @@ describe('Pull request/review comment processing', () => { // test that the new request got queued and that the doc has the right stuff const newRequest = queue.pop(); - newRequest.document = {id: 1, url: 'http://pull_request/1'} + newRequest.document = { id: 1, url: 'http://pull_request/1' } testPullRequestReviewCommentEvent(newRequest, 'LegacyPullRequestReviewCommentEvent'); }); }); diff --git a/test/unit/githubFetcherTests.js b/test/unit/githubFetcherTests.js index 4f06dc1..f8cdfbe 100644 --- a/test/unit/githubFetcherTests.js +++ b/test/unit/githubFetcherTests.js @@ -3,15 +3,15 @@ const assert = require('chai').assert; const chai = require('chai'); -const Crawler = require('../lib/crawler'); +const Crawler = require('../../lib/crawler'); const expect = require('chai').expect; const extend = require('extend'); -const GitHubFetcher = require('../lib/githubFetcher'); +const GitHubFetcher = require('../../providers/fetcher/githubFetcher'); const Q = require('q'); -const QueueSet = require('../lib/queueSet'); -const Request = require('../lib/request'); +const QueueSet = require('../../providers/queuing/queueSet'); +const Request = require('../../lib/request'); const sinon = require('sinon'); -const TraversalPolicy = require('../lib/traversalPolicy'); +const TraversalPolicy = require('../../lib/traversalPolicy'); const URL = require('url'); describe('GitHub fetcher', () => { @@ -250,7 +250,7 @@ function createResponse(body, code = 200, etag = null, remaining = 4000, reset = }; } -function createBaseStore({etag = null, upsert = null, get = null} = {}) { +function createBaseStore({ etag = null, upsert = null, get = null } = {}) { const result = {}; result.etag = etag || (() => { assert.fail('should not etag'); }); result.upsert = upsert || (() => { assert.fail('should not upsert'); }); @@ -296,7 +296,7 @@ function createBaseOptions(logger = createBaseLog()) { }; } -function createBaseLog({info = null, warn = null, error = null, verbose = null, silly = null} = {}) { +function createBaseLog({ info = null, warn = null, error = null, verbose = null, silly = null } = {}) { const result = {}; result.info = info || (() => { }); result.warn = warn || (() => { }); diff --git a/test/unit/processingTests.js b/test/unit/processingTests.js index 38886bb..36aff22 100644 --- a/test/unit/processingTests.js +++ b/test/unit/processingTests.js @@ -4,7 +4,7 @@ const expect = require('chai').expect; const CrawlerFactory = require('../../lib/crawlerFactory'); const Q = require('q'); -const Request = require('ghcrawler').request; +const Request = require('../../index').request; const sinon = require('sinon'); let crawler = null; diff --git a/test/unit/queueSetTests.js b/test/unit/queueSetTests.js index 6c18597..5c83ccb 100644 --- a/test/unit/queueSetTests.js +++ b/test/unit/queueSetTests.js @@ -3,9 +3,9 @@ const assert = require('chai').assert; const expect = require('chai').expect; -const Request = require('../lib/request.js'); +const Request = require('../../lib/request.js'); const Q = require('q'); -const QueueSet = require('../lib/queueSet.js'); +const QueueSet = require('../../providers/queuing/queueSet.js'); const sinon = require('sinon'); describe('QueueSet construction', () => { @@ -253,7 +253,7 @@ function createBaseQueues(queues, deadletter, weights = [1]) { return new QueueSet(queues, deadletter || createBaseQueue('deadletter'), createOptions(weights)); } -function createBaseQueue(name, { pop = null, push = null, done = null, abandon = null, subscribe = null, unsubscribe = null} = {}) { +function createBaseQueue(name, { pop = null, push = null, done = null, abandon = null, subscribe = null, unsubscribe = null } = {}) { const result = { name: name }; result.getName = () => { return name; }; result.pop = pop || (() => assert.fail('should not pop')); diff --git a/test/unit/requestTests.js b/test/unit/requestTests.js index 62d0ee4..ef20221 100644 --- a/test/unit/requestTests.js +++ b/test/unit/requestTests.js @@ -3,9 +3,9 @@ const assert = require('chai').assert; const expect = require('chai').expect; -const Request = require('../lib/request.js'); +const Request = require('../../lib/request.js'); const sinon = require('sinon'); -const TraversalPolicy = require('../lib/traversalPolicy'); +const TraversalPolicy = require('../../lib/traversalPolicy'); describe('Request context/qualifier', () => { it('will not queueRoot if none transitivity', () => { diff --git a/test/unit/requestTrackerTests.js b/test/unit/requestTrackerTests.js index ca49cf5..28a0979 100644 --- a/test/unit/requestTrackerTests.js +++ b/test/unit/requestTrackerTests.js @@ -7,7 +7,7 @@ const expect = require('chai').expect; const extend = require('extend'); const Q = require('q'); const redlock = require('redlock'); -const Request = require('ghcrawler').request; +const Request = require('../../index').request; const RequestTracker = require('../../providers/queuing/redisRequestTracker.js'); const sinon = require('sinon'); diff --git a/test/unit/visitorMapTests.js b/test/unit/visitorMapTests.js index c7d9ce0..35d5e85 100644 --- a/test/unit/visitorMapTests.js +++ b/test/unit/visitorMapTests.js @@ -4,7 +4,7 @@ const assert = require('chai').assert; const expect = require('chai').expect; const sinon = require('sinon'); -const VisitorMap = require('../lib/visitorMap'); +const VisitorMap = require('../../lib/visitorMap'); describe('Visitor Map', () => {