зеркало из https://github.com/microsoft/ghcrawler.git
Add event handling, update driver structure
This commit is contained in:
Jeff McAffer
Родитель
2325dc9ee8
Коммит
4208ec4e30
187
lib/crawler.js
187
lib/crawler.js
|
|
@ -76,10 +76,6 @@ class Crawler {
|
|||
}
|
||||
let document = null;
|
||||
switch (request.type) {
|
||||
case 'event': {
|
||||
document = self._processEvent(request.response, request.context);
|
||||
break;
|
||||
}
|
||||
case 'orgs': {
|
||||
document = this._processCollection(request.response, 'login', request.context);
|
||||
break;
|
||||
|
|
@ -136,6 +132,12 @@ class Crawler {
|
|||
document = this._processTeamRepos(request.response, request.context);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
const handler = this[request.type];
|
||||
if (handler && typeof handler === 'function') {
|
||||
document = handler.call(this, reqeust.response, request.context);
|
||||
}
|
||||
}
|
||||
}
|
||||
request.document = document;
|
||||
return Q.resolve(request);
|
||||
|
|
@ -173,16 +175,6 @@ class Crawler {
|
|||
return null;
|
||||
}
|
||||
|
||||
_processEvent(document, context) {
|
||||
document._metadata.links.self = { type: 'self', href: `urn:repo:${context.repo}:event:${document.id}` };
|
||||
document._metadata.links.repo = { type: 'self', href: `urn:repo:${context.repo}` };
|
||||
document._metadata.links.parent = document._metadata.links.repo;
|
||||
document._metadata.links.user = { type: 'self', href: `urn:login:${document.user.id}` };
|
||||
this.queue.push('login', document.user.url);
|
||||
// TODO push the elements of the event dependent on the event type
|
||||
return document;
|
||||
}
|
||||
|
||||
_processRepo(document) {
|
||||
document._metadata.links.self = { type: 'self', href: `urn:repo:${document.id}` };
|
||||
document._metadata.links.owner = { type: 'self', href: `urn:login:${document.owner.id}` };
|
||||
|
|
@ -275,6 +267,173 @@ class Crawler {
|
|||
document._metadata.links.self = { type: 'self', href: `urn:org:${document.organization.id}:team:${document.id}:repos` };
|
||||
return document;
|
||||
}
|
||||
|
||||
// Event handlers
|
||||
CommitCommentEvent(document, context) {
|
||||
const payload = _eventHelper(document);
|
||||
document._metadata.links.self = { type: 'self', href: `urn:repo:${context.repo}:commit_comment_event:${document.id}` };
|
||||
document._metadata.links.comment = { type: 'self', href: `urn:repo:${context.repo}:comment:${payload.comment.id}` };
|
||||
// TODO siblings?
|
||||
this.queue.push('comment', payload.comment.url);
|
||||
return document;
|
||||
}
|
||||
|
||||
CreateEvent(document, context) {
|
||||
const payload = _eventHelper(document);
|
||||
document._metadata.links.self = { type: 'self', href: `urn:repo:${context.repo}:create_event:${document.id}` };
|
||||
return document;
|
||||
}
|
||||
|
||||
DeleteEvent(document, context) {
|
||||
const payload = _eventHelper(document);
|
||||
document._metadata.links.self = { type: 'self', href: `urn:repo:${context.repo}:delete_event:${document.id}` };
|
||||
// TODO do something for interesting deletions e.g., where ref-type === 'repository'
|
||||
return document;
|
||||
}
|
||||
|
||||
DeploymentEvent(document, context) {
|
||||
const payload = _eventHelper(document);
|
||||
document._metadata.links.self = { type: 'self', href: `urn:repo:${context.repo}:deployment_event:${document.id}` };
|
||||
document._metadata.links.deployment = { type: 'self', href: `urn:repo:${context.repo}:deployment:${payload.deployment.id}` };
|
||||
this.queue.push('deployment', payload.deployment.url);
|
||||
return document;
|
||||
}
|
||||
|
||||
DeploymentStatusEvent(document, context) {
|
||||
const payload = _eventHelper(document);
|
||||
document._metadata.links.self = { type: 'self', href: `urn:repo:${context.repo}:deployment_status_event:${document.id}` };
|
||||
document._metadata.links.deployment_status = { type: 'self', href: `urn:repo:${context.repo}:deployment:${payload.deployment.id}:status:${payload.deployment_status.id}` };
|
||||
document._metadata.links.deployment = { type: 'self', href: `urn:repo:${context.repo}:deployment:${payload.deployment.id}` };
|
||||
this.queue.push('deployment', payload.deployment.url);
|
||||
return document;
|
||||
}
|
||||
|
||||
ForkEvent(document, context) {
|
||||
const payload = _eventHelper(document);
|
||||
document._metadata.links.self = { type: 'self', href: `urn:repo:${context.repo}:fork_event:${document.id}` };
|
||||
// TODO figure out what else to do
|
||||
return document;
|
||||
}
|
||||
|
||||
GollumEvent(document, context) {
|
||||
const payload = _eventHelper(document);
|
||||
document._metadata.links.self = { type: 'self', href: `urn:repo:${context.repo}:gollum_event:${document.id}` };
|
||||
return document;
|
||||
}
|
||||
|
||||
IssueCommentEvent(document, context) {
|
||||
const payload = _eventHelper(document);
|
||||
document._metadata.links.self = { type: 'self', href: `urn:repo:${context.repo}:issue_comment_event:${document.id}` };
|
||||
document._metadata.links.issue = { type: 'self', href: `urn:repo:${context.repo}:issue:${payload.issue.id}` };
|
||||
document._metadata.links.comment = { type: 'self', href: `urn:repo:${context.repo}:comment:${payload.comment.id}` };
|
||||
this.queue.push('comment', payload.comment.url);
|
||||
this.queue.push('issue', payload.issue.url);
|
||||
return document;
|
||||
}
|
||||
|
||||
IssuesEvent(document, context) {
|
||||
const payload = _eventHelper(document);
|
||||
document._metadata.links.self = { type: 'self', href: `urn:repo:${context.repo}:issued_event:${document.id}` };
|
||||
document._metadata.links.issue = { type: 'self', href: `urn:repo:${context.repo}:issue:${payload.issue.id}` };
|
||||
this.queue.push('issue', payload.issue.url);
|
||||
return document;
|
||||
}
|
||||
|
||||
LabelEvent(document, context) {
|
||||
const payload = _eventHelper(document);
|
||||
document._metadata.links.self = { type: 'self', href: `urn:repo:${context.repo}:label_event:${document.id}` };
|
||||
return document;
|
||||
}
|
||||
|
||||
MemberEvent(document, context) {
|
||||
const payload = _eventHelper(document);
|
||||
document._metadata.links.self = { type: 'self', href: `urn:repo:${context.repo}:member_event:${document.id}` };
|
||||
document._metadata.links.member = { type: 'self', href: `urn:login:${payload.member.id}` };
|
||||
this.queue.push('login', payload.member.url);
|
||||
return document;
|
||||
}
|
||||
|
||||
MembershipEvent(document, context) {
|
||||
const payload = _eventHelper(document);
|
||||
document._metadata.links.self = { type: 'self', href: `urn:team:${payload.team.id}:membership_event:${document.id}` };
|
||||
document._metadata.links.member = { type: 'self', href: `urn:login:${payload.member.id}` };
|
||||
document._metadata.links.team = { type: 'self', href: `urn:team:${payload.team.id}` };
|
||||
document._metadata.links.org = { type: 'self', href: `urn:org:${payload.organization.id}` };
|
||||
this.queue.push('login', payload.member.url);
|
||||
this.queue.push('org', payload.organization.url);
|
||||
this.queue.push('team', payload.team.url);
|
||||
return document;
|
||||
}
|
||||
|
||||
MilestoneEvent(document, context) {
|
||||
const payload = _eventHelper(document);
|
||||
document._metadata.links.self = { type: 'self', href: `urn:repo:${context.repo}:milestone_event:${document.id}` };
|
||||
document._metadata.links.milestone = { type: 'self', href: `urn:repo:${context.repo}:milestone:${payload.milestone.id}` };
|
||||
this.queue.push('milestone', payload.milestone.url);
|
||||
return document;
|
||||
}
|
||||
|
||||
PageBuildEvent(document, context) {
|
||||
const payload = _eventHelper(document);
|
||||
document._metadata.links.self = { type: 'self', href: `urn:repo:${context.repo}:page_build_event:${document.id}` };
|
||||
document._metadata.links.page_build = { type: 'self', href: `urn:repo:${context.repo}:page_builds:${payload.id}` };
|
||||
this.queue.push('page_build', payload.build.url);
|
||||
return document;
|
||||
}
|
||||
|
||||
PublicEvent(document, context) {
|
||||
const payload = _eventHelper(document);
|
||||
document._metadata.links.self = { type: 'self', href: `urn:repo:${context.repo}:public_event:${document.id}` };
|
||||
return document;
|
||||
}
|
||||
|
||||
PullRequestEvent(document, context) {
|
||||
const payload = _eventHelper(document);
|
||||
document._metadata.links.self = { type: 'self', href: `urn:repo:${context.repo}:pull_request_event:${document.id}` };
|
||||
document._metadata.links.pull = { type: 'self', href: `urn:repo:${context.repo}:pull:${payload.pull_request.id}` };
|
||||
this.queue.push('pull', payload.pull_request.url);
|
||||
return document;
|
||||
}
|
||||
|
||||
PullRequestReviewEvent(document, context) {
|
||||
const payload = _eventHelper(document);
|
||||
document._metadata.links.self = { type: 'self', href: `urn:repo:${context.repo}:pull_request_review_event:${document.id}` };
|
||||
document._metadata.links.review = { type: 'self', href: `urn:repo:${context.repo}:pull:${payload.pull_request.id}:review:${payload.review.id}` };
|
||||
document._metadata.links.pull = { type: 'self', href: `urn:repo:${context.repo}:pull:${payload.pull_request.id}` };
|
||||
this.queue.push('pull_review', payload.pull_request.review_comment_url.replace('{/number}', `/${payload.review.id}`));
|
||||
this.queue.push('pull', payload.pull_request.url);
|
||||
return document;
|
||||
}
|
||||
|
||||
PullRequestReviewCommentEvent(document, context) {
|
||||
const payload = _eventHelper(document);
|
||||
document._metadata.links.self = { type: 'self', href: `urn:repo:${context.repo}:pull_request_review_comment_event:${document.id}` };
|
||||
document._metadata.links.comment = { type: 'self', href: `urn:repo:${context.repo}:pull:${payload.pull_request.id}:comment:${payload.comment.id}` };
|
||||
document._metadata.links.pull = { type: 'self', href: `urn:repo:${context.repo}:pull:${payload.pull_request.id}` };
|
||||
// TODO see if all the various comments can be the same type
|
||||
this.queue.push('pull_comment', payload.comment.url);
|
||||
this.queue.push('pull', payload.pull_request.url);
|
||||
return document;
|
||||
}
|
||||
|
||||
PushEvent(document, context) {
|
||||
const payload = _eventHelper(document);
|
||||
document._metadata.links.self = { type: 'self', href: `urn:repo:${context.repo}:push_event:${document.id}` };
|
||||
// TODO figure out what to do with the commits
|
||||
return document;
|
||||
}
|
||||
|
||||
|
||||
_eventHelper(document) {
|
||||
// TODO understand if the actor is typically the same as the creator or pusher in the payload
|
||||
document._metadata.links.actor = { type: 'self', href: `urn:login:${document.actor.id}` };
|
||||
document._metadata.links.repo = { type: 'self', href: `urn:repo:${document.repo.id}` };
|
||||
document._metadata.links.org = { type: 'self', href: `urn:org:${document.org.id}` };
|
||||
this.queue.push('login', document.actor.url);
|
||||
this.queue.push('repo', document.repo.url);
|
||||
this.queue.push('org', document.org.url);
|
||||
return document.payload;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = Crawler;
|
||||
|
|
@ -1,32 +1,42 @@
|
|||
const async = require('async');
|
||||
|
||||
class WebhookDriver {
|
||||
|
||||
static watch(trigger, eventFinder, eventSink) {
|
||||
async.whilst(
|
||||
() => true,
|
||||
(completion) => {
|
||||
setTimeout(() => { WebhookDriver.handleNext(trigger, eventFinder, eventSink, completion); }, 0);
|
||||
});
|
||||
constructor(trigger, eventFinder, eventSink) {
|
||||
this.trigger = trigger;
|
||||
this.eventFinder = eventFinder;
|
||||
this.eventSink = eventSink;
|
||||
}
|
||||
|
||||
static handleNext(trigger, eventFinder, eventSink, completion) {
|
||||
trigger.pop().then(message => {
|
||||
const source = WebhookDriver._chooseSource(message);
|
||||
eventFinder.discoverAndQueue(source, eventSink).then(
|
||||
() => { completion(); },
|
||||
(err) => { completion(err); });
|
||||
start() {
|
||||
return this.trigger.pop()
|
||||
.then(this._handleRequest.bind(this))
|
||||
.then(this._deleteFromQueue.bind(this))
|
||||
.then(this._startNext.bind(this));
|
||||
}
|
||||
|
||||
_startNext() {
|
||||
setTimeout(this.start.bind(this), 0);
|
||||
}
|
||||
|
||||
_handleRequest(request) {
|
||||
const source = this._chooseSource(request);
|
||||
return this.eventFinder.discoverAndQueue(source, this.eventSink).then(() => {
|
||||
return request;
|
||||
});
|
||||
}
|
||||
|
||||
static _chooseSource(message) {
|
||||
if (!message.qualifier) {
|
||||
_deleteFromQueue(request) {
|
||||
return this.trigger.done(request).then(() => { return request; });
|
||||
}
|
||||
|
||||
_chooseSource(request) {
|
||||
if (!request.qualifier) {
|
||||
return null;
|
||||
}
|
||||
if (message.qualifier.includes('/')) {
|
||||
return `https://api.github.com/repos/${message.qualifier}/events`;
|
||||
if (request.qualifier.includes('/')) {
|
||||
return `https://api.github.com/repos/${request.qualifier}/events`;
|
||||
}
|
||||
return `https://api.github.com/orgs/${message.qualifier}/events`;
|
||||
return `https://api.github.com/orgs/${request.qualifier}/events`;
|
||||
}
|
||||
}
|
||||
module.exports = WebhookDriver;
|
||||
Загрузка…
Ссылка в новой задаче