зеркало из https://github.com/microsoft/ghcrawler.git
Remove comments and improve logging
This commit is contained in:
Родитель
0e8203242b
Коммит
763d951996
|
@ -13,18 +13,18 @@ class Crawler {
|
|||
|
||||
start() {
|
||||
return this.queue.pop()
|
||||
.catch()
|
||||
.then(this._convertToRequest.bind(this))
|
||||
.then((request) => {
|
||||
console.log(request.crawl.url);
|
||||
.then(this._filterSeen.bind(this))
|
||||
.then(request => {
|
||||
if (request) {
|
||||
this.logger.log('info', `${request.crawl.url} [${request.crawl.type}]`);
|
||||
}
|
||||
return request;
|
||||
})
|
||||
.then(this._filterSeen.bind(this))
|
||||
.then(this._fetch.bind(this))
|
||||
.then(this._convertToDocument.bind(this))
|
||||
.then(this._processDocument.bind(this))
|
||||
.then(this._storeDocument.bind(this))
|
||||
.catch()
|
||||
.then(this._deleteFromQueue.bind(this))
|
||||
.then(this._markSeen.bind(this))
|
||||
.then(this._startNext.bind(this));
|
||||
|
|
Загрузка…
Ссылка в новой задаче