зеркало из https://github.com/github/lightcrawler.git
Initial commit
This commit is contained in:
Коммит
a07f24ea09
|
@ -0,0 +1 @@
|
|||
node_modules
|
|
@ -0,0 +1,6 @@
|
|||
{
|
||||
"extends": "lighthouse:default",
|
||||
"settings": {
|
||||
"onlyAudits": ["color-contrast"]
|
||||
}
|
||||
}
|
|
@ -0,0 +1,60 @@
|
|||
const cheerio = require('cheerio')
|
||||
const Crawler = require('simplecrawler')
|
||||
const ChildProcess = require('child_process')
|
||||
const path = require('path')
|
||||
|
||||
const crawler = new Crawler('https://electron.atom.io')
|
||||
crawler.respectRobotsTxt = false
|
||||
crawler.parseHTMLComments = false
|
||||
crawler.parseScriptTags = false
|
||||
crawler.maxDepth = 1
|
||||
|
||||
crawler.discoverResources = (buffer, item) => {
|
||||
const page = cheerio.load(buffer.toString('utf8'))
|
||||
const links = page('a[href]').map(function () {
|
||||
return page(this).attr('href')
|
||||
}).get()
|
||||
|
||||
return links
|
||||
}
|
||||
|
||||
crawler.on('fetchcomplete', (queueItem, responseBuffer, response) => {
|
||||
console.log('running lighthouse on ', queueItem.url)
|
||||
runLighthouse(queueItem.url)
|
||||
})
|
||||
|
||||
function runLighthouse (url) {
|
||||
const lighthouse = ChildProcess.spawn(path.join(__dirname, 'node_modules', '.bin', 'lighthouse'), [
|
||||
url,
|
||||
'--output=json',
|
||||
'--output-path=stdout',
|
||||
'--disable-device-emulation',
|
||||
'--disable-cpu-throttling',
|
||||
'--disable-network-throttling',
|
||||
'--chrome-flags="--headless --disable-gpu"',
|
||||
`--config-path=${path.join(__dirname, 'config.json')}`
|
||||
])
|
||||
|
||||
let output = ''
|
||||
lighthouse.stdout.on('data', (data) => {
|
||||
output += data
|
||||
})
|
||||
lighthouse.once('close', () => {
|
||||
const report = JSON.parse(output)
|
||||
|
||||
report.reportCategories.forEach((category) => {
|
||||
category.audits.forEach((audit) => {
|
||||
if (audit.score !== 100) {
|
||||
console.log(`${url} failed ${audit.id}`)
|
||||
audit.result.extendedInfo.value.nodes.forEach((result) => {
|
||||
console.log(result.failureSummary)
|
||||
console.log(result.path)
|
||||
console.log(result.html)
|
||||
})
|
||||
}
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
crawler.start()
|
|
@ -0,0 +1,17 @@
|
|||
{
|
||||
"name": "lightcrawler",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"cheerio": "^1.0.0-rc.1",
|
||||
"lighthouse": "^2.1.0",
|
||||
"simplecrawler": "^1.1.3"
|
||||
}
|
||||
}
|
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
Загрузка…
Ссылка в новой задаче