Refactoring to use a common request configuration.
Replacing promised-io with bluebird in app.js, loadjs and loadcss Adding linting configurations
This commit is contained in:
molant
Родитель
a59f45e344
Коммит
bc3c4df6ab
|
|
@ -0,0 +1,12 @@
|
|||
# http://editorconfig.org
|
||||
root = true
|
||||
|
||||
[*]
|
||||
indent_style = tab
|
||||
indent_size = 4
|
||||
charset = utf-8
|
||||
trim_trailing_whitespace = true
|
||||
insert_final_newline = false
|
||||
|
||||
[*.md]
|
||||
trim_trailing_whitespace = false
|
||||
|
|
@ -0,0 +1,149 @@
|
|||
{
|
||||
"env": {
|
||||
"browser": false,
|
||||
"node": true,
|
||||
"amd": false,
|
||||
"mocha": true,
|
||||
"jasmine": false
|
||||
},
|
||||
|
||||
"rules": {
|
||||
"no-alert": 2,
|
||||
"no-array-constructor": 2,
|
||||
"no-bitwise": 0,
|
||||
"no-caller": 2,
|
||||
"no-catch-shadow": 2,
|
||||
"no-cond-assign": 2,
|
||||
"no-console": 0,
|
||||
"no-constant-condition": 2,
|
||||
"no-control-regex": 2,
|
||||
"no-debugger": 2,
|
||||
"no-delete-var": 2,
|
||||
"no-div-regex": 0,
|
||||
"no-dupe-keys": 2,
|
||||
"no-else-return": 0,
|
||||
"no-empty": 2,
|
||||
"no-empty-class": 2,
|
||||
"no-empty-label": 2,
|
||||
"no-eq-null": 0,
|
||||
"no-eval": 2,
|
||||
"no-ex-assign": 2,
|
||||
"no-extend-native": 2,
|
||||
"no-extra-bind": 2,
|
||||
"no-extra-boolean-cast": 2,
|
||||
"no-extra-parens": 0,
|
||||
"no-extra-semi": 2,
|
||||
"no-fallthrough": 2,
|
||||
"no-floating-decimal": 0,
|
||||
"no-func-assign": 2,
|
||||
"no-implied-eval": 2,
|
||||
"no-inline-comments": 0,
|
||||
"no-inner-declarations": [2, "functions"],
|
||||
"no-invalid-regexp": 2,
|
||||
"no-irregular-whitespace": 2,
|
||||
"no-iterator": 2,
|
||||
"no-label-var": 2,
|
||||
"no-labels": 2,
|
||||
"no-lone-blocks": 2,
|
||||
"no-lonely-if": 0,
|
||||
"no-loop-func": 2,
|
||||
"no-mixed-requires": [0, false],
|
||||
"no-mixed-spaces-and-tabs": [2, false],
|
||||
"no-multi-spaces": 2,
|
||||
"no-multi-str": 2,
|
||||
"no-multiple-empty-lines": [0, {"max": 2}],
|
||||
"no-native-reassign": 2,
|
||||
"no-negated-in-lhs": 2,
|
||||
"no-nested-ternary": 0,
|
||||
"no-new": 2,
|
||||
"no-new-func": 2,
|
||||
"no-new-object": 2,
|
||||
"no-new-require": 0,
|
||||
"no-new-wrappers": 2,
|
||||
"no-obj-calls": 2,
|
||||
"no-octal": 2,
|
||||
"no-octal-escape": 2,
|
||||
"no-path-concat": 0,
|
||||
"no-plusplus": 0,
|
||||
"no-process-env": 0,
|
||||
"no-process-exit": 2,
|
||||
"no-proto": 2,
|
||||
"no-redeclare": 0,
|
||||
"no-regex-spaces": 2,
|
||||
"no-reserved-keys": 0,
|
||||
"no-restricted-modules": 0,
|
||||
"no-return-assign": 2,
|
||||
"no-script-url": 2,
|
||||
"no-self-compare": 0,
|
||||
"no-sequences": 2,
|
||||
"no-shadow": 2,
|
||||
"no-shadow-restricted-names": 2,
|
||||
"no-spaced-func": 2,
|
||||
"no-sparse-arrays": 2,
|
||||
"no-sync": 0,
|
||||
"no-ternary": 0,
|
||||
"no-trailing-spaces": 0,
|
||||
"no-undef": 2,
|
||||
"no-undef-init": 2,
|
||||
"no-undefined": 2,
|
||||
"no-underscore-dangle": 0,
|
||||
"no-unreachable": 2,
|
||||
"no-unused-expressions": 2,
|
||||
"no-unused-vars": [2, {"vars": "all", "args": "after-used"}],
|
||||
"no-use-before-define": 2,
|
||||
"no-void": 0,
|
||||
"no-warning-comments": [0, { "terms": ["todo", "fixme", "xxx"], "location": "start" }],
|
||||
"no-with": 2,
|
||||
"no-wrap-func": 2,
|
||||
"block-scoped-var": 0,
|
||||
"brace-style": [0, "1tbs"],
|
||||
"camelcase": 0,
|
||||
"comma-spacing": 2,
|
||||
"comma-style": 0,
|
||||
"comma-dangle": [2, "never"],
|
||||
"complexity": [0, 11],
|
||||
"consistent-return": 2,
|
||||
"consistent-this": [0, "that"],
|
||||
"curly": [2, "all"],
|
||||
"default-case": 0,
|
||||
"dot-notation": 2,
|
||||
"eol-last": 0,
|
||||
"eqeqeq": 2,
|
||||
"func-names": 0,
|
||||
"func-style": [2, "expression"],
|
||||
"guard-for-in": 0,
|
||||
"handle-callback-err": 0,
|
||||
"key-spacing": [2, { "beforeColon": false, "afterColon": true }],
|
||||
"max-depth": [0, 4],
|
||||
"max-len": [0, 80, 4],
|
||||
"max-nested-callbacks": [0, 2],
|
||||
"max-params": [0, 3],
|
||||
"max-statements": [0, 10],
|
||||
"new-cap": 2,
|
||||
"new-parens": 2,
|
||||
"one-var": 0,
|
||||
"operator-assignment": [0, "always"],
|
||||
"padded-blocks": 0,
|
||||
"quote-props": 0,
|
||||
"quotes": [2, "single"],
|
||||
"radix": 0,
|
||||
"semi": 2,
|
||||
"sort-vars": 0,
|
||||
"space-after-keywords": [0, "always"],
|
||||
"space-before-blocks": [0, "always"],
|
||||
"space-in-brackets": [0, "never"],
|
||||
"space-in-parens": [0, "never"],
|
||||
"space-infix-ops": 2,
|
||||
"space-return-throw-case": 2,
|
||||
"space-unary-ops": [2, { "words": true, "nonwords": false }],
|
||||
"spaced-line-comment": [0, "always"],
|
||||
"strict": [2, "global"],
|
||||
"use-isnan": 2,
|
||||
"valid-jsdoc": 0,
|
||||
"valid-typeof": 2,
|
||||
"vars-on-top": 0,
|
||||
"wrap-iife": [2, "outside"],
|
||||
"wrap-regex": 0,
|
||||
"yoda": [2, "never"]
|
||||
}
|
||||
}
|
||||
549
app.js
549
app.js
|
|
@ -15,38 +15,30 @@
|
|||
* and limitations under the License.
|
||||
*/
|
||||
|
||||
"use strict";
|
||||
'use strict';
|
||||
|
||||
var url = require('url'),
|
||||
fs = require('fs'),
|
||||
port = process.env.PORT || 1337,
|
||||
request = require('request'),
|
||||
express = require('express'),
|
||||
app = express(),
|
||||
bodyParser = require('body-parser'),
|
||||
cheerio = require('cheerio'),
|
||||
promises = require('promised-io/promise'),
|
||||
Deferred = require('promised-io').Deferred,
|
||||
promised = require("promised-io/promise"),
|
||||
cssLoader = require('./lib/checks/loadcss.js'),
|
||||
jsLoader = require('./lib/checks/loadjs.js'),
|
||||
tests = require('./lib/checks/loadchecks.js').tests,
|
||||
http = require('http'),
|
||||
path = require('path'),
|
||||
zlib = require('zlib'),
|
||||
sanitize = require('validator').sanitize,
|
||||
charset = 'utf-8',
|
||||
querystring = require('querystring'),
|
||||
version = JSON.parse(fs.readFileSync('package.json')).version,
|
||||
request = request.defaults({followAllRedirects: true,
|
||||
encoding: null,
|
||||
jar: false,
|
||||
proxy: process.env.HTTP_PROXY || process.env.http_proxy,
|
||||
headers: {
|
||||
'Accept': 'text/html, application/xhtml+xml, */*',
|
||||
'Accept-Encoding': 'gzip,deflate',
|
||||
'Accept-Language': 'en-US,en;q=0.5',
|
||||
'User-Agent': 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)'}});
|
||||
fs = require('fs'),
|
||||
port = process.env.PORT || 1337,
|
||||
requester = require('./lib/requester.js'),
|
||||
express = require('express'),
|
||||
app = express(),
|
||||
bodyParser = require('body-parser'),
|
||||
cheerio = require('cheerio'),
|
||||
promises = require('promised-io/promise'),
|
||||
Deferred = require('promised-io').Deferred,
|
||||
promised = require('promised-io/promise'),
|
||||
bluebird = require('bluebird'),
|
||||
cssLoader = require('./lib/checks/loadcss.js'),
|
||||
jsLoader = require('./lib/checks/loadjs.js'),
|
||||
tests = require('./lib/checks/loadchecks.js').tests,
|
||||
//http = require('http'),
|
||||
path = require('path'),
|
||||
zlib = require('zlib'),
|
||||
sanitize = require('validator').sanitize,
|
||||
charset = 'utf-8',
|
||||
querystring = require('querystring'),
|
||||
version = JSON.parse(fs.readFileSync('package.json')).version;
|
||||
|
||||
/**
|
||||
* Serializes a test results array and sends it via the response
|
||||
|
|
@ -54,294 +46,367 @@ var url = require('url'),
|
|||
* @param {Timestamp} start The start timestamp
|
||||
* @param {Array} resultsArray The results of all the tests
|
||||
* */
|
||||
function sendResults(res, start, resultsArray) {
|
||||
var results = {};
|
||||
var sendResults = function (res, start, resultsArray) {
|
||||
var results = {};
|
||||
|
||||
for (var i = 0; i < resultsArray.length; i++) {
|
||||
results[resultsArray[i].testName] = resultsArray[i];
|
||||
}
|
||||
res.writeHeader(200, {"Content-Type": "application/json",
|
||||
"X-Content-Type-Options": "nosniff" });
|
||||
res.write(JSON.stringify({version: version, url: {uri: (this && this.url && this.url.href) || 'http://private'}, processTime: (Date.now() - start)/1000, results: results}));
|
||||
res.end();
|
||||
}
|
||||
for (var i = 0; i < resultsArray.length; i++) {
|
||||
results[resultsArray[i].testName] = resultsArray[i];
|
||||
}
|
||||
res.writeHeader(200, {
|
||||
'Content-Type': 'application/json',
|
||||
'X-Content-Type-Options': 'nosniff'
|
||||
});
|
||||
res.write(JSON.stringify({
|
||||
version: version,
|
||||
url: {uri: (this && this.url && this.url.href) || 'http://private'},
|
||||
processTime: (Date.now() - start) / 1000,
|
||||
results: results
|
||||
}));
|
||||
res.end();
|
||||
};
|
||||
|
||||
/**
|
||||
* Responds with a bad request error
|
||||
* */
|
||||
function sendBadRequest(res){
|
||||
res.writeHeader(400, {"Content-Type": "text/plain"});
|
||||
res.write('Your package is malformed' + '\n');
|
||||
res.end();
|
||||
}
|
||||
var sendBadRequest = function (res) {
|
||||
res.writeHeader(400, {'Content-Type': 'text/plain'});
|
||||
res.write('Your package is malformed' + '\n');
|
||||
res.end();
|
||||
};
|
||||
|
||||
/**
|
||||
* Responds with an internal server error
|
||||
* */
|
||||
function sendInternalServerError(error, res) {
|
||||
res.writeHeader(500, {"Content-Type": "text/plain"});
|
||||
res.write(JSON.stringify(error) + '\n');
|
||||
res.end();
|
||||
}
|
||||
var sendInternalServerError = function (error, res) {
|
||||
res.writeHeader(500, {'Content-Type': 'text/plain'});
|
||||
res.write(JSON.stringify(error) + '\n');
|
||||
res.end();
|
||||
};
|
||||
|
||||
/**
|
||||
* Responds with the error and message passed as parameters
|
||||
* */
|
||||
function remoteErrorResponse(response, statusCode, message) {
|
||||
response.writeHead(200, {"Content-Type": "application/json"});
|
||||
response.write(JSON.stringify({statusCode: statusCode, message: message}));
|
||||
response.end();
|
||||
}
|
||||
var remoteErrorResponse = function (response, statusCode, message) {
|
||||
response.writeHead(200, {'Content-Type': 'application/json'});
|
||||
response.write(JSON.stringify({statusCode: statusCode, message: message}));
|
||||
response.end();
|
||||
};
|
||||
|
||||
/**
|
||||
* Decompresses a byte array using the decompression method passed by type.
|
||||
* It supports gunzip and deflate
|
||||
* */
|
||||
function decompress(body, type) {
|
||||
var deferred = new Deferred();
|
||||
var decompress = function (body, type) {
|
||||
var deferred = new Deferred();
|
||||
|
||||
if (type === 'gzip') {
|
||||
zlib.gunzip(body, function (err, data) {
|
||||
if (!err) {
|
||||
deferred.resolve({
|
||||
body: data.toString(charset),
|
||||
compression: 'gzip'
|
||||
});
|
||||
} else {
|
||||
deferred.reject('Error found: can\'t gunzip content ' + err);
|
||||
}
|
||||
});
|
||||
} else if (type === 'deflate') {
|
||||
zlib.inflateRaw(body, function (err, data) {
|
||||
if (!err) {
|
||||
deferred.resolve({
|
||||
body: data.toString(charset),
|
||||
compression: 'deflate'}
|
||||
);
|
||||
} else {
|
||||
deferred.reject('Error found: can\'t deflate content' + err);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
process.nextTick(function () {
|
||||
deferred.reject("Unknown content encoding: " + type);
|
||||
});
|
||||
}
|
||||
if (type === 'gzip') {
|
||||
zlib.gunzip(body, function (err, data) {
|
||||
if (!err) {
|
||||
deferred.resolve({
|
||||
body: data.toString(charset),
|
||||
compression: 'gzip'
|
||||
});
|
||||
} else {
|
||||
deferred.reject('Error found: can\'t gunzip content ' + err);
|
||||
}
|
||||
});
|
||||
} else if (type === 'deflate') {
|
||||
zlib.inflateRaw(body, function (err, data) {
|
||||
if (!err) {
|
||||
deferred.resolve({
|
||||
body: data.toString(charset),
|
||||
compression: 'deflate'
|
||||
}
|
||||
);
|
||||
} else {
|
||||
deferred.reject('Error found: can\'t deflate content' + err);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
process.nextTick(function () {
|
||||
deferred.reject('Unknown content encoding: ' + type);
|
||||
});
|
||||
}
|
||||
|
||||
return deferred.promise;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the body of a pages and decompresses if needed
|
||||
* */
|
||||
function getBody(res, body) {
|
||||
var deferred = new Deferred();
|
||||
if (res.headers['content-encoding']) {
|
||||
return decompress(body, res.headers['content-encoding']);
|
||||
} else {
|
||||
process.nextTick(function () {
|
||||
if (body) {
|
||||
deferred.resolve({
|
||||
body: body.toString(charset),
|
||||
compression: 'none'});
|
||||
} else {
|
||||
deferred.reject('Error found: Empty body');
|
||||
}
|
||||
});
|
||||
}
|
||||
return deferred.promise;
|
||||
}
|
||||
return deferred.promise;
|
||||
};
|
||||
|
||||
/**
|
||||
* Launches and returns an array with the promises of all the non parallel tests
|
||||
* (browser detection, css prefixes, etc.)
|
||||
* */
|
||||
function launchNonParallelTests(promisesArray, website) {
|
||||
var deferred = new Deferred();
|
||||
var launchNonParallelTests = function (promisesArray, website) {
|
||||
var deferred = new Deferred();
|
||||
|
||||
process.nextTick(function () {
|
||||
process.nextTick(function () {
|
||||
|
||||
tests.forEach(function (test) {
|
||||
if (!test.parallel) {
|
||||
promisesArray.push(test.check(website));
|
||||
}
|
||||
});
|
||||
tests.forEach(function (test) {
|
||||
if (!test.parallel) {
|
||||
promisesArray.push(test.check(website));
|
||||
}
|
||||
});
|
||||
|
||||
deferred.resolve(promisesArray);
|
||||
});
|
||||
deferred.resolve(promisesArray);
|
||||
});
|
||||
|
||||
return deferred.promise;
|
||||
}
|
||||
return deferred.promise;
|
||||
};
|
||||
|
||||
/**
|
||||
* Since several tests need HTML/JS/CSS content, fetch it all at once
|
||||
* before calling any of the tests. Note that the tests still could
|
||||
* retrieve additional content async, since they return a promise.
|
||||
*/
|
||||
function analyze(data, content, res) {
|
||||
var start = Date.now(),
|
||||
promisesTests = [];
|
||||
// function analyze(data, content, res) {
|
||||
// var start = Date.now(),
|
||||
// promisesTests = [];
|
||||
|
||||
var website = {
|
||||
url: url.parse(data.uri),
|
||||
auth: data.auth,
|
||||
content: content.body,
|
||||
compression: content.compression,
|
||||
$: cheerio.load(content.body, { lowerCaseTags: true, lowerCaseAttributeNames: true })
|
||||
};
|
||||
// var website = {
|
||||
// url: url.parse(data.uri),
|
||||
// auth: data.auth,
|
||||
// content: content.body,
|
||||
// compression: content.compression,
|
||||
// $: cheerio.load(content.body, { lowerCaseTags: true, lowerCaseAttributeNames: true })
|
||||
// };
|
||||
|
||||
tests.forEach(function (test) {
|
||||
if (test.parallel) {
|
||||
promisesTests.push(test.check(website));
|
||||
}
|
||||
});
|
||||
// tests.forEach(function (test) {
|
||||
// if (test.parallel) {
|
||||
// promisesTests.push(test.check(website));
|
||||
// }
|
||||
// });
|
||||
|
||||
cssLoader.loadCssFiles(website)
|
||||
.then(jsLoader.loadjsFiles)
|
||||
.then(launchNonParallelTests.bind(null, promisesTests))
|
||||
.then(promises.all)
|
||||
.then(sendResults.bind(website, res, start), sendInternalServerError.bind(website, res));
|
||||
}
|
||||
// cssLoader.loadCssFiles(website)
|
||||
// .then(jsLoader.loadjsFiles)
|
||||
// .then(launchNonParallelTests.bind(null, promisesTests))
|
||||
// .then(promises.all)
|
||||
// .then(sendResults.bind(website, res, start), sendInternalServerError.bind(website, res));
|
||||
// }
|
||||
|
||||
/**
|
||||
* Gets the body of a pages and decompresses if needed
|
||||
* */
|
||||
var getBody = function (request, website) {
|
||||
//var deferred = new Deferred();
|
||||
console.log(website.url);
|
||||
return request.getAsync(website.url)
|
||||
.then(function (result) {
|
||||
var res = result[0];
|
||||
var body = result[1];
|
||||
|
||||
if(!body){
|
||||
return bluebird.reject('Error found: Empty body');
|
||||
}
|
||||
|
||||
website.url = url.parse(res.request.href); //In case there have been redirects
|
||||
if (res.headers['content-encoding']) {
|
||||
website.compression = res.headers['content-encoding'];
|
||||
}
|
||||
|
||||
website.content = body.toString(charset);
|
||||
website.$ = cheerio.load(website.content, {lowerCaseTags: true, lowerCaseAttributeNames: true});
|
||||
console.log('body loaded');
|
||||
return bluebird.resolve(website);
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Handler for the request to get the body of a page and start all the process
|
||||
* */
|
||||
function processResponse(response, auth) {
|
||||
return function (err, res, body) {
|
||||
if (!err && res.statusCode === 200) {
|
||||
getBody(res, body)
|
||||
.then(function (result) {
|
||||
analyze({uri: res.request.href, auth: auth}, result, response);
|
||||
}, remoteErrorResponse.bind(null, response, res.statusCode));
|
||||
} else {
|
||||
remoteErrorResponse(response, res ? res.statusCode : 'No response', 'Error found: ' + err);
|
||||
}
|
||||
};
|
||||
}
|
||||
// function processResponse(response, auth) {
|
||||
// return function (err, res, body) {
|
||||
// if (!err && res.statusCode === 200) {
|
||||
// getBody(res, body)
|
||||
// .then(function (result) {
|
||||
// analyze({uri: res.request.href, auth: auth}, result, response);
|
||||
// }, remoteErrorResponse.bind(null, response, res.statusCode));
|
||||
// } else {
|
||||
// remoteErrorResponse(response, res ? res.statusCode : 'No response', 'Error found: ' + err);
|
||||
// }
|
||||
// };
|
||||
// }
|
||||
|
||||
/**
|
||||
* Returns the local scan page
|
||||
* */
|
||||
function returnMainPage(response) {
|
||||
fs.readFile(path.join(__dirname, "lib", "index.html"), function (err, data) {
|
||||
if (!err) {
|
||||
response.writeHeader(200, {"Content-Type": "text/html"});
|
||||
var returnMainPage = function (response) {
|
||||
fs.readFile(path.join(__dirname, 'lib', 'index.html'), function (err, data) {
|
||||
if (!err) {
|
||||
response.writeHeader(200, {'Content-Type': 'text/html'});
|
||||
|
||||
} else {
|
||||
response.writeHeader(500, {"Content-Type": "text/plain"});
|
||||
data = "Server error: " + err + "\n";
|
||||
}
|
||||
response.write(data);
|
||||
response.end();
|
||||
});
|
||||
}
|
||||
} else {
|
||||
response.writeHeader(500, {'Content-Type': 'text/plain'});
|
||||
data = 'Server error: ' + err + '\n';
|
||||
}
|
||||
response.write(data);
|
||||
response.end();
|
||||
});
|
||||
};
|
||||
|
||||
var getDomain = function (req, res) {
|
||||
res.writeHeader(200, {'Content-Type': 'text/plain'});
|
||||
res.write(process.env.USERDOMAIN || process.env.USERDNSDOMAIN);
|
||||
res.end();
|
||||
};
|
||||
|
||||
/**
|
||||
* Decides what action needs to be done: show the main page or analyze a website
|
||||
* */
|
||||
function handleRequest(req, response) {
|
||||
if (req.url === '/') {
|
||||
// Return the "local scan" page
|
||||
returnMainPage(response);
|
||||
return;
|
||||
}
|
||||
var handleRequest = function (req, response) {
|
||||
if (req.url === '/') {
|
||||
// Return the "local scan" page
|
||||
returnMainPage(response);
|
||||
return;
|
||||
}
|
||||
|
||||
var requestUrl = url.parse(req.url),
|
||||
parameters = querystring.parse(requestUrl.query),
|
||||
urlToAnalyze = sanitize(decodeURIComponent(parameters.url)).xss(),
|
||||
user = sanitize(decodeURIComponent(parameters.user)).xss(),
|
||||
password = sanitize(decodeURIComponent(parameters.password)).xss(),
|
||||
auth;
|
||||
var requestUrl = url.parse(req.url),
|
||||
parameters = querystring.parse(requestUrl.query),
|
||||
urlToAnalyze = sanitize(decodeURIComponent(parameters.url)).xss(),
|
||||
user = sanitize(decodeURIComponent(parameters.user)).xss(),
|
||||
password = sanitize(decodeURIComponent(parameters.password)).xss(),
|
||||
domain = sanitize(decodeURIComponent(parameters.domain)).xss(),
|
||||
auth;
|
||||
|
||||
// If the request gave a user/pass, send it along. Wait for 401 response before sending passwords.
|
||||
if (user !== "undefined" && password !== "undefined") {
|
||||
auth = {
|
||||
'user': user,
|
||||
'pass': password,
|
||||
'sendImmediately': false
|
||||
};
|
||||
request(urlToAnalyze,
|
||||
{auth: auth},
|
||||
processResponse(response, auth));
|
||||
} else {
|
||||
request(urlToAnalyze, processResponse(response));
|
||||
}
|
||||
}
|
||||
var website = {
|
||||
url: urlToAnalyze,
|
||||
auth: null, // We might not need this field
|
||||
$: null,
|
||||
content: '',
|
||||
compression: ''
|
||||
};
|
||||
|
||||
var request;
|
||||
|
||||
// If the request gave a user/pass, send it along. Wait for 401 response before sending passwords.
|
||||
if (user !== 'undefined' && password !== 'undefined') {
|
||||
auth = {
|
||||
'user': user,
|
||||
'pass': password,
|
||||
'sendImmediately': false
|
||||
};
|
||||
|
||||
request = requester(auth);
|
||||
website.auth = auth;
|
||||
// request(urlToAnalyze,
|
||||
// {auth: auth},
|
||||
// processResponse(response, auth));
|
||||
} else {
|
||||
request = requester();
|
||||
}
|
||||
|
||||
website.request = request;
|
||||
|
||||
var start = Date.now(),
|
||||
promisesTests = [];
|
||||
|
||||
tests.forEach(function (test) {
|
||||
if (test.parallel) {
|
||||
promisesTests.push(test.check(website));
|
||||
}
|
||||
});
|
||||
|
||||
getBody(request, website)
|
||||
.then(function (web) {
|
||||
console.log('CSSLoader');
|
||||
return cssLoader.loadCssFiles(web);
|
||||
})
|
||||
.then(function (web) {
|
||||
console.log('JS loader');
|
||||
return jsLoader.loadjsFiles(web);
|
||||
})
|
||||
.then(launchNonParallelTests.bind(null, promisesTests))
|
||||
.then(promises.all)
|
||||
.then(function (results) {
|
||||
sendResults(response, start, results);
|
||||
}, function (error) {
|
||||
sendInternalServerError(error, response);
|
||||
});
|
||||
// request(urlToAnalyze, function(err, res, body){
|
||||
// if (!err && res.statusCode === 200) {
|
||||
// getBody(res, body)
|
||||
// .then(function (result) {
|
||||
// analyze({uri: res.request.href}, result, response);
|
||||
// }, remoteErrorResponse.bind(null, response, res.statusCode));
|
||||
// } else {
|
||||
// remoteErrorResponse(response, res ? res.statusCode : 'No response', 'Error found: ' + err);
|
||||
// }
|
||||
// });
|
||||
};
|
||||
|
||||
/**
|
||||
* Handles the content of a package sent via any of the plugins
|
||||
* */
|
||||
function handlePackage(req, res) {
|
||||
if (!req.body.js || !req.body.css || !req.body.html || !req.body.url) {
|
||||
remoteErrorResponse(res, 400, "Missing information");
|
||||
}
|
||||
var start = Date.now(),
|
||||
cssPromises = [],
|
||||
website;
|
||||
var handlePackage = function (req, res) {
|
||||
if (!req.body.js || !req.body.css || !req.body.html || !req.body.url) {
|
||||
remoteErrorResponse(res, 400, 'Missing information');
|
||||
}
|
||||
var start = Date.now(),
|
||||
cssPromises = [],
|
||||
website;
|
||||
|
||||
//TODO: try/catch this
|
||||
try {
|
||||
website = {
|
||||
url: req.body.url ? url.parse(req.body.url.replace(/"/g, '')) : "http://privates.ite",
|
||||
content: req.body.html,
|
||||
css: null,
|
||||
js: JSON.parse(req.body.js),
|
||||
$: cheerio.load(req.body.html, { lowerCaseTags: true, lowerCaseAttributeNames: true })
|
||||
};
|
||||
} catch (e) {
|
||||
sendBadRequest(res);
|
||||
return;
|
||||
}
|
||||
|
||||
var remoteCSS = JSON.parse(req.body.css);
|
||||
remoteCSS.forEach(function (parsedCSS) {
|
||||
if (parsedCSS.content !== '') {
|
||||
cssPromises.push(cssLoader.parseCSS(parsedCSS.content, parsedCSS.url, null, null, website));
|
||||
}
|
||||
});
|
||||
//TODO: try/catch this
|
||||
try {
|
||||
website = {
|
||||
url: req.body.url ? url.parse(req.body.url.replace(/"/g, '')) : 'http://privates.ite',
|
||||
content: req.body.html,
|
||||
css: null,
|
||||
js: JSON.parse(req.body.js),
|
||||
$: cheerio.load(req.body.html, {lowerCaseTags: true, lowerCaseAttributeNames: true})
|
||||
};
|
||||
} catch (e) {
|
||||
sendBadRequest(res);
|
||||
return;
|
||||
}
|
||||
|
||||
promised.all(cssPromises)
|
||||
.then(function (results) {
|
||||
var cssResults = [],
|
||||
promisesTests = [];
|
||||
var remoteCSS = JSON.parse(req.body.css);
|
||||
remoteCSS.forEach(function (parsedCSS) {
|
||||
if (parsedCSS.content !== '') {
|
||||
cssPromises.push(cssLoader.parseCSS(parsedCSS.content, parsedCSS.url, null, null, website));
|
||||
}
|
||||
});
|
||||
|
||||
cssResults.concat.apply(cssResults, results);
|
||||
website.css = cssResults;
|
||||
promised.all(cssPromises)
|
||||
.then(function (results) {
|
||||
var cssResults = [],
|
||||
promisesTests = [];
|
||||
|
||||
for (var i = 0; i < tests.length; i++) {
|
||||
// Call each test and save its returned promise
|
||||
promisesTests.push(tests[i].check(website));
|
||||
}
|
||||
cssResults.concat.apply(cssResults, results);
|
||||
website.css = cssResults;
|
||||
|
||||
promises.all(promisesTests)
|
||||
.then(sendResults.bind(null, res, start));
|
||||
});
|
||||
}
|
||||
for (var i = 0; i < tests.length; i++) {
|
||||
// Call each test and save its returned promise
|
||||
promisesTests.push(tests[i].check(website));
|
||||
}
|
||||
|
||||
promises.all(promisesTests)
|
||||
.then(sendResults.bind(null, res, start));
|
||||
});
|
||||
};
|
||||
|
||||
// ## CORS middleware
|
||||
//
|
||||
// see: http://stackoverflow.com/questions/7067966/how-to-allow-cors-in-express-nodejs
|
||||
var allowCrossDomain = function (req, res, next) {
|
||||
res.header('Access-Control-Allow-Origin', '*');
|
||||
res.header('Access-Control-Allow-Methods', 'POST');
|
||||
res.header('Access-Control-Allow-Headers', 'Content-Type, Authorization');
|
||||
res.header('Access-Control-Allow-Origin', '*');
|
||||
res.header('Access-Control-Allow-Methods', 'POST');
|
||||
res.header('Access-Control-Allow-Headers', 'Content-Type, Authorization');
|
||||
|
||||
// intercept OPTIONS method
|
||||
if ('OPTIONS' == req.method) {
|
||||
res.send(204);
|
||||
}
|
||||
else {
|
||||
next();
|
||||
}
|
||||
// intercept OPTIONS method
|
||||
if (req.method === 'OPTIONS') {
|
||||
res.send(204);
|
||||
}
|
||||
else {
|
||||
next();
|
||||
}
|
||||
};
|
||||
app.use(allowCrossDomain);
|
||||
|
||||
app.use(bodyParser.json());
|
||||
app.get('/', handleRequest);
|
||||
app.get('/domain', getDomain);
|
||||
app.post('/package', handlePackage);
|
||||
app.listen(port);
|
||||
|
||||
console.log('Server started on port ' + port);
|
||||
console.log('To scan a private url go to http://localhost:' + port + '/ and follow the instructions');
|
||||
|
||||
module.exports.port = port;
|
||||
module.exports.port = port;
|
||||
|
|
@ -23,7 +23,7 @@
|
|||
|
||||
var Deferred = require('promised-io').Deferred,
|
||||
Promise = require('promised-io/promise'),
|
||||
request = require('request'),
|
||||
// request = require('request'),
|
||||
rules = [
|
||||
"navigator.userAgent",
|
||||
"navigator.appVersion",
|
||||
|
|
@ -43,13 +43,13 @@ var Deferred = require('promised-io').Deferred,
|
|||
"protoaculous"
|
||||
];
|
||||
|
||||
request = request.defaults({
|
||||
jar: false,
|
||||
proxy: process.env.HTTP_PROXY || process.env.http_proxy,
|
||||
headers: {
|
||||
'Accept-Language': 'en-US,en;q=0.5',
|
||||
'User-Agent': 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)'}
|
||||
});
|
||||
// request = request.defaults({
|
||||
// jar: false,
|
||||
// proxy: process.env.HTTP_PROXY || process.env.http_proxy,
|
||||
// headers: {
|
||||
// 'Accept-Language': 'en-US,en;q=0.5',
|
||||
// 'User-Agent': 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)'}
|
||||
// });
|
||||
|
||||
function checkScript(script) {
|
||||
var deferred = new Deferred();
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ function checkVersion(library, version) {
|
|||
return vinfo;
|
||||
}
|
||||
|
||||
var libraries = [
|
||||
var libraries = [
|
||||
{
|
||||
name: "Prototype",
|
||||
minVersions: [
|
||||
|
|
@ -87,7 +87,7 @@ var libraries = [
|
|||
minVersions: [
|
||||
{ major: "1.2.", minor: "6" },
|
||||
{ major: "1.4.", minor: "5" },
|
||||
{ major: "1.5.", minor: "" }
|
||||
{ major: "1.5.", minor: "1" }
|
||||
],
|
||||
check: function (scriptText) {
|
||||
var version = scriptText.match(/this.MooTools\s*=\s*\{version:\s*'(\d+\.\d+\.\d+)/m);
|
||||
|
|
@ -107,7 +107,7 @@ var libraries = [
|
|||
{
|
||||
name: "jQuery Form Plugin",
|
||||
minVersions: [
|
||||
{ major: "3.", minor: "22" }
|
||||
{ major: "3.", minor: "51" }
|
||||
],
|
||||
check: function (scriptText) {
|
||||
var version = scriptText.match(/Form Plugin\s+\*\s+version: (\d+\.\d+)/m);
|
||||
|
|
@ -119,7 +119,8 @@ var libraries = [
|
|||
minVersions: [
|
||||
{ major: "2.5.", minor: "2" },
|
||||
{ major: "2.6.", minor: "2" },
|
||||
{ major: "2.7.", minor: "1" }
|
||||
{ major: "2.7.", minor: "2" },
|
||||
{ major: "2.8.", minor: "3" }
|
||||
],
|
||||
check: function (scriptText) {
|
||||
// Static analysis. :( The version is set as a local variable, far from
|
||||
|
|
|
|||
|
|
@ -16,162 +16,126 @@
|
|||
* and limitations under the License.
|
||||
*/
|
||||
|
||||
"use strict";
|
||||
'use strict';
|
||||
|
||||
var request = require('request'),
|
||||
CSSLintRules = ['auto-imports'],
|
||||
CSSLint = require('./csslint.js').CSSLint,
|
||||
promised = require("promised-io/promise"),
|
||||
Deferred = require('promised-io').Deferred,
|
||||
url = require('url'),
|
||||
cssPromises;
|
||||
var CSSLintRules = ['auto-imports'],
|
||||
CSSLint = require('./csslint.js').CSSLint,
|
||||
bluebird = require('bluebird'),
|
||||
url = require('url'),
|
||||
cssPromises;
|
||||
|
||||
request = request.defaults({
|
||||
jar: false,
|
||||
proxy: process.env.HTTP_PROXY || process.env.http_proxy,
|
||||
headers: {
|
||||
'Accept-Language': 'en-US,en;q=0.5',
|
||||
'User-Agent': 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)'}});
|
||||
// parseCSS and parseCSSfromUrl call each other so we tell eslint to cool it
|
||||
/*eslint-disable no-use-before-define */
|
||||
|
||||
// parseCSS and parseCSSfromUrl call each other so we tell jshint to cool it
|
||||
/*jshint latedef: false*/
|
||||
var parseCSS = function (text, cssUrl, media, isInline, website) {
|
||||
var report,
|
||||
imports = [];
|
||||
|
||||
function parseCSSfromUrl(cssUrl, media, website) {
|
||||
var deferred = new Deferred(),
|
||||
params = {
|
||||
uri: cssUrl,
|
||||
headers: {'Accept': 'text/html, application/xhtml+xml, */*'}
|
||||
},
|
||||
auth = website.auth;
|
||||
if (auth) {
|
||||
params.auth = auth;
|
||||
}
|
||||
report = CSSLint.verify(text);
|
||||
imports.push({cssUrl: isInline ? 'embed' : cssUrl, media: media, cssBody: text, report: report});
|
||||
report.messages.forEach(function (result) {
|
||||
if (CSSLintRules.indexOf(result.rule.id) !== -1) {
|
||||
result.message.forEach(function (cssImport) {
|
||||
//resolve url here
|
||||
var importUrl = url.resolve(cssUrl, cssImport.url);
|
||||
if (website.cssParsedUrls.indexOf(importUrl) === -1) {
|
||||
imports.push(parseCSSfromUrl(importUrl, cssImport.media, website));
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
website.cssParsedUrls.push(cssUrl);
|
||||
request(params, function (error, response, body) {
|
||||
if (!error && response.statusCode === 200) {
|
||||
parseCSS(body, cssUrl, media, false, website).then(function (results) {
|
||||
deferred.resolve(results);
|
||||
});
|
||||
} else {
|
||||
console.warn("Request for " + cssUrl + " returned " + error);
|
||||
// Silently skip the troublesome file
|
||||
deferred.resolve([]);
|
||||
}
|
||||
}
|
||||
);
|
||||
return bluebird.all(imports).then(function (results) {
|
||||
// Results may be different because @imports may have @imported other files (ad nauseum);
|
||||
// the .concat() will flatten one level of nested arrays which is all we need
|
||||
return bluebird.resolve([].concat.apply([], results));
|
||||
});
|
||||
};
|
||||
|
||||
return deferred.promise;
|
||||
}
|
||||
var parseCSSfromUrl = function (cssUrl, media, website) {
|
||||
console.log(cssUrl);
|
||||
website.cssParsedUrls.push(cssUrl);
|
||||
return website.request.getAsync(cssUrl)
|
||||
.then(function (result) {
|
||||
var body = result[1].toString('utf-8');
|
||||
return parseCSS(body, cssUrl, media, false, website);
|
||||
})
|
||||
.catch(function (error) {
|
||||
console.warn('Request for ' + cssUrl + ' returned ' + error);
|
||||
// Silently skip the troublesome file
|
||||
return bluebird.resolve([]);
|
||||
});
|
||||
};
|
||||
|
||||
CSSLint.addRule({
|
||||
id: "auto-imports",
|
||||
name: "auto imports",
|
||||
desc: "downloads css imports",
|
||||
browsers: "All",
|
||||
init: function (parser, reporter) {
|
||||
var rule = this,
|
||||
localImports = [];
|
||||
id: 'auto-imports',
|
||||
name: 'auto imports',
|
||||
desc: 'downloads css imports',
|
||||
browsers: 'All',
|
||||
init: function (parser, reporter) {
|
||||
var rule = this,
|
||||
localImports = [];
|
||||
|
||||
parser.addListener('startstylesheet', function(){
|
||||
localImports = [];
|
||||
});
|
||||
parser.addListener('startstylesheet', function () {
|
||||
localImports = [];
|
||||
});
|
||||
|
||||
parser.addListener('import', function importFound(event) {
|
||||
localImports.push({
|
||||
url: event.uri.replace('url(', '').replace(')', ''),
|
||||
media: event.media
|
||||
});
|
||||
});
|
||||
parser.addListener('import', function importFound(event) {
|
||||
localImports.push({
|
||||
url: event.uri.replace('url(', '').replace(')', ''),
|
||||
media: event.media
|
||||
});
|
||||
});
|
||||
|
||||
parser.addListener('endstylesheet', function fileEnded() {
|
||||
if(localImports.length) {
|
||||
parser.addListener('endstylesheet', function fileEnded() {
|
||||
if (localImports.length) {
|
||||
reporter.report(localImports, 0, 0, rule, '');
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
function parseCSS(text, cssUrl, media, isInline, website) {
|
||||
var report,
|
||||
deferred = new Deferred(),
|
||||
imports = [];
|
||||
//
|
||||
// auth = website.auth;
|
||||
var check = function (website) {
|
||||
var cssLinks = website.$('link[rel="stylesheet"]'),
|
||||
cssHref,
|
||||
cssUrl,
|
||||
cssTags;
|
||||
|
||||
process.nextTick(function () {
|
||||
report = CSSLint.verify(text);
|
||||
imports.push({cssUrl: isInline ? "embed" : cssUrl, media: media, cssBody: text, report: report});
|
||||
report.messages.forEach(function (result) {
|
||||
if(CSSLintRules.indexOf(result.rule.id) !== -1){
|
||||
result.message.forEach(function(cssImport){
|
||||
//resolve url here
|
||||
var importUrl = url.resolve(cssUrl, cssImport.url);
|
||||
if (website.cssParsedUrls.indexOf(importUrl) === -1) {
|
||||
imports.push(parseCSSfromUrl(importUrl, cssImport.media, website));
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
cssPromises = [];
|
||||
website.css = [];
|
||||
website.cssParsedUrls = [];
|
||||
|
||||
promised.all(imports).then(function (results) {
|
||||
// Results may be different because @imports may have @imported other files (ad nauseum);
|
||||
// the .concat() will flatten one level of nested arrays which is all we need
|
||||
deferred.resolve([].concat.apply([], results));
|
||||
});
|
||||
});
|
||||
for (var i = 0; i < cssLinks.length; i++) {
|
||||
cssHref = cssLinks[i].attribs.href;
|
||||
|
||||
return deferred.promise;
|
||||
}
|
||||
// If the attributes don't have a space then the library doesn't parse it correctly.
|
||||
if (cssHref) {
|
||||
cssUrl = url.resolve(website.url, cssHref);
|
||||
if (website.cssParsedUrls.indexOf(cssUrl) === -1) {
|
||||
cssPromises.push(parseCSSfromUrl(cssUrl, cssLinks[i].attribs.media, website));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function check(website) {
|
||||
var deferred = new Deferred(),
|
||||
cssLinks = website.$('link[rel="stylesheet"]'),
|
||||
cssHref,
|
||||
cssUrl,
|
||||
cssTags;
|
||||
cssTags = website.$('style');
|
||||
for (i = 0; i < cssTags.length; i++) {
|
||||
if (cssTags[i].children && cssTags[i].children.length > 0 && cssTags[i].children[0].data && cssTags[i].children[0].data !== '') {
|
||||
cssPromises.push(parseCSS(cssTags[i].children[0].data, website.url, cssTags[i].attribs.media, true, website));
|
||||
}
|
||||
}
|
||||
|
||||
cssPromises = [];
|
||||
website.css = [];
|
||||
website.cssParsedUrls = [];
|
||||
|
||||
for (var i = 0; i < cssLinks.length; i++) {
|
||||
cssHref = cssLinks[i].attribs.href;
|
||||
|
||||
// If the attributes don't have a space then the library doesn't parse it correctly.
|
||||
if (cssHref) {
|
||||
cssUrl = url.resolve(website.url, cssHref);
|
||||
if (website.cssParsedUrls.indexOf(cssUrl) === -1) {
|
||||
cssPromises.push(parseCSSfromUrl(cssUrl, cssLinks[i].attribs.media, website));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cssTags = website.$('style');
|
||||
for (i = 0; i < cssTags.length; i++) {
|
||||
if (cssTags[i].children && cssTags[i].children.length > 0 && cssTags[i].children[0].data && cssTags[i].children[0].data !== '') {
|
||||
cssPromises.push(parseCSS(cssTags[i].children[0].data, website.url, cssTags[i].attribs.media, true, website));
|
||||
}
|
||||
}
|
||||
|
||||
if (cssPromises.length > 0) {
|
||||
promised.all(cssPromises).then(function (array) {
|
||||
// Flatten the nested arrays
|
||||
array = [].concat.apply([], array);
|
||||
website.css = array;
|
||||
deferred.resolve(website);
|
||||
}, function () {
|
||||
deferred.reject();
|
||||
});
|
||||
} else {
|
||||
// No style sheets
|
||||
process.nextTick(function(){
|
||||
deferred.resolve(website);
|
||||
});
|
||||
}
|
||||
|
||||
return deferred.promise;
|
||||
}
|
||||
if (cssPromises.length > 0) {
|
||||
return bluebird.all(cssPromises).then(function (array) {
|
||||
// Flatten the nested arrays
|
||||
array = [].concat.apply([], array);
|
||||
website.css = array;
|
||||
return bluebird.resolve(website);
|
||||
});
|
||||
} else {
|
||||
// No style sheets
|
||||
return bluebird.resolve(website);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
module.exports.parseCSS = parseCSS;
|
||||
|
|
|
|||
|
|
@ -16,90 +16,61 @@
|
|||
* and limitations under the License.
|
||||
*/
|
||||
|
||||
"use strict";
|
||||
'use strict';
|
||||
|
||||
var request = require('request'),
|
||||
promised = require("promised-io/promise"),
|
||||
Deferred = require('promised-io').Deferred,
|
||||
url = require('url');
|
||||
var bluebird = require('bluebird'),
|
||||
url = require('url');
|
||||
|
||||
request = request.defaults({
|
||||
jar: false,
|
||||
proxy: process.env.HTTP_PROXY || process.env.http_proxy,
|
||||
headers: {
|
||||
'Accept-Language': 'en-US,en;q=0.5',
|
||||
'User-Agent': 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)'}});
|
||||
var downloadJS = function (jsUrl, jsHref, website) {
|
||||
console.log(jsUrl);
|
||||
return website.request.getAsync(jsUrl)
|
||||
.then(function (result) {
|
||||
var response = result[0];
|
||||
var body = result[1].toString('utf-8');
|
||||
return bluebird.resolve({url: url, jsUrl: jsHref, finalUrl: response.request.href, content: body});
|
||||
});
|
||||
};
|
||||
|
||||
function downloadJS(jsUrl, jsHref, auth) {
|
||||
var jsDeferred = new Deferred(),
|
||||
parameters = {uri: jsUrl,
|
||||
headers: {
|
||||
'Accept': 'text/html, application/xhtml+xml, */*'}};
|
||||
var check = function (website) {
|
||||
var jsLinks = website.$('script'),
|
||||
js = [],
|
||||
jsPromises = [];
|
||||
|
||||
if (auth) {
|
||||
parameters.auth = auth;
|
||||
}
|
||||
website.js = [];
|
||||
|
||||
request(parameters, function (error, response, body) {
|
||||
if (!error && response.statusCode === 200) {
|
||||
response.on('error', function (e) {
|
||||
console.log('loading CSS' + e);
|
||||
});
|
||||
jsDeferred.resolve({url: url, jsUrl: jsHref, finalUrl: response.request.href, content: body});
|
||||
} else {
|
||||
jsDeferred.resolve({});
|
||||
}
|
||||
});
|
||||
for (var i = 0; i < jsLinks.length; i++) {
|
||||
var jsHref = jsLinks[i].attribs.src,
|
||||
jsUrl;
|
||||
|
||||
return jsDeferred.promise;
|
||||
}
|
||||
if (jsHref) {
|
||||
if (jsHref) {
|
||||
jsUrl = url.resolve(website.url, jsHref);
|
||||
jsPromises.push(downloadJS(jsUrl, jsHref, website));
|
||||
}
|
||||
} else if (jsLinks[i].children[0] && jsLinks[i].children[0].data) {
|
||||
// Some <script> tags that do not contain anything. We ignore those
|
||||
js.push({jsUrl: 'embed', content: jsLinks[i].children[0].data});
|
||||
}
|
||||
}
|
||||
|
||||
function check(website) {
|
||||
var deferred = new Deferred(),
|
||||
jsLinks = website.$('script'),
|
||||
js = [],
|
||||
jsPromises = [];
|
||||
if (jsPromises.length > 0) {
|
||||
return bluebird.all(jsPromises)
|
||||
.then(function (array) {
|
||||
for (i = 0; i < array.length; i++) {
|
||||
if (array[i].finalUrl) {
|
||||
js.push(array[i]);
|
||||
}
|
||||
}
|
||||
|
||||
website.js = [];
|
||||
website.js = js;
|
||||
|
||||
for (var i = 0; i < jsLinks.length; i++) {
|
||||
var jsHref = jsLinks[i].attribs.src,
|
||||
jsUrl;
|
||||
|
||||
if (jsHref) {
|
||||
if (jsHref) {
|
||||
jsUrl = url.resolve(website.url, jsHref);
|
||||
jsPromises.push(downloadJS(jsUrl, jsHref, website.auth));
|
||||
}
|
||||
} else if (jsLinks[i].children[0] && jsLinks[i].children[0].data) {
|
||||
// Some <script> tags that do not contain anything. We ignore those
|
||||
js.push({jsUrl: 'embed', content: jsLinks[i].children[0].data});
|
||||
}
|
||||
}
|
||||
|
||||
if (jsPromises.length > 0) {
|
||||
promised.all(jsPromises).then(function (array) {
|
||||
for (i = 0; i < array.length; i++) {
|
||||
if (array[i].finalUrl) {
|
||||
js.push(array[i]);
|
||||
}
|
||||
}
|
||||
|
||||
website.js = js;
|
||||
|
||||
deferred.resolve(website);
|
||||
}, function () {
|
||||
deferred.reject();
|
||||
});
|
||||
} else {
|
||||
// There aren't any external JS but we could have embedded JS
|
||||
process.nextTick(function(){
|
||||
website.js = js;
|
||||
deferred.resolve(website);
|
||||
});
|
||||
}
|
||||
|
||||
return deferred.promise;
|
||||
}
|
||||
return bluebird.resolve(website);
|
||||
});
|
||||
} else {
|
||||
// There aren't any external JS but we could have embedded JS
|
||||
website.js = js;
|
||||
return bluebird.resolve(website);
|
||||
}
|
||||
};
|
||||
|
||||
module.exports.loadjsFiles = check;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,80 @@
|
|||
'use strict';
|
||||
|
||||
var request = require('request'),
|
||||
httpntlm = require('httpntlm'),
|
||||
http = require('http'),
|
||||
bluebird = require('bluebird');
|
||||
|
||||
var internalRequest = request.defaults({
|
||||
jar: false,
|
||||
proxy: process.env.HTTP_PROXY || process.env.http_proxy,
|
||||
headers: {
|
||||
'Accept-Language': 'en-US,en;q=0.5',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko'
|
||||
}
|
||||
});
|
||||
|
||||
var authInfos = {
|
||||
type: "none",
|
||||
username: "",
|
||||
password: "",
|
||||
domain: "",
|
||||
};
|
||||
|
||||
// ## HTTP Headers lowercase error
|
||||
//
|
||||
// see: http://stackoverflow.com/questions/20643699/how-do-i-send-uppercase-headers-in-http
|
||||
var automaticHeaders = {
|
||||
connection: true,
|
||||
'content-length': true,
|
||||
'transfer-encoding': true,
|
||||
date: true
|
||||
};
|
||||
http.OutgoingMessage.prototype.setHeader = function (name, value) {
|
||||
if (arguments.length < 2) {
|
||||
throw new Error('`name` and `value` are required for setHeader().');
|
||||
}
|
||||
|
||||
if (this._header) {
|
||||
throw new Error('Can\'t set headers after they are sent.');
|
||||
}
|
||||
|
||||
// NO LOWER CASE
|
||||
var key = name;//.toLowerCase();
|
||||
this._headers = this._headers || {};
|
||||
this._headerNames = this._headerNames || {};
|
||||
this._headers[key] = value;
|
||||
this._headerNames[key] = name;
|
||||
|
||||
if (automaticHeaders[key]) {
|
||||
if (this._removedHeader) {
|
||||
this._removedHeader[key] = false;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
module.exports = function(auth){
|
||||
if(!auth || auth.type !== 'ntlm') {
|
||||
var defaultRequest = request.defaults({
|
||||
followAllRedirects: true,
|
||||
encoding: null,
|
||||
auth: auth,
|
||||
gzip: true,
|
||||
jar: false,
|
||||
proxy: process.env.HTTP_PROXY || process.env.http_proxy,
|
||||
headers: {
|
||||
'Accept': 'text/html, application/xhtml+xml, */*',
|
||||
'Accept-Encoding': 'gzip,deflate',
|
||||
'Accept-Language': 'en-US,en;q=0.5',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko' /* Set IE11 UA */
|
||||
}
|
||||
});
|
||||
|
||||
return bluebird.promisifyAll(defaultRequest);
|
||||
}
|
||||
};
|
||||
|
||||
// exports.authInfos = authInfos;
|
||||
// exports.defaultRequest = defaultRequest;
|
||||
// exports.internalRequest = internalRequest;
|
||||
|
|
@ -24,12 +24,14 @@
|
|||
"test": "grunt test"
|
||||
},
|
||||
"dependencies": {
|
||||
"bluebird": "^2.9.30",
|
||||
"body-parser": "^1.13.1",
|
||||
"cheerio": "^0.19.0",
|
||||
"cssom": ">=0.2.5",
|
||||
"express": "^4.12.4",
|
||||
"express": "^4.12.4",
|
||||
"httpntlm": "^1.5.2",
|
||||
"parserlib": "~0.2.3",
|
||||
"promised-io": ">=0.3.0",
|
||||
"promised-io": "^0.3.5",
|
||||
"request": "^2.58.0",
|
||||
"validator": "~0.4.22",
|
||||
"xml2js": ">=0.2.0",
|
||||
|
|
@ -46,4 +48,4 @@
|
|||
"csslint",
|
||||
"scanner"
|
||||
]
|
||||
}
|
||||
}
|
||||
Загрузка…
Ссылка в новой задаче