Merge branch 'url-expander' into develop

config/default.yml

@@ -49,11 +49,6 @@ proxy:
   # SSL certificate path.
   sslCertPath: ./keys/crt.pem
 
-  # Whether or not the proxy will follow redirects
-  # itself or pass them back to the client.
-  followRedirects: false
-
-
 # PAC (proxy auto config) server settings.
 pac:
   # PAC server listen port.
@@ -147,6 +142,14 @@ adblock:
   optional: true
   listUrl: 'http://pgl.yoyo.org/adservers/serverlist.php?&mimetype=plaintext'
 
+# URL expander settings (resolves shortened URLs).
+urlexpander:
+  enabled: false
+  optional: true
+  # Number of redirections after which the request will be directly forwarded.
+  maxRedirect: 5
+  hostsFile: 'config/urlshorteners.txt'
+
 # XZ compression settings.
 xz:
   enabled: true

config/urlshorteners.txt

@@ -0,0 +1,24 @@
+bit.ly
+chkit.in
+digs.by
+fb.me
+feedproxy.google.com
+ff.im
+goo.gl
+is.gd
+is.gd
+ow.ly
+p.ly
+ping.fm
+snipurl.com
+snurl.com
+t.co
+tcrn.ch
+tiny.cc
+tinyurl.com
+trib.al
+twurl.nl
+u.nu
+ur1.ca
+wp.me
+wrd.cm

lib/plugins/plugins.yaml

@@ -1,6 +1,6 @@
 # Request plugins go here, and are serviced in the order given.
 request:
-  [adblock, cache, dom/gif2video]
+  [urlexpander, adblock, cache, dom/gif2video]
 
 # Response plugins go here, and the response pipeline is
 # constructed in the order given.

lib/plugins/urlexpander.js

@@ -0,0 +1,68 @@
+'use strict';
+
+var url = require('url');
+var http = require('http');
+var config = require('config');
+var fs = require('fs');
+
+var log = require('../log');
+
+var NAME = exports.name = 'urlexpander';
+
+var metrics = require('../metrics').session(NAME);
+
+var hosts = {};
+
+exports.init = function() {
+  fs.readFile(config.urlexpander.hostsFile, function(err, data) {
+    if (err) {
+      log.error('Unable to read URL shorteners list:',
+                config.urlexpander.hostsFile);
+      return;
+    }
+    var lines = data.toString().split('\n');
+    lines.forEach(function(line) {
+      var shortener = line.trim();
+      if (shortener !== '') {
+        hosts[shortener] = true;
+      }
+    });
+  });
+};
+
+function sendNewLocation(request, response, callback) {
+  if (request.redirected) {
+    response.writeHead(302, '', { location: request.url });
+    response.end();
+    callback(null, true);
+  } else {
+    callback(null, false);
+  }
+}
+
+function resolve(request, response, redirectCount, callback) {
+  var currentUrl = url.parse(request.url);
+
+  if (redirectCount < config.urlexpander.maxRedirect &&
+      currentUrl.hostname in hosts) {
+    http.get(request.url, function(res) {
+      // We need to consume the data so the socket is not kept open.
+      res.on('data', function() {});
+
+      if (res.statusCode >= 300 && res.statusCode < 400) {
+        request.url = res.headers.location;
+        request.redirected = true;
+        metrics.count('hit');
+        resolve(request, response, redirectCount + 1, callback);
+      } else {
+        sendNewLocation(request, response, callback);
+      }
+    });
+  } else {
+    sendNewLocation(request, response, callback);
+  }
+}
+
+exports.handleRequest = function(request, response, options, callback) {
+  resolve(request, response, 0, callback);
+};

lib/proxy.js

@@ -205,11 +205,6 @@ var SpdyProxy = function(options) {
     }
   }
 
-  if (options.proxy.followRedirects) {
-    // Resolve redirects before responding.
-    http = require('follow-redirects').http;
-  }
-
   spdy.server.Server.call(this, {
     key: fs.readFileSync(options.proxy.sslKeyPath),
     cert: fs.readFileSync(options.proxy.sslCertPath)

package.json

@@ -26,7 +26,6 @@
   "dependencies": {
     "cheerio": "^0.17.0",
     "config": "^0.4.36",
-    "follow-redirects": "0.0.3",
     "hiredis": "^0.1.17",
     "iconv": "^2.1.4",
     "imagetype": "^0.3.0",

test/helper/testHelper.js

@@ -47,7 +47,7 @@ exports.setupLocalServer = function(path, cb) {
   localServer.stderr.on('data', function() {});
 };
 
-exports.cleanAll = function() {
+exports.cleanAll = function(done) {
   // unload the proxy
   if (proxy) {
     proxy.close();
@@ -57,8 +57,15 @@ exports.cleanAll = function() {
 
   // kill the local webserver
   if (localServer !== null) {
+    if (done) {
+      localServer.on('exit', function() {
+        done();
+      });
+    }
     localServer.kill('SIGINT');
     localServer = null;
+  } else if (done) {
+    done();
   }
 };
 

test/tests/gif2video.js

@@ -15,8 +15,8 @@ module.exports = {
         helper.setupLocalServer('test/helper/content/gif2video/', done);
       },
 
-      after: function() {
-        helper.cleanAll();
+      after: function(done) {
+        helper.cleanAll(done);
       },
 
       testConversion: function(done) {

test/tests/imgcompression.js

@@ -44,12 +44,12 @@ module.exports = {
         helper.setupLocalServer('test/helper/content/imgs/', done);
       },
 
-      after: function() {
+      after: function(done) {
         console.log('\n\n\tTotal image compression: %d% (%dKB -> %dKB).',
                     (100 - (compressedSize * 100) / baseSize).toFixed(2),
                     Math.round(util.byteToKb(baseSize)),
                     Math.round(util.byteToKb(compressedSize)));
-        helper.cleanAll();
+        helper.cleanAll(done);
       },
     },
   },

test/tests/urlexpander.js

@@ -0,0 +1,97 @@
+'use strict';
+
+var rewire = require('rewire');
+var http = require('http');
+var url = require('url');
+
+var helper = require('../helper/testHelper');
+var DummyRequest = require('../helper/dummyRequest');
+var DummyResponse = require('../helper/dummyResponse');
+var urlExpander = rewire('../../lib/plugins/urlexpander.js');
+var config = require('config').test;
+
+require('chai').should();
+
+var server = null;
+
+function requestHandler(request, response) {
+  var requestedUrl = url.parse(request.url);
+
+  if (requestedUrl.pathname === 'destination') {
+    response.writeHead(200);
+    response.write('Hello!');
+  } else if (requestedUrl.pathname === '/simpleredirect') {
+    response.writeHead(302, { location: helper.getLocalUrl('destination') });
+  } else if (requestedUrl.pathname === '/multipleredirect1') {
+    response.writeHead(302,
+                       { location: helper.getLocalUrl('multipleredirect2') });
+  } else if (requestedUrl.pathname === '/multipleredirect2') {
+    response.writeHead(302, { location: helper.getLocalUrl('destination') });
+  } else if (requestedUrl.pathname === '/loopredirect1') {
+    response.writeHead(302, { location: helper.getLocalUrl('loopredirect2') });
+  } else if (requestedUrl.pathname === '/loopredirect2') {
+    response.writeHead(302, { location: helper.getLocalUrl('loopredirect1') });
+  } else if (requestedUrl.pathname === '/external') {
+    response.writeHead(302, { location: 'http://example.com/' });
+  } else {
+    response.writeHead(404);
+  }
+
+  response.end();
+}
+
+module.exports = {
+  'url expander': {
+    before: function() {
+      urlExpander.__get__('hosts')['127.0.0.1'] = true;
+      server = http.createServer(requestHandler);
+      server.listen(config.localServer.port);
+    },
+
+    after: function() {
+      server.close();
+    },
+
+    'follow simple redirect': function(done) {
+      var req = new DummyRequest(helper.getLocalUrl('simpleredirect'));
+      var res = new DummyResponse();
+
+      urlExpander.handleRequest(req, res, null, function(err, handled) {
+        handled.should.equal(true);
+        done();
+      });
+    },
+
+    'circular redirects': function(done) {
+      var req = new DummyRequest(helper.getLocalUrl('loopredirect1'));
+      var res = new DummyResponse();
+
+      urlExpander.handleRequest(req, res, null, function() {
+        // it just has to stop
+        done();
+      });
+    },
+
+    'external redirect': function(done) {
+      var req = new DummyRequest(helper.getLocalUrl('external'));
+      var res = new DummyResponse();
+
+      urlExpander.handleRequest(req, res, null, function(err, handled) {
+        handled.should.equal(true);
+        done();
+      });
+    },
+
+    'unlisted domain': function(done) {
+      var reqUrl = 'http://localhost:' + config.localServer.port +
+                   '/simpleredirect';
+      var req = new DummyRequest(reqUrl);
+      var res = new DummyResponse();
+
+      urlExpander.handleRequest(req, res, null, function(err, handled) {
+        handled.should.equal(false);
+        done();
+      });
+    },
+  },
+};