mozilla
/
webmaker-screenshot
зеркало из https://github.com/mozilla/webmaker-screenshot.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

RedisCache methods now take options objects.

This commit is contained in:
Atul Varma 2015-01-23 09:28:27 -05:00
Родитель 01d4c3a705
Коммит 74164de4b9
4 изменённых файлов: 165 добавлений и 85 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

47
app.js
Просмотреть файл

@ -90,7 +90,6 @@ app.get('/js/bundle.js', function(req, res, next) {
app.post('/', function(req, res, next) {
var url = makes.validateAndNormalizeUrl(req.body.url);
var wait = req.body.wait ? EXTENDED_WAIT : DEFAULT_WAIT;
var cacheFunc = cacheScreenshot.bind(null, wait);
var key;
if (!url)
@ -98,11 +97,15 @@ app.post('/', function(req, res, next) {
key = keys.fromMakeUrl(url);
redisCache.lockAndSet(key, cacheFunc, function done(err, info) {
if (err) return next(err);
if (info.status != 302)
return res.send(400, {error: info.reason});
return res.send({screenshot: info.url});
redisCache.lockAndSet({
key: key,
cache: cacheScreenshot.bind(null, wait),
done: function(err, info) {
if (err) return next(err);
if (info.status != 302)
return res.send(400, {error: info.reason});
return res.send({screenshot: info.url});
}
});
});
@ -111,22 +114,26 @@ app.use(function(req, res, next) {
if (!(req.method == 'GET' && keys.isWellFormed(key)))
return next();
redisCache.get(key, function cache(key, cb) {
var s3url = S3_WEBSITE + key;
redisCache.get({
key: key,
cache: function(key, cb) {
var s3url = S3_WEBSITE + key;
request.head(s3url, function(err, s3res) {
if (err) return cb(err);
if (s3res.statusCode == 200)
return cb(null, {status: 302, url: s3url});
request.head(s3url, function(err, s3res) {
if (err) return cb(err);
if (s3res.statusCode == 200)
return cb(null, {status: 302, url: s3url});
cacheScreenshot(DEFAULT_WAIT, key, cb);
});
}, function done(err, info) {
if (err) return next(err);
if (info.status == 404) return next();
if (info.status == 302)
return res.redirect(info.url);
return next(new Error("invalid status: " + info.status));
cacheScreenshot(DEFAULT_WAIT, key, cb);
});
},
done: function(err, info) {
if (err) return next(err);
if (info.status == 404) return next();
if (info.status == 302)
return res.redirect(info.url);
return next(new Error("invalid status: " + info.status));
}
});
});

1
package.json
Просмотреть файл

@ -5,6 +5,7 @@
"dependencies": {
"body-parser": "~1.2.0",
"redis": "0.12.1",
"underscore": "1.7.0",
"browserify": "8.1.1",
"express": "~4.2.0",
"request": "2.34.0"

31
redis-cache.js
Просмотреть файл

@ -1,5 +1,6 @@
var redis = require('redis');
var urlParse = require("url").parse;
var _ = require('underscore');
var redis = require('redis');
var LOCK_EXPIRY_IN_SECONDS = 10;
var INFO_EXPIRY_IN_SECONDS = 60 * 60;
@ -36,15 +37,22 @@ RedisCache.prototype = {
_getInfoKey: function(baseKey) {
return this.prefix + "info_" + baseKey
},
lockAndSet: function(baseKey, cacheCb, doneCb, retryMethod) {
lockAndSet: function(options) {
options = _.defaults(options || {}, {
retry: this.lockAndSet
});
var self = this;
var baseKey = options.key;
var cacheCb = options.cache;
var doneCb = options.done;
var retryCb = options.retry;
var setTimeout = options.setTimeout || global.setTimeout;
var infoKey = this._getInfoKey(baseKey);
var lockToken = Math.random().toString();
var lockKey = this.prefix + "lock_" + baseKey;
retryMethod = retryMethod || this.lockAndSet;
if (typeof(retryMethod) != 'function')
throw new Error('retryMethod is not a function');
if (typeof(retryCb) != 'function')
throw new Error('retry is not a function');
self.client.set([
lockKey, lockToken, "NX", "EX",
@ -52,9 +60,7 @@ RedisCache.prototype = {
], function(err, result) {
if (err) return doneCb(err);
if (result === null) {
setTimeout(retryMethod.bind(self, baseKey, cacheCb, doneCb,
retryMethod),
self.unlockWaitMs);
setTimeout(retryCb.bind(self, options), self.unlockWaitMs);
} else {
var releaseArgs = [RELEASE_LOCK_SCRIPT, "1", lockKey, lockToken];
cacheCb(baseKey, function(err, info) {
@ -74,8 +80,11 @@ RedisCache.prototype = {
}
});
},
get: function(baseKey, cacheCb, doneCb) {
get: function(options) {
options = options || {};
var self = this;
var baseKey = options.key;
var doneCb = options.done;
var infoKey = self._getInfoKey(baseKey);
self.client.get(infoKey, function(err, info) {
@ -88,7 +97,9 @@ RedisCache.prototype = {
}
return doneCb(null, info);
} else {
return self.lockAndSet(baseKey, cacheCb, doneCb, self.get);
return self.lockAndSet(_.extend({}, options, {
retry: self.get
}));
}
});
}

171
test/redis-cache.test.js
Просмотреть файл

@ -7,12 +7,14 @@ function noCache() {
throw new Error("this should never be called");
}
var UNLOCK_WAIT_MS = 10;
describe("RedisCache", function() {
var cache, client;
beforeEach(function() {
client = redis.createClient();
cache = new RedisCache(client, 'TESTING_', 10);
cache = new RedisCache(client, 'TESTING_', UNLOCK_WAIT_MS);
});
afterEach(function(done) {
@ -23,25 +25,64 @@ describe("RedisCache", function() {
});
describe("get()", function() {
it("should use cached value if available", function(done) {
cache.get("blop", function cache(key, cb) {
cb(null, {thing: "yay " + key});
}, function(err, info) {
if (err) return done(err);
cache.get("blop", noCache, function(err, info) {
it("should retry", function(done) {
var retrySecondGet;
cache.get({
key: "bip",
cache: function(key, firstCacheDoneCb) {
cache.get({
key: "bip",
cache: noCache,
setTimeout: function(cb, ms) {
retrySecondGet = cb;
process.nextTick(firstCacheDoneCb.bind(null, null, "sup"));
},
done: function(err, info) {
if (err) return done(err);
info.should.eql("sup");
done();
}
});
},
done: function(err) {
if (err) return done(err);
info.should.eql({thing: "yay blop"});
done();
});
process.nextTick(retrySecondGet);
}
});
});
it("should use cached value if available", function(done) {
cache.get({
key: "blop",
cache: function(key, cb) {
cb(null, {thing: "yay " + key});
},
done: function(err, info) {
if (err) return done(err);
cache.get({
key: "blop",
cache: noCache,
done: function(err, info) {
if (err) return done(err);
info.should.eql({thing: "yay blop"});
done();
}
});
}
});
});
it("should report JSON errors", function(done) {
client.set("TESTING_info_blah", "sup", function(err) {
if (err) return done(err);
cache.get("blah", noCache, function(err, info) {
err.toString().should.match(/unexpected token/i);
done();
cache.get({
key: "blah",
cache: noCache,
done: function(err, info) {
err.toString().should.match(/unexpected token/i);
done();
}
});
});
});
@ -49,60 +90,80 @@ describe("RedisCache", function() {
describe("lockAndSet()", function() {
it("should pass arguments as expected", function(done) {
cache.lockAndSet("foo", function cache(key, cb) {
key.should.eql("foo");
process.nextTick(cb.bind(null, null, {thing: "bar"}));
}, function(err, info) {
if (err) return done(err);
info.should.eql({thing: "bar"});
done();
cache.lockAndSet({
key: "foo",
cache: function(key, cb) {
key.should.eql("foo");
process.nextTick(cb.bind(null, null, {thing: "bar"}));
},
done: function(err, info) {
if (err) return done(err);
info.should.eql({thing: "bar"});
done();
}
});
});
it("should hold lock while caching, release after", function(done) {
cache.lockAndSet("bar", function cache(key, cb) {
client.get("TESTING_lock_bar", function(err, val) {
cache.lockAndSet({
key: "bar",
cache: function(key, cb) {
client.get("TESTING_lock_bar", function(err, val) {
if (err) return done(err);
should.exist(val);
cb(null, "yay");
});
},
done: function(err, info) {
if (err) return done(err);
should.exist(val);
cb(null, "yay");
});
}, function(err, info) {
if (err) return done(err);
client.get("TESTING_lock_bar", function(err, val) {
if (err) return done(err);
should.not.exist(val);
done();
});
client.get("TESTING_lock_bar", function(err, val) {
if (err) return done(err);
should.not.exist(val);
done();
});
}
});
});
it("should release lock after caching error", function(done) {
cache.lockAndSet("bar", function cache(key, cb) {
process.nextTick(cb.bind(null, new Error("blah")));
}, function(err, info) {
err.message.should.eql("blah");
client.get("TESTING_lock_bar", function(err, val) {
if (err) return done(err);
should.not.exist(val);
done();
});
cache.lockAndSet({
key: "bar",
cache: function(key, cb) {
process.nextTick(cb.bind(null, new Error("blah")));
},
done: function(err, info) {
err.message.should.eql("blah");
client.get("TESTING_lock_bar", function(err, val) {
if (err) return done(err);
should.not.exist(val);
done();
});
}
});
});
it("should call retryMethod when lock is taken", function(done) {
cache.lockAndSet("foo", function firstCache(key, cb) {
var secondDone = function(err, info) {
throw new Error("this should not be called");
};
var retry = function(key, cacheCb, doneCb, retryMethod) {
key.should.eql("foo");
cacheCb.should.equal(noCache);
doneCb.should.equal(secondDone);
retryMethod.should.equal(retry);
cb(null, "hooray");
};
cache.lockAndSet("foo", noCache, secondDone, retry);
}, done);
it("should call retry when lock is taken", function(done) {
cache.lockAndSet({
key: "foo",
cache: function(key, cb) {
var o = {
key: "foo",
cache: noCache,
done: function(err, info) {
throw new Error("this should not be called");
},
retry: function(options) {
options.key.should.eql("foo");
options.cache.should.equal(o.cache);
options.done.should.equal(o.done);
options.retry.should.equal(o.retry);
cb(null, "hooray");
}
};
cache.lockAndSet(o);
},
done: done
});
});
});
});