Merge pull request #787 from k88hudson/updating
Refactor cache updating strategy
This commit is contained in:
Kate Hudson
Коммит
ccab2f2fc3
|
|
@ -5,7 +5,6 @@ const {Cu} = require("chrome");
|
|||
const ss = require("sdk/simple-storage");
|
||||
const simplePrefs = require("sdk/simple-prefs");
|
||||
const self = require("sdk/self");
|
||||
const {setTimeout, clearTimeout} = require("sdk/timers");
|
||||
const {TippyTopProvider} = require("lib/TippyTopProvider");
|
||||
|
||||
const EMBEDLY_PREF = "embedly.endpoint";
|
||||
|
|
@ -32,7 +31,6 @@ const DEFAULT_OPTIONS = {
|
|||
cacheCleanupPeriod: 86400000, // a cache clearing job runs at most once every 24 hours
|
||||
cacheRefreshAge: 259200000, // refresh a link every 3 days
|
||||
cacheTTL: 2592000000, // cached items expire if they haven't been accessed in 30 days
|
||||
cacheUpdateInterval: 3600000, // a cache update job runs every hour
|
||||
proxyMaxLinks: 25, // number of links embedly proxy accepts per request
|
||||
initFresh: false,
|
||||
};
|
||||
|
|
@ -43,11 +41,9 @@ function PreviewProvider(tabTracker, options = {}) {
|
|||
this._tippyTopProvider = new TippyTopProvider();
|
||||
this._tabTracker = tabTracker;
|
||||
this.init();
|
||||
this._runPeriodicUpdate();
|
||||
}
|
||||
|
||||
PreviewProvider.prototype = {
|
||||
_updateTimeout: null,
|
||||
|
||||
/**
|
||||
* Clean-up the preview cache
|
||||
|
|
@ -89,13 +85,10 @@ PreviewProvider.prototype = {
|
|||
break;
|
||||
case ENABLED_PREF:
|
||||
if (this.enabled) {
|
||||
// if disabling, remove cache and update
|
||||
this._clearPeriodicUpdate();
|
||||
this.clearCache();
|
||||
} else {
|
||||
// if enabling, create cache and start updating
|
||||
// if enabling, create cache
|
||||
ss.storage.embedlyData = {};
|
||||
this._runPeriodicUpdate();
|
||||
}
|
||||
this.enabled = simplePrefs.prefs[ENABLED_PREF];
|
||||
break;
|
||||
|
|
@ -223,13 +216,30 @@ PreviewProvider.prototype = {
|
|||
return results;
|
||||
},
|
||||
|
||||
/**
|
||||
* Determine if a cached link has expired
|
||||
*/
|
||||
isLinkExpired(link) {
|
||||
const cachedLink = ss.storage.embedlyData[link.cacheKey];
|
||||
if (!cachedLink) {
|
||||
return false;
|
||||
}
|
||||
let currentTime = Date.now();
|
||||
return (currentTime - cachedLink.refreshTime) > this.options.cacheRefreshAge;
|
||||
},
|
||||
|
||||
/**
|
||||
* Request links from embedly, optionally filtering out known links
|
||||
*/
|
||||
asyncSaveLinks: Task.async(function*(links, event = {}, newOnly = true, updateAccessTime = true) {
|
||||
let linksList = this._uniqueLinks(links)
|
||||
.filter(link => link)
|
||||
// If a request is in progress, don't re-request it
|
||||
.filter(link => !this._alreadyRequested.has(link.cacheKey))
|
||||
// If we already have the link in the cache, don't request it again...
|
||||
// ... UNLESS it has expired
|
||||
.filter(link => !newOnly || !ss.storage.embedlyData[link.cacheKey] || this.isLinkExpired(link));
|
||||
|
||||
// optionally filter out known links, and links which already have a request in process
|
||||
let linksList = this._uniqueLinks(links).filter(link => link && (!newOnly || !ss.storage.embedlyData[link.cacheKey]) && !this._alreadyRequested.has(link.cacheKey));
|
||||
linksList.forEach(link => this._alreadyRequested.add(link.cacheKey));
|
||||
|
||||
let requestQueue = [];
|
||||
|
|
@ -245,35 +255,6 @@ PreviewProvider.prototype = {
|
|||
yield Promise.all(promises);
|
||||
}),
|
||||
|
||||
/**
|
||||
* Asynchronously update links
|
||||
*/
|
||||
asyncUpdateLinks: Task.async(function*() {
|
||||
let links = [];
|
||||
let currentTime = Date.now();
|
||||
for (let key in ss.storage.embedlyData) {
|
||||
let link = ss.storage.embedlyData[key];
|
||||
if (!link.refreshTime || (currentTime - link.refreshTime) > this.options.cacheRefreshAge) {
|
||||
links.push(link);
|
||||
}
|
||||
}
|
||||
const event = this._tabTracker.generateEvent({source: "UPDATE_LINKS"});
|
||||
let linksToUpdate = this.processLinks(links);
|
||||
yield this.asyncSaveLinks(linksToUpdate, event, false, false);
|
||||
Services.obs.notifyObservers(null, "activity-streams-preview-cache-update", null);
|
||||
}),
|
||||
|
||||
/**
|
||||
* Runs a periodic update
|
||||
*/
|
||||
_runPeriodicUpdate() {
|
||||
this._updateTimeout = setTimeout(() => {
|
||||
this.asyncUpdateLinks().then(() => {
|
||||
this._runPeriodicUpdate();
|
||||
});
|
||||
}, this.options.cacheUpdateInterval);
|
||||
},
|
||||
|
||||
/**
|
||||
* Makes the necessary requests to embedly to get data for each link
|
||||
*/
|
||||
|
|
@ -358,20 +339,10 @@ PreviewProvider.prototype = {
|
|||
delete ss.storage.embedlyData;
|
||||
},
|
||||
|
||||
/**
|
||||
* Clear update timeout
|
||||
*/
|
||||
_clearPeriodicUpdate() {
|
||||
if (this._updateTimeout) {
|
||||
clearTimeout(this._updateTimeout);
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Uninit the preview provider
|
||||
*/
|
||||
uninit() {
|
||||
this._clearPeriodicUpdate();
|
||||
simplePrefs.off("", this._onPrefChange);
|
||||
this._alreadyRequested = new Set();
|
||||
},
|
||||
|
|
|
|||
|
|
@ -104,72 +104,6 @@ exports.test_periodic_cleanup = function*(assert) {
|
|||
gPreviewProvider.options.cacheCleanupPeriod = oldThreshold;
|
||||
};
|
||||
|
||||
exports.test_periodic_update = function*(assert) {
|
||||
let oldTimeout = gPreviewProvider.options.cacheUpdateInterval;
|
||||
gPreviewProvider.options.cacheUpdateInterval = 10;
|
||||
|
||||
// cycle enabled pref to reset timeouts
|
||||
simplePrefs.prefs["previews.enabled"] = false;
|
||||
simplePrefs.prefs["previews.enabled"] = true;
|
||||
|
||||
let countingUpdatePromise = new Promise(resolve => {
|
||||
let notif = "activity-streams-preview-cache-update";
|
||||
let count = 0;
|
||||
let waitForNotif = (subject, topic, data) => {
|
||||
if (topic === notif) {
|
||||
count++;
|
||||
if (count === 3) {
|
||||
Services.obs.removeObserver(waitForNotif, notif);
|
||||
resolve(count);
|
||||
}
|
||||
}
|
||||
};
|
||||
Services.obs.addObserver(waitForNotif, notif);
|
||||
});
|
||||
let count = yield countingUpdatePromise;
|
||||
assert.equal(count, 3, "update called expected number of times");
|
||||
gPreviewProvider.options.cacheUpdateInterval = oldTimeout;
|
||||
};
|
||||
|
||||
exports.test_update_links = function*(assert) {
|
||||
let currentTime = Date.now();
|
||||
let fourDaysAgo = currentTime - (4 * 24 * 60 * 60 * 1000);
|
||||
ss.storage.embedlyData["example.com/1"] = {url: "http://example.com/1", accessTime: fourDaysAgo, sanitizedURL: "http://example.com/1", cacheKey: "example.com/1"};
|
||||
ss.storage.embedlyData["example.com/2"] = {url: "http://example.com/2", accessTime: fourDaysAgo, refreshTime: fourDaysAgo, sanitizedURL: "http://example.com/2", cacheKey: "example.com/2"};
|
||||
ss.storage.embedlyData["example.com/3"] = {url: "http://example.com/3", accessTime: currentTime, refreshTime: currentTime, sanitizedURL: "http://example.com/3", cacheKey: "example.com/3"};
|
||||
|
||||
assert.ok(gPreviewProvider._embedlyEndpoint, "The embedly endpoint is set");
|
||||
let srv = httpd.startServerAsync(gPort);
|
||||
|
||||
srv.registerPathHandler("/embedlyLinkData", function handle(request, response) {
|
||||
let data = JSON.parse(
|
||||
NetUtil.readInputStreamToString(
|
||||
request.bodyInputStream,
|
||||
request.bodyInputStream.available()
|
||||
)
|
||||
);
|
||||
data.urls.forEach(link => assert.notEqual(link, null, "there are no null links"));
|
||||
let urls = {};
|
||||
for (let url of data.urls) {
|
||||
urls[url] = {"embedlyMetaData": "some embedly metadata"};
|
||||
}
|
||||
response.setHeader("Content-Type", "application/json", false);
|
||||
response.write(JSON.stringify({urls}));
|
||||
});
|
||||
yield gPreviewProvider.asyncUpdateLinks();
|
||||
|
||||
assert.equal(ss.storage.embedlyData["example.com/1"].accessTime, fourDaysAgo, "link 1 access time is unchanged");
|
||||
assert.ok(ss.storage.embedlyData["example.com/1"].refreshTime, "link 1 refresh time is set");
|
||||
assert.equal(ss.storage.embedlyData["example.com/2"].accessTime, fourDaysAgo, "link 2 access time is unchanged");
|
||||
assert.notEqual(ss.storage.embedlyData["example.com/2"].refreshTime, fourDaysAgo, "link 2 refresh time is updated");
|
||||
assert.equal(ss.storage.embedlyData["example.com/3"].accessTime, currentTime, "link 3 access time is unchanged");
|
||||
assert.equal(ss.storage.embedlyData["example.com/3"].refreshTime, currentTime, "link 3 refresh time is unchanged");
|
||||
|
||||
yield new Promise(resolve => {
|
||||
srv.stop(resolve);
|
||||
});
|
||||
};
|
||||
|
||||
exports.test_only_request_links_once = function*(assert) {
|
||||
const msg1 = [{"url": "a.com", "sanitizedURL": "a.com", "cacheKey": "a.com"},
|
||||
{"url": "b.com", "sanitizedURL": "b.com", "cacheKey": "b.com"},
|
||||
|
|
@ -207,6 +141,49 @@ exports.test_only_request_links_once = function*(assert) {
|
|||
});
|
||||
};
|
||||
|
||||
exports.test_is_link_expired = function(assert) {
|
||||
const refreshTime = Date.now() - (gPreviewProvider.options.cacheRefreshAge + 1000);
|
||||
ss.storage.embedlyData["a.com"] = {"url": "a.com", "sanitizedURL": "a.com", "cacheKey": "a.com", refreshTime};
|
||||
assert.equal(gPreviewProvider.isLinkExpired({cacheKey: "a.com"}), true, "expired link should return true");
|
||||
ss.storage.embedlyData["b.com"] = {"url": "b.com", "sanitizedURL": "b.com", "cacheKey": "b.com", refreshTime: new Date()};
|
||||
assert.equal(gPreviewProvider.isLinkExpired({cacheKey: "b.com"}), false, "non-expired link should return false");
|
||||
};
|
||||
|
||||
exports.test_request_links_if_expired = function*(assert) {
|
||||
const oldTime = Date.now() - (gPreviewProvider.options.cacheRefreshAge + 1000);
|
||||
const links = [{"url": "a.com", "sanitizedURL": "a.com", "cacheKey": "a.com"},
|
||||
{"url": "b.com", "sanitizedURL": "b.com", "cacheKey": "b.com"},
|
||||
{"url": "c.com", "sanitizedURL": "c.com", "cacheKey": "c.com"}];
|
||||
links.forEach(link => {
|
||||
ss.storage.embedlyData[link.cacheKey] = Object.assign({}, link, {refreshTime: new Date()});
|
||||
});
|
||||
ss.storage.embedlyData["a.com"].refreshTime = oldTime;
|
||||
|
||||
assert.ok(gPreviewProvider._embedlyEndpoint, "The embedly endpoint is set");
|
||||
let srv = httpd.startServerAsync(gPort);
|
||||
|
||||
let urlsRequested = [];
|
||||
srv.registerPathHandler("/embedlyLinkData", function handle(request, response) {
|
||||
let data = JSON.parse(
|
||||
NetUtil.readInputStreamToString(
|
||||
request.bodyInputStream,
|
||||
request.bodyInputStream.available()
|
||||
)
|
||||
);
|
||||
data.urls.forEach(url => urlsRequested.push(url));
|
||||
response.setHeader("Content-Type", "application/json", false);
|
||||
response.write(JSON.stringify({"urls": {urlsRequested}}));
|
||||
});
|
||||
|
||||
yield gPreviewProvider.asyncSaveLinks(links);
|
||||
|
||||
assert.deepEqual(urlsRequested, ["a.com"], "we should only request the expired URL");
|
||||
|
||||
yield new Promise(resolve => {
|
||||
srv.stop(resolve);
|
||||
});
|
||||
};
|
||||
|
||||
exports.test_filter_urls = function*(assert) {
|
||||
const fakeData = {
|
||||
get validLinks() {
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче