From 09bce320bad2c083fd453f1b65327fd65b8771b5 Mon Sep 17 00:00:00 2001 From: Marina Samuel Date: Fri, 9 May 2014 11:24:30 -0400 Subject: [PATCH] Bug 986521 - Fetch Directory Links data from a network location to cache locally. r=adw --- bug986521 | 0 toolkit/modules/DirectoryLinksProvider.jsm | 123 ++++++--- .../xpcshell/test_DirectoryLinksProvider.js | 233 +++++++++++++----- 3 files changed, 259 insertions(+), 97 deletions(-) create mode 100644 bug986521 diff --git a/bug986521 b/bug986521 new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/toolkit/modules/DirectoryLinksProvider.jsm b/toolkit/modules/DirectoryLinksProvider.jsm index 2be6eba75596..8dff9a10a591 100644 --- a/toolkit/modules/DirectoryLinksProvider.jsm +++ b/toolkit/modules/DirectoryLinksProvider.jsm @@ -9,44 +9,21 @@ this.EXPORTED_SYMBOLS = ["DirectoryLinksProvider"]; const Ci = Components.interfaces; const Cc = Components.classes; const Cu = Components.utils; +const XMLHttpRequest = + Components.Constructor("@mozilla.org/xmlextras/xmlhttprequest;1", "nsIXMLHttpRequest"); Cu.import("resource://gre/modules/XPCOMUtils.jsm"); Cu.import("resource://gre/modules/Services.jsm"); XPCOMUtils.defineLazyModuleGetter(this, "NetUtil", "resource://gre/modules/NetUtil.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "OS", + "resource://gre/modules/osfile.jsm") +XPCOMUtils.defineLazyModuleGetter(this, "Promise", + "resource://gre/modules/Promise.jsm"); -/** - * Gets the currently selected locale for display. - * @return the selected locale or "en-US" if none is selected - */ -function getLocale() { - let matchOS; - try { - matchOS = Services.prefs.getBoolPref(PREF_MATCH_OS_LOCALE); - } - catch (e) {} - - if (matchOS) { - return Services.locale.getLocaleComponentForUserAgent(); - } - - try { - let locale = Services.prefs.getComplexValue(PREF_SELECTED_LOCALE, - Ci.nsIPrefLocalizedString); - if (locale) { - return locale.data; - } - } - catch (e) {} - - try { - return Services.prefs.getCharPref(PREF_SELECTED_LOCALE); - } - catch (e) {} - - return "en-US"; -} +// The filename where directory links are stored locally +const DIRECTORY_LINKS_FILE = "directoryLinks.json"; // The preference that tells whether to match the OS locale const PREF_MATCH_OS_LOCALE = "intl.locale.matchOS"; @@ -77,7 +54,7 @@ let DirectoryLinksProvider = { _observers: [], - get _prefs() Object.freeze({ + get _observedPrefs() Object.freeze({ linksURL: PREF_DIRECTORY_SOURCE, matchOSLocale: PREF_MATCH_OS_LOCALE, prefSelectedLocale: PREF_SELECTED_LOCALE, @@ -86,7 +63,7 @@ let DirectoryLinksProvider = { get _linksURL() { if (!this.__linksURL) { try { - this.__linksURL = Services.prefs.getCharPref(this._prefs["linksURL"]); + this.__linksURL = Services.prefs.getCharPref(this._observedPrefs["linksURL"]); } catch (e) { Cu.reportError("Error fetching directory links url from prefs: " + e); @@ -95,11 +72,43 @@ let DirectoryLinksProvider = { return this.__linksURL; }, + /** + * Gets the currently selected locale for display. + * @return the selected locale or "en-US" if none is selected + */ + get locale() { + let matchOS; + try { + matchOS = Services.prefs.getBoolPref(PREF_MATCH_OS_LOCALE); + } + catch (e) {} + + if (matchOS) { + return Services.locale.getLocaleComponentForUserAgent(); + } + + try { + let locale = Services.prefs.getComplexValue(PREF_SELECTED_LOCALE, + Ci.nsIPrefLocalizedString); + if (locale) { + return locale.data; + } + } + catch (e) {} + + try { + return Services.prefs.getCharPref(PREF_SELECTED_LOCALE); + } + catch (e) {} + + return "en-US"; + }, + get linkTypes() LINK_TYPES, observe: function DirectoryLinksProvider_observe(aSubject, aTopic, aData) { if (aTopic == "nsPref:changed") { - if (aData == this._prefs["linksURL"]) { + if (aData == this._observedPrefs["linksURL"]) { delete this.__linksURL; } this._callObservers("onManyLinksChanged"); @@ -107,15 +116,15 @@ let DirectoryLinksProvider = { }, _addPrefsObserver: function DirectoryLinksProvider_addObserver() { - for (let pref in this._prefs) { - let prefName = this._prefs[pref]; + for (let pref in this._observedPrefs) { + let prefName = this._observedPrefs[pref]; Services.prefs.addObserver(prefName, this, false); } }, _removePrefsObserver: function DirectoryLinksProvider_removeObserver() { - for (let pref in this._prefs) { - let prefName = this._prefs[pref]; + for (let pref in this._observedPrefs) { + let prefName = this._observedPrefs[pref]; Services.prefs.removeObserver(prefName, this); } }, @@ -133,7 +142,7 @@ let DirectoryLinksProvider = { let json = NetUtil.readInputStreamToString(aInputStream, aInputStream.available(), {charset: "UTF-8"}); - let locale = getLocale(); + let locale = this.locale; output = JSON.parse(json)[locale]; } catch (e) { @@ -152,6 +161,42 @@ let DirectoryLinksProvider = { } }, + _fetchAndCacheLinks: function DirectoryLinksProvider_fetchAndCacheLinks(uri) { + let deferred = Promise.defer(); + let xmlHttp = new XMLHttpRequest(); + xmlHttp.overrideMimeType("application/json"); + + let self = this; + xmlHttp.onload = function(aResponse) { + let json = this.responseText; + if (this.status && this.status != 200) { + json = "{}"; + } + let directoryLinksFilePath = OS.Path.join(OS.Constants.Path.localProfileDir, DIRECTORY_LINKS_FILE); + OS.File.writeAtomic(directoryLinksFilePath, json, {tmpPath: directoryLinksFilePath + ".tmp"}) + .then(() => { + deferred.resolve(); + self._callObservers("onManyLinksChanged"); + }, + () => { + deferred.reject("Error writing uri data in profD."); + }); + }; + + xmlHttp.onerror = function(e) { + deferred.reject("Fetching " + uri + " results in error code: " + e.target.status); + }; + + try { + xmlHttp.open('POST', uri); + xmlHttp.send(JSON.stringify({ locale: this.locale })); + } catch (e) { + deferred.reject("Error fetching " + uri); + Cu.reportError(e); + } + return deferred.promise; + }, + /** * Gets the current set of directory links. * @param aCallback The function that the array of links is passed to. diff --git a/toolkit/modules/tests/xpcshell/test_DirectoryLinksProvider.js b/toolkit/modules/tests/xpcshell/test_DirectoryLinksProvider.js index b3ca58400889..e77ad87e40b3 100644 --- a/toolkit/modules/tests/xpcshell/test_DirectoryLinksProvider.js +++ b/toolkit/modules/tests/xpcshell/test_DirectoryLinksProvider.js @@ -7,13 +7,60 @@ * This file tests the DirectoryLinksProvider singleton in the DirectoryLinksProvider.jsm module. */ -const { classes: Cc, interfaces: Ci, results: Cr, utils: Cu } = Components; +const { classes: Cc, interfaces: Ci, results: Cr, utils: Cu, Constructor: CC } = Components; Cu.import("resource://gre/modules/Services.jsm"); Cu.import("resource://gre/modules/DirectoryLinksProvider.jsm"); Cu.import("resource://gre/modules/Promise.jsm"); +Cu.import("resource://gre/modules/Http.jsm"); +Cu.import("resource://testing-common/httpd.js"); +Cu.import("resource://gre/modules/osfile.jsm") +Cu.import("resource://gre/modules/XPCOMUtils.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "NetUtil", + "resource://gre/modules/NetUtil.jsm"); +do_get_profile(); + +const DIRECTORY_LINKS_FILE = "directoryLinks.json"; const DIRECTORY_FRECENCY = 1000; -const kTestSource = 'data:application/json,{"en-US": [{"url":"http://example.com","title":"TestSource"}]}'; +const kURLData = {"en-US": [{"url":"http://example.com","title":"LocalSource"}]}; +const kTestURL = 'data:application/json,' + JSON.stringify(kURLData); + +// DirectoryLinksProvider preferences +const kLocalePref = DirectoryLinksProvider._observedPrefs.prefSelectedLocale; +const kSourceUrlPref = DirectoryLinksProvider._observedPrefs.linksURL; + +// httpd settings +var server; +const kDefaultServerPort = 9000; +const kBaseUrl = "http://localhost:" + kDefaultServerPort; +const kExamplePath = "/exampleTest/"; +const kFailPath = "/fail/"; +const kExampleURL = kBaseUrl + kExamplePath; +const kFailURL = kBaseUrl + kFailPath; + +const kHttpHandlerData = {}; +kHttpHandlerData[kExamplePath] = {"en-US": [{"url":"http://example.com","title":"RemoteSource"}]}; + +const bodyData = JSON.stringify({ locale: DirectoryLinksProvider.locale }); +const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1", + "nsIBinaryInputStream", + "setInputStream"); + +function getHttpHandler(path) { + let code = 200; + let body = JSON.stringify(kHttpHandlerData[path]); + if (path == kFailPath) { + code = 204; + } + return function(aRequest, aResponse) { + let bodyStream = new BinaryInputStream(aRequest.bodyInputStream); + do_check_eq(NetUtil.readInputStreamToString(bodyStream, bodyStream.available()), bodyData); + + aResponse.setStatusLine(null, code); + aResponse.setHeader("Content-Type", "application/json"); + aResponse.write(body); + }; +} function isIdentical(actual, expected) { if (expected == null) { @@ -33,20 +80,114 @@ function isIdentical(actual, expected) { } } -function fetchData(provider) { +function fetchData() { let deferred = Promise.defer(); - provider.getLinks(linkData => { + DirectoryLinksProvider.getLinks(linkData => { deferred.resolve(linkData); }); return deferred.promise; } -function run_test() { - run_next_test(); +function readJsonFile(jsonFile = DIRECTORY_LINKS_FILE) { + let decoder = new TextDecoder(); + let directoryLinksFilePath = OS.Path.join(OS.Constants.Path.localProfileDir, jsonFile); + return OS.File.read(directoryLinksFilePath).then(array => { + let json = decoder.decode(array); + return JSON.parse(json); + }, () => { return "" }); } -add_task(function test_DirectoryLinksProvider__linkObservers() { +function cleanJsonFile(jsonFile = DIRECTORY_LINKS_FILE) { + let directoryLinksFilePath = OS.Path.join(OS.Constants.Path.localProfileDir, jsonFile); + return OS.File.remove(directoryLinksFilePath); +} + +// All tests that call setupDirectoryLinksProvider() must also call cleanDirectoryLinksProvider(). +function setupDirectoryLinksProvider(options = {}) { + let linksURL = options.linksURL || kTestURL; + DirectoryLinksProvider.init(); + Services.prefs.setCharPref(kLocalePref, options.locale || "en-US"); + Services.prefs.setCharPref(kSourceUrlPref, linksURL); + do_check_eq(DirectoryLinksProvider._linksURL, linksURL); +} + +function cleanDirectoryLinksProvider() { + DirectoryLinksProvider.reset(); + Services.prefs.clearUserPref(kLocalePref); + Services.prefs.clearUserPref(kSourceUrlPref); +} + +function run_test() { + // Set up a mock HTTP server to serve a directory page + server = new HttpServer(); + server.registerPrefixHandler(kExamplePath, getHttpHandler(kExamplePath)); + server.registerPrefixHandler(kFailPath, getHttpHandler(kFailPath)); + server.start(kDefaultServerPort); + + run_next_test(); + + // Teardown. + do_register_cleanup(function() { + server.stop(function() { }); + }); +} + +add_task(function test_fetchAndCacheLinks_local() { + yield cleanJsonFile(); + // Trigger cache of data or chrome uri files in profD + yield DirectoryLinksProvider._fetchAndCacheLinks(kTestURL); + let data = yield readJsonFile(); + isIdentical(data, kURLData); +}); + +add_task(function test_fetchAndCacheLinks_remote() { + yield cleanJsonFile(); + // this must trigger directory links json download and save it to cache file + yield DirectoryLinksProvider._fetchAndCacheLinks(kExampleURL); + let data = yield readJsonFile(); + isIdentical(data, kHttpHandlerData[kExamplePath]); +}); + +add_task(function test_fetchAndCacheLinks_malformedURI() { + yield cleanJsonFile(); + let someJunk = "some junk"; + try { + yield DirectoryLinksProvider._fetchAndCacheLinks(someJunk); + do_throw("Malformed URIs should fail") + } catch (e) { + do_check_eq(e, "Error fetching " + someJunk) + } + + // File should be empty. + let data = yield readJsonFile(); + isIdentical(data, ""); +}); + +add_task(function test_fetchAndCacheLinks_unknownHost() { + yield cleanJsonFile(); + let nonExistentServer = "http://nosuchhost"; + try { + yield DirectoryLinksProvider._fetchAndCacheLinks(nonExistentServer); + do_throw("BAD URIs should fail"); + } catch (e) { + do_check_true(e.startsWith("Fetching " + nonExistentServer + " results in error code: ")) + } + + // File should be empty. + let data = yield readJsonFile(); + isIdentical(data, ""); +}); + +add_task(function test_fetchAndCacheLinks_non200Status() { + yield cleanJsonFile(); + yield DirectoryLinksProvider._fetchAndCacheLinks(kFailURL); + let data = yield readJsonFile(); + isIdentical(data, {}); +}); + +// To test onManyLinksChanged observer, trigger a fetch +add_task(function test_linkObservers() { let deferred = Promise.defer(); let testObserver = { onManyLinksChanged: function() { @@ -54,21 +195,19 @@ add_task(function test_DirectoryLinksProvider__linkObservers() { } } - let provider = DirectoryLinksProvider; - provider.init(); - provider.addObserver(testObserver); - do_check_eq(provider._observers.length, 1); - Services.prefs.setCharPref(provider._prefs['linksURL'], kTestSource); + DirectoryLinksProvider.init(); + DirectoryLinksProvider.addObserver(testObserver); + do_check_eq(DirectoryLinksProvider._observers.length, 1); + DirectoryLinksProvider._fetchAndCacheLinks(kTestURL); yield deferred.promise; - provider._removeObservers(); - do_check_eq(provider._observers.length, 0); + DirectoryLinksProvider._removeObservers(); + do_check_eq(DirectoryLinksProvider._observers.length, 0); - provider.reset(); - Services.prefs.clearUserPref(provider._prefs['linksURL']); + cleanDirectoryLinksProvider(); }); -add_task(function test_DirectoryLinksProvider__linksURL_locale() { +add_task(function test_linksURL_locale() { let data = { "en-US": [{url: "http://example.com", title: "US"}], "zh-CN": [ @@ -78,25 +217,19 @@ add_task(function test_DirectoryLinksProvider__linksURL_locale() { }; let dataURI = 'data:application/json,' + JSON.stringify(data); - let provider = DirectoryLinksProvider; - Services.prefs.setCharPref(provider._prefs['linksURL'], dataURI); - Services.prefs.setCharPref('general.useragent.locale', 'en-US'); - - // set up the observer - provider.init(); - do_check_eq(provider._linksURL, dataURI); + setupDirectoryLinksProvider({linksURL: dataURI}); let links; let expected_data; - links = yield fetchData(provider); + links = yield fetchData(); do_check_eq(links.length, 1); expected_data = [{url: "http://example.com", title: "US", frecency: DIRECTORY_FRECENCY, lastVisitDate: 1}]; isIdentical(links, expected_data); Services.prefs.setCharPref('general.useragent.locale', 'zh-CN'); - links = yield fetchData(provider); + links = yield fetchData(); do_check_eq(links.length, 2) expected_data = [ {url: "http://example.net", title: "CN", frecency: DIRECTORY_FRECENCY, lastVisitDate: 2}, @@ -104,49 +237,33 @@ add_task(function test_DirectoryLinksProvider__linksURL_locale() { ]; isIdentical(links, expected_data); - provider.reset(); - Services.prefs.clearUserPref('general.useragent.locale'); - Services.prefs.clearUserPref(provider._prefs['linksURL']); + cleanDirectoryLinksProvider(); }); -add_task(function test_DirectoryLinksProvider__prefObserver_url() { - let provider = DirectoryLinksProvider; - Services.prefs.setCharPref('general.useragent.locale', 'en-US'); - Services.prefs.setCharPref(provider._prefs['linksURL'], kTestSource); +add_task(function test_prefObserver_url() { + setupDirectoryLinksProvider({linksURL: kTestURL}); - // set up the observer - provider.init(); - do_check_eq(provider._linksURL, kTestSource); - - let links = yield fetchData(provider); + let links = yield fetchData(); do_check_eq(links.length, 1); - let expectedData = [{url: "http://example.com", title: "TestSource", frecency: DIRECTORY_FRECENCY, lastVisitDate: 1}]; + let expectedData = [{url: "http://example.com", title: "LocalSource", frecency: DIRECTORY_FRECENCY, lastVisitDate: 1}]; isIdentical(links, expectedData); // tests these 2 things: - // 1. observer trigger on pref change - // 2. invalid source url - let exampleUrl = 'http://example.com/bad'; - Services.prefs.setCharPref(provider._prefs['linksURL'], exampleUrl); + // 1. _linksURL is properly set after the pref change + // 2. invalid source url is correctly handled + let exampleUrl = 'http://nosuchhost/bad'; + Services.prefs.setCharPref(kSourceUrlPref, exampleUrl); + do_check_eq(DirectoryLinksProvider._linksURL, exampleUrl); - do_check_eq(provider._linksURL, exampleUrl); - - let newLinks = yield fetchData(provider); + let newLinks = yield fetchData(); isIdentical(newLinks, []); - provider.reset(); - Services.prefs.clearUserPref('general.useragent.locale') - Services.prefs.clearUserPref(provider._prefs['linksURL']); + cleanDirectoryLinksProvider(); }); -add_task(function test_DirectoryLinksProvider_getLinks_noLocaleData() { - let provider = DirectoryLinksProvider; - Services.prefs.setCharPref('general.useragent.locale', 'zh-CN'); - Services.prefs.setCharPref(provider._prefs['linksURL'], kTestSource); - - let links = yield fetchData(provider); +add_task(function test_getLinks_noLocaleData() { + setupDirectoryLinksProvider({locale: 'zh-CN'}); + let links = yield fetchData(); do_check_eq(links.length, 0); - provider.reset(); - Services.prefs.clearUserPref('general.useragent.locale') - Services.prefs.clearUserPref(provider._prefs['linksURL']); + cleanDirectoryLinksProvider(); });