Bug 1333980 - Introduce L10nRegistry.jsm. r=mossop

This patch introduces a new registry for localization resources to replace
ChromeRegistry. It uses asynchronous I/O and iterators to generate
permutations of possible sources of language resources for use in the new
Localization API.

In the initial form the API handles packages resources and has API for
interacting with the AddonsManager which will report language packs.

MozReview-Commit-ID: LfERDYMROh

--HG--
extra : rebase_source : a6e5b790142e5afb1ce750478190e5ad87012f8d

intl/l10n/L10nRegistry.jsm

@@ -0,0 +1,346 @@
+const { Services } = Components.utils.import('resource://gre/modules/Services.jsm', {});
+const { MessageContext } = Components.utils.import("resource://gre/modules/MessageContext.jsm", {});
+Components.utils.importGlobalProperties(["fetch"]); /* globals fetch */
+
+/**
+ * L10nRegistry is a localization resource management system for Gecko.
+ *
+ * It manages the list of resource sources provided with the app and allows
+ * for additional sources to be added and updated.
+ *
+ * It's primary purpose is to allow for building an iterator over MessageContext objects
+ * that will be utilized by a localization API.
+ *
+ * The generator creates all possible permutations of locales and sources to allow for
+ * complete fallbacking.
+ *
+ * Example:
+ *
+ *   FileSource1:
+ *     name: 'app'
+ *     locales: ['en-US', 'de']
+ *     resources: [
+ *       '/browser/menu.ftl',
+ *       '/platform/toolkit.ftl',
+ *     ]
+ *   FileSource2:
+ *     name: 'platform'
+ *     locales: ['en-US', 'de']
+ *     resources: [
+ *       '/platform/toolkit.ftl',
+ *     ]
+ *
+ * If the user will request:
+ *   L10nRegistry.generateContexts(['de', 'en-US'], [
+ *     '/browser/menu.ftl',
+ *     '/platform/toolkit.ftl'
+ *   ]);
+ *
+ * the generator will return an iterator over the following contexts:
+ *
+ *   {
+ *     locale: 'de',
+ *     resources: [
+ *       ['app', '/browser/menu.ftl'],
+ *       ['app', '/platform/toolkit.ftl'],
+ *     ]
+ *   },
+ *   {
+ *     locale: 'de',
+ *     resources: [
+ *       ['app', '/browser/menu.ftl'],
+ *       ['platform', '/platform/toolkit.ftl'],
+ *     ]
+ *   },
+ *   {
+ *     locale: 'en-US',
+ *     resources: [
+ *       ['app', '/browser/menu.ftl'],
+ *       ['app', '/platform/toolkit.ftl'],
+ *     ]
+ *   },
+ *   {
+ *     locale: 'en-US',
+ *     resources: [
+ *       ['app', '/browser/menu.ftl'],
+ *       ['platform', '/platform/toolkit.ftl'],
+ *     ]
+ *   }
+ *
+ * This allows the localization API to consume the MessageContext and lazily fallback
+ * on the next in case of a missing string or error.
+ *
+ * If during the life-cycle of the app a new source is added, the generator can be called again
+ * and will produce a new set of permutations placing the language pack provided resources
+ * at the top.
+ */
+
+const L10nRegistry = {
+  sources: new Map(),
+  ctxCache: new Map(),
+
+  /**
+   * Based on the list of requested languages and resource Ids,
+   * this function returns an lazy iterator over message context permutations.
+   *
+   * @param {Array} requestedLangs
+   * @param {Array} resourceIds
+   * @returns {Iterator<MessageContext>}
+   */
+  * generateContexts(requestedLangs, resourceIds) {
+    const sourcesOrder = Array.from(this.sources.keys()).reverse();
+    for (const locale of requestedLangs) {
+      yield * generateContextsForLocale(locale, sourcesOrder, resourceIds);
+    }
+  },
+
+  /**
+   * Adds a new resource source to the L10nRegistry.
+   *
+   * @param {FileSource} source
+   */
+  registerSource(source) {
+    if (this.sources.has(source.name)) {
+      throw new Error(`Source with name "${source.name}" already registered.`);
+    }
+    this.sources.set(source.name, source);
+    Services.obs.notifyObservers(null, 'l10n:available-locales-changed', null);
+  },
+
+  /**
+   * Updates an existing source in the L10nRegistry
+   *
+   * That will usually happen when a new version of a source becomes
+   * available (for example, an updated version of a language pack).
+   *
+   * @param {FileSource} source
+   */
+  updateSource(source) {
+    if (!this.sources.has(source.name)) {
+      throw new Error(`Source with name "${source.name}" is not registered.`);
+    }
+    this.sources.set(source.name, source);
+    this.ctxCache.clear();
+    Services.obs.notifyObservers(null, 'l10n:available-locales-changed', null);
+  },
+
+  /**
+   * Removes a source from the L10nRegistry.
+   *
+   * @param {String} sourceId
+   */
+  removeSource(sourceName) {
+    this.sources.delete(sourceName);
+    Services.obs.notifyObservers(null, 'l10n:available-locales-changed', null);
+  },
+
+  /**
+   * Returns a list of locales for which at least one source
+   * has resources.
+   *
+   * @returns {Array<String>}
+   */
+  getAvailableLocales() {
+    const locales = new Set();
+
+    for (const source of this.sources.values()) {
+      for (const locale of source.locales) {
+        locales.add(locale);
+      }
+    }
+    return Array.from(locales);
+  },
+};
+
+/**
+ * A helper function for generating unique context ID used for caching
+ * MessageContexts.
+ *
+ * @param {String} locale
+ * @param {Array} sourcesOrder
+ * @param {Array} resourceIds
+ * @returns {String}
+ */
+function generateContextID(locale, sourcesOrder, resourceIds) {
+  const sources = sourcesOrder.join(',');
+  const ids = resourceIds.join(',');
+  return `${locale}|${sources}|${ids}`;
+}
+
+/**
+ * This function generates an iterator over MessageContexts for a single locale
+ * for a given list of resourceIds for all possible combinations of sources.
+ *
+ * This function is called recursively to generate all possible permutations
+ * and uses the last, optional parameter, to pass the already resolved
+ * sources order.
+ *
+ * @param {String} locale
+ * @param {Array} sourcesOrder
+ * @param {Array} resourceIds
+ * @param {Array} [resolvedOrder]
+ * @returns {Iterator<MessageContext>}
+ */
+function* generateContextsForLocale(locale, sourcesOrder, resourceIds, resolvedOrder = []) {
+  const resolvedLength = resolvedOrder.length;
+  const resourcesLength = resourceIds.length;
+
+  // Inside that loop we have a list of resources and the sources for them, like this:
+  //   ['test.ftl', 'menu.ftl', 'foo.ftl']
+  //   ['app', 'platform', 'app']
+  for (const sourceName of sourcesOrder) {
+    const order = resolvedOrder.concat(sourceName);
+
+    // We bail only if the hasFile returns a strict false here,
+    // because for FileSource it may also return undefined, which means
+    // that we simply don't know if the source contains the file and we'll
+    // have to perform the I/O to learn.
+    if (L10nRegistry.sources.get(sourceName).hasFile(locale, resourceIds[resolvedOrder.length]) === false) {
+      continue;
+    }
+
+    // If the number of resolved sources equals the number of resources,
+    // create the right context and return it if it loads.
+    if (resolvedLength + 1 === resourcesLength) {
+      yield generateContext(locale, order, resourceIds);
+    } else {
+      // otherwise recursively load another generator that walks over the
+      // partially resolved list of sources.
+      yield * generateContextsForLocale(locale, sourcesOrder, resourceIds, order);
+    }
+  }
+}
+
+/**
+ * Generates a single MessageContext by loading all resources
+ * from the listed sources for a given locale.
+ *
+ * @param {String} locale
+ * @param {Array} sourcesOrder
+ * @param {Array} resourceIds
+ * @returns {Promise<MessageContext>}
+ */
+async function generateContext(locale, sourcesOrder, resourceIds) {
+  const ctxId = generateContextID(locale, sourcesOrder, resourceIds);
+  if (!L10nRegistry.ctxCache.has(ctxId)) {
+    const ctx = new MessageContext(locale);
+    for (let i = 0; i < resourceIds.length; i++) {
+      const data = await L10nRegistry.sources.get(sourcesOrder[i]).fetchFile(locale, resourceIds[i]);
+      if (data === null) {
+        return false;
+      }
+      ctx.addMessages(data);
+    }
+    L10nRegistry.ctxCache.set(ctxId, ctx);
+  }
+  return L10nRegistry.ctxCache.get(ctxId);
+}
+
+/**
+ * This is a basic Source for L10nRegistry.
+ * It registers its own locales and a pre-path, and when asked for a file
+ * it attempts to download and cache it.
+ *
+ * The Source caches the downloaded files so any consecutive loads will
+ * come from the cache.
+ **/
+class FileSource {
+  constructor(name, locales, prePath) {
+    this.name = name;
+    this.locales = locales;
+    this.prePath = prePath;
+    this.cache = {};
+  }
+
+  getPath(locale, path) {
+    return (this.prePath + path).replace(/\{locale\}/g, locale);
+  }
+
+  hasFile(locale, path) {
+    if (!this.locales.includes(locale)) {
+      return false;
+    }
+
+    const fullPath = this.getPath(locale, path);
+    if (!this.cache.hasOwnProperty(fullPath)) {
+      return undefined;
+    }
+
+    if (this.cache[fullPath] === null) {
+      return false;
+    }
+    return true;
+  }
+
+  async fetchFile(locale, path) {
+    if (!this.locales.includes(locale)) {
+      return null;
+    }
+
+    const fullPath = this.getPath(locale, path);
+    if (this.hasFile(locale, path) === undefined) {
+      let file = await L10nRegistry.load(fullPath);
+
+      if (file === undefined) {
+        this.cache[fullPath] = null;
+      } else {
+        this.cache[fullPath] = file;
+      }
+    }
+    return this.cache[fullPath];
+  }
+}
+
+/**
+ * This is an extension of the FileSource which should be used
+ * for sources that can provide the list of files available in the source.
+ *
+ * This allows for a faster lookup in cases where the source does not
+ * contain most of the files that the app will request for (e.g. an addon).
+ **/
+class IndexedFileSource extends FileSource {
+  constructor(name, locales, prePath, paths) {
+    super(name, locales, prePath);
+    this.paths = paths;
+  }
+
+  hasFile(locale, path) {
+    if (!this.locales.includes(locale)) {
+      return false;
+    }
+    const fullPath = this.getPath(locale, path);
+    return this.paths.includes(fullPath);
+  }
+
+  async fetchFile(locale, path) {
+    if (!this.locales.includes(locale)) {
+      return null;
+    }
+
+    const fullPath = this.getPath(locale, path);
+    if (this.paths.includes(fullPath)) {
+      let file = await L10nRegistry.load(fullPath);
+
+      if (file === undefined) {
+        return null;
+      } else {
+        return file;
+      }
+    } else {
+      return null;
+    }
+  }
+}
+
+/**
+ * We keep it as a method to make it easier to override for testing purposes.
+ **/
+L10nRegistry.load = function(url) {
+  return fetch(url).then(data => data.text()).catch(() => undefined);
+};
+
+this.L10nRegistry = L10nRegistry;
+this.FileSource = FileSource;
+this.IndexedFileSource = IndexedFileSource;
+
+this.EXPORTED_SYMBOLS = [];

intl/l10n/moz.build

@@ -5,6 +5,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 EXTRA_JS_MODULES += [
+    'L10nRegistry.jsm',
     'MessageContext.jsm',
 ]
 

intl/l10n/test/test_l10nregistry.js

@@ -0,0 +1,312 @@
+/* Any copyrighequal dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const {
+  L10nRegistry,
+  FileSource,
+  IndexedFileSource
+} = Components.utils.import("resource://gre/modules/L10nRegistry.jsm", {});
+
+let fs;
+L10nRegistry.load = async function(url) {
+  return fs[url];
+}
+
+add_task(function test_methods_presence() {
+  equal(typeof L10nRegistry.generateContexts, "function");
+  equal(typeof L10nRegistry.getAvailableLocales, "function");
+  equal(typeof L10nRegistry.registerSource, "function");
+  equal(typeof L10nRegistry.updateSource, "function");
+});
+
+/**
+ * This test tests generation of a proper context for a single
+ * source scenario
+ */
+add_task(async function test_methods_calling() {
+  fs = {
+    '/localization/en-US/browser/menu.ftl': 'key = Value',
+  };
+  const originalLoad = L10nRegistry.load;
+
+  const source = new FileSource('test', ['en-US'], '/localization/{locale}');
+  L10nRegistry.registerSource(source);
+
+  const ctxs = L10nRegistry.generateContexts(['en-US'], ['/browser/menu.ftl']);
+
+  const ctx = await ctxs.next().value;
+
+  equal(ctx.hasMessage('key'), true);
+
+  // cleanup
+  L10nRegistry.sources.clear();
+  L10nRegistry.ctxCache.clear();
+});
+
+/**
+ * This test verifies that the public methods return expected values
+ * for the single source scenario
+ */
+add_task(async function test_has_one_source() {
+  let oneSource = new FileSource('app', ['en-US'], './app/data/locales/{locale}/');
+  fs = {
+    './app/data/locales/en-US/test.ftl': 'key = value en-US'
+  };
+  L10nRegistry.registerSource(oneSource);
+
+
+  // has one source
+
+  equal(L10nRegistry.sources.size, 1);
+  equal(L10nRegistry.sources.has('app'), true);
+
+
+  // returns a single context
+
+  let ctxs = L10nRegistry.generateContexts(['en-US'], ['test.ftl']);
+  let ctx0 = await ctxs.next().value;
+  equal(ctx0.hasMessage('key'), true);
+
+  equal(ctxs.next().done, true);
+
+
+  // returns no contexts for missing locale
+
+  ctxs = L10nRegistry.generateContexts(['pl'], ['test.ftl']);
+
+  equal(ctxs.next().done, true);
+
+  // cleanup
+  L10nRegistry.sources.clear();
+  L10nRegistry.ctxCache.clear();
+});
+
+/**
+ * This test verifies that public methods return expected values
+ * for the dual source scenario.
+ */
+add_task(async function test_has_two_sources() {
+  let oneSource = new FileSource('platform', ['en-US'], './platform/data/locales/{locale}/');
+  L10nRegistry.registerSource(oneSource);
+
+  let secondSource = new FileSource('app', ['pl'], './app/data/locales/{locale}/');
+  L10nRegistry.registerSource(secondSource);
+  fs = {
+    './platform/data/locales/en-US/test.ftl': 'key = platform value',
+    './app/data/locales/pl/test.ftl': 'key = app value'
+  };
+
+
+  // has two sources
+
+  equal(L10nRegistry.sources.size, 2);
+  equal(L10nRegistry.sources.has('app'), true);
+  equal(L10nRegistry.sources.has('platform'), true);
+
+
+  // returns correct contexts for en-US
+
+  let ctxs = L10nRegistry.generateContexts(['en-US'], ['test.ftl']);
+  let ctx0 = await ctxs.next().value;
+
+  equal(ctx0.hasMessage('key'), true);
+  let msg = ctx0.getMessage('key');
+  equal(ctx0.format(msg), 'platform value');
+
+  equal(ctxs.next().done, true);
+
+
+  // returns correct contexts for [pl, en-US]
+
+  ctxs = L10nRegistry.generateContexts(['pl', 'en-US'], ['test.ftl']);
+  ctx0 = await ctxs.next().value;
+  equal(ctx0.locales[0], 'pl');
+  equal(ctx0.hasMessage('key'), true);
+  let msg0 = ctx0.getMessage('key');
+  equal(ctx0.format(msg0), 'app value');
+
+  let ctx1 = await ctxs.next().value;
+  equal(ctx1.locales[0], 'en-US');
+  equal(ctx1.hasMessage('key'), true);
+  let msg1 = ctx1.getMessage('key');
+  equal(ctx1.format(msg1), 'platform value');
+
+  equal(ctxs.next().done, true);
+
+  // cleanup
+  L10nRegistry.sources.clear();
+  L10nRegistry.ctxCache.clear();
+});
+
+/**
+ * This test verifies that behavior specific to the IndexedFileSource
+ * works correctly.
+ *
+ * In particular it tests that IndexedFileSource correctly returns
+ * missing files as `false` instead of `undefined`.
+ */
+add_task(async function test_indexed() {
+  let oneSource = new IndexedFileSource('langpack-pl', ['pl'], '/data/locales/{locale}/', [
+    '/data/locales/pl/test.ftl',
+  ]);
+  L10nRegistry.registerSource(oneSource);
+  fs = {
+    '/data/locales/pl/test.ftl': 'key = value'
+  };
+
+  equal(L10nRegistry.sources.size, 1);
+  equal(L10nRegistry.sources.has('langpack-pl'), true);
+
+  equal(oneSource.getPath('pl', 'test.ftl'), '/data/locales/pl/test.ftl');
+  equal(oneSource.hasFile('pl', 'test.ftl'), true);
+  equal(oneSource.hasFile('pl', 'missing.ftl'), false);
+
+  // cleanup
+  L10nRegistry.sources.clear();
+  L10nRegistry.ctxCache.clear();
+});
+
+/**
+ * This test checks if the correct order of contexts is used for
+ * scenarios where a new file source is added on top of the default one.
+ */
+add_task(async function test_override() {
+  let fileSource = new FileSource('app', ['pl'], '/app/data/locales/{locale}/');
+  L10nRegistry.registerSource(fileSource);
+
+  let oneSource = new IndexedFileSource('langpack-pl', ['pl'], '/data/locales/{locale}/', [
+    '/data/locales/pl/test.ftl'
+  ]);
+  L10nRegistry.registerSource(oneSource);
+
+  fs = {
+    '/app/data/locales/pl/test.ftl': 'key = value',
+    '/data/locales/pl/test.ftl': 'key = addon value'
+  };
+
+  equal(L10nRegistry.sources.size, 2);
+  equal(L10nRegistry.sources.has('langpack-pl'), true);
+
+  let ctxs = L10nRegistry.generateContexts(['pl'], ['test.ftl']);
+  let ctx0 = await ctxs.next().value;
+  equal(ctx0.locales[0], 'pl');
+  equal(ctx0.hasMessage('key'), true);
+  let msg0 = ctx0.getMessage('key');
+  equal(ctx0.format(msg0), 'addon value');
+
+  let ctx1 = await ctxs.next().value;
+  equal(ctx1.locales[0], 'pl');
+  equal(ctx1.hasMessage('key'), true);
+  let msg1 = ctx1.getMessage('key');
+  equal(ctx1.format(msg1), 'value');
+
+  equal(ctxs.next().done, true);
+
+  // cleanup
+  L10nRegistry.sources.clear();
+  L10nRegistry.ctxCache.clear();
+});
+
+/**
+ * This test verifies that new contexts are returned
+ * after source update.
+ */
+add_task(async function test_updating() {
+  let oneSource = new IndexedFileSource('langpack-pl', ['pl'], '/data/locales/{locale}/', [
+    '/data/locales/pl/test.ftl',
+  ]);
+  L10nRegistry.registerSource(oneSource);
+  fs = {
+    '/data/locales/pl/test.ftl': 'key = value'
+  };
+
+  let ctxs = L10nRegistry.generateContexts(['pl'], ['test.ftl']);
+  let ctx0 = await ctxs.next().value;
+  equal(ctx0.locales[0], 'pl');
+  equal(ctx0.hasMessage('key'), true);
+  let msg0 = ctx0.getMessage('key');
+  equal(ctx0.format(msg0), 'value');
+
+
+  const newSource = new IndexedFileSource('langpack-pl', ['pl'], '/data/locales/{locale}/', [
+    '/data/locales/pl/test.ftl'
+  ]);
+  fs['/data/locales/pl/test.ftl'] = 'key = new value';
+  L10nRegistry.updateSource(newSource);
+
+  equal(L10nRegistry.sources.size, 1);
+  ctxs = L10nRegistry.generateContexts(['pl'], ['test.ftl']);
+  ctx0 = await ctxs.next().value;
+  msg0 = ctx0.getMessage('key');
+  equal(ctx0.format(msg0), 'new value');
+
+  // cleanup
+  L10nRegistry.sources.clear();
+  L10nRegistry.ctxCache.clear();
+});
+
+/**
+ * This test verifies that generated contexts return correct values
+ * after sources are being removed.
+ */
+add_task(async function test_removing() {
+  let fileSource = new FileSource('app', ['pl'], '/app/data/locales/{locale}/');
+  L10nRegistry.registerSource(fileSource);
+
+  let oneSource = new IndexedFileSource('langpack-pl', ['pl'], '/data/locales/{locale}/', [
+    '/data/locales/pl/test.ftl'
+  ]);
+  L10nRegistry.registerSource(oneSource);
+
+  fs = {
+    '/app/data/locales/pl/test.ftl': 'key = value',
+    '/data/locales/pl/test.ftl': 'key = addon value'
+  };
+
+  equal(L10nRegistry.sources.size, 2);
+  equal(L10nRegistry.sources.has('langpack-pl'), true);
+
+  let ctxs = L10nRegistry.generateContexts(['pl'], ['test.ftl']);
+  let ctx0 = await ctxs.next().value;
+  equal(ctx0.locales[0], 'pl');
+  equal(ctx0.hasMessage('key'), true);
+  let msg0 = ctx0.getMessage('key');
+  equal(ctx0.format(msg0), 'addon value');
+
+  let ctx1 = await ctxs.next().value;
+  equal(ctx1.locales[0], 'pl');
+  equal(ctx1.hasMessage('key'), true);
+  let msg1 = ctx1.getMessage('key');
+  equal(ctx1.format(msg1), 'value');
+
+  equal(ctxs.next().done, true);
+
+  // Remove langpack
+
+  L10nRegistry.removeSource('langpack-pl');
+
+  equal(L10nRegistry.sources.size, 1);
+  equal(L10nRegistry.sources.has('langpack-pl'), false);
+
+  ctxs = L10nRegistry.generateContexts(['pl'], ['test.ftl']);
+  ctx0 = await ctxs.next().value;
+  equal(ctx0.locales[0], 'pl');
+  equal(ctx0.hasMessage('key'), true);
+  msg0 = ctx0.getMessage('key');
+  equal(ctx0.format(msg0), 'value');
+
+  equal(ctxs.next().done, true);
+
+  // Remove app source
+
+  L10nRegistry.removeSource('app');
+
+  equal(L10nRegistry.sources.size, 0);
+
+  ctxs = L10nRegistry.generateContexts(['pl'], ['test.ftl']);
+  equal(ctxs.next().done, true);
+
+  // cleanup
+  L10nRegistry.sources.clear();
+  L10nRegistry.ctxCache.clear();
+});

intl/l10n/test/xpcshell.ini

@@ -1,4 +1,5 @@
 [DEFAULT]
 head =
 
+[test_l10nregistry.js]
 [test_messagecontext.js]

python/mozbuild/mozpack/packager/formats.py

@@ -338,5 +338,6 @@ class OmniJarSubFormatter(PiecemealFormatter):
             'modules',
             'greprefs.js',
             'hyphenation',
+            'localization',
             'update.locale',
         ]