--- ./dist/Fluent.jsm 2018-10-19 08:40:36.557032837 -0600 +++ /home/zbraniecki/projects/mozilla-unified/intl/l10n/Fluent.jsm 2018-10-19 21:22:35.174315857 -0600 @@ -16,7 +16,7 @@ */ -/* fluent-dom@0.4.0 */ +/* fluent@fa25466f (October 12, 2018) */ /* global Intl */ @@ -139,7 +139,53 @@ return unwrapped; } -/* global Intl */ +/** + * @overview + * + * The role of the Fluent resolver is to format a translation object to an + * instance of `FluentType` or an array of instances. + * + * Translations can contain references to other messages or variables, + * conditional logic in form of select expressions, traits which describe their + * grammatical features, and can use Fluent builtins which make use of the + * `Intl` formatters to format numbers, dates, lists and more into the + * bundle's language. See the documentation of the Fluent syntax for more + * information. + * + * In case of errors the resolver will try to salvage as much of the + * translation as possible. In rare situations where the resolver didn't know + * how to recover from an error it will return an instance of `FluentNone`. + * + * `MessageReference`, `VariantExpression`, `AttributeExpression` and + * `SelectExpression` resolve to raw Runtime Entries objects and the result of + * the resolution needs to be passed into `Type` to get their real value. + * This is useful for composing expressions. Consider: + * + * brand-name[nominative] + * + * which is a `VariantExpression` with properties `id: MessageReference` and + * `key: Keyword`. If `MessageReference` was resolved eagerly, it would + * instantly resolve to the value of the `brand-name` message. Instead, we + * want to get the message object and look for its `nominative` variant. + * + * All other expressions (except for `FunctionReference` which is only used in + * `CallExpression`) resolve to an instance of `FluentType`. The caller should + * use the `toString` method to convert the instance to a native value. + * + * + * All functions in this file pass around a special object called `env`. + * This object stores a set of elements used by all resolve functions: + * + * * {FluentBundle} bundle + * bundle for which the given resolution is happening + * * {Object} args + * list of developer provided arguments that can be used + * * {Array} errors + * list of errors collected while resolving + * * {WeakSet} dirty + * Set of patterns already encountered during this resolution. + * This is used to prevent cyclic resolutions. + */ // Prevent expansion of too long placeables. const MAX_PLACEABLE_LENGTH = 2500; @@ -1319,14 +1365,6 @@ } } -/* - * @module fluent - * @overview - * - * `fluent` is a JavaScript implementation of Project Fluent, a localization - * framework designed to unleash the expressive power of the natural language. - * - */ - this.FluentBundle = FluentBundle; -this.EXPORTED_SYMBOLS = ["FluentBundle"]; +this.FluentResource = FluentResource; +var EXPORTED_SYMBOLS = ["FluentBundle", "FluentResource"]; --- ./dist/Localization.jsm 2018-10-19 08:40:36.773712561 -0600 +++ /home/zbraniecki/projects/mozilla-unified/intl/l10n/Localization.jsm 2018-10-19 21:20:57.295233460 -0600 @@ -16,27 +16,34 @@ */ -/* fluent-dom@0.4.0 */ +/* fluent-dom@fa25466f (October 12, 2018) */ + +/* eslint no-console: ["error", { allow: ["warn", "error"] }] */ +/* global console */ + +const { L10nRegistry } = ChromeUtils.import("resource://gre/modules/L10nRegistry.jsm", {}); +const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm", {}); +const { AppConstants } = ChromeUtils.import("resource://gre/modules/AppConstants.jsm", {}); /* * Base CachedIterable class. */ class CachedIterable extends Array { - /** - * Create a `CachedIterable` instance from an iterable or, if another - * instance of `CachedIterable` is passed, return it without any - * modifications. - * - * @param {Iterable} iterable - * @returns {CachedIterable} - */ - static from(iterable) { - if (iterable instanceof this) { - return iterable; - } - - return new this(iterable); + /** + * Create a `CachedIterable` instance from an iterable or, if another + * instance of `CachedIterable` is passed, return it without any + * modifications. + * + * @param {Iterable} iterable + * @returns {CachedIterable} + */ + static from(iterable) { + if (iterable instanceof this) { + return iterable; } + + return new this(iterable); + } } /* @@ -46,88 +53,100 @@ * iterable. */ class CachedAsyncIterable extends CachedIterable { - /** - * Create an `CachedAsyncIterable` instance. - * - * @param {Iterable} iterable - * @returns {CachedAsyncIterable} - */ - constructor(iterable) { - super(); - - if (Symbol.asyncIterator in Object(iterable)) { - this.iterator = iterable[Symbol.asyncIterator](); - } else if (Symbol.iterator in Object(iterable)) { - this.iterator = iterable[Symbol.iterator](); - } else { - throw new TypeError("Argument must implement the iteration protocol."); - } - } + /** + * Create an `CachedAsyncIterable` instance. + * + * @param {Iterable} iterable + * @returns {CachedAsyncIterable} + */ + constructor(iterable) { + super(); - /** - * Synchronous iterator over the cached elements. - * - * Return a generator object implementing the iterator protocol over the - * cached elements of the original (async or sync) iterable. - */ - [Symbol.iterator]() { - const cached = this; - let cur = 0; - - return { - next() { - if (cached.length === cur) { - return {value: undefined, done: true}; - } - return cached[cur++]; - } - }; + if (Symbol.asyncIterator in Object(iterable)) { + this.iterator = iterable[Symbol.asyncIterator](); + } else if (Symbol.iterator in Object(iterable)) { + this.iterator = iterable[Symbol.iterator](); + } else { + throw new TypeError("Argument must implement the iteration protocol."); } + } - /** - * Asynchronous iterator caching the yielded elements. - * - * Elements yielded by the original iterable will be cached and available - * synchronously. Returns an async generator object implementing the - * iterator protocol over the elements of the original (async or sync) - * iterable. - */ - [Symbol.asyncIterator]() { - const cached = this; - let cur = 0; - - return { - async next() { - if (cached.length <= cur) { - cached.push(await cached.iterator.next()); - } - return cached[cur++]; - } - }; - } + /** + * Synchronous iterator over the cached elements. + * + * Return a generator object implementing the iterator protocol over the + * cached elements of the original (async or sync) iterable. + */ + [Symbol.iterator]() { + const cached = this; + let cur = 0; + + return { + next() { + if (cached.length === cur) { + return {value: undefined, done: true}; + } + return cached[cur++]; + } + }; + } - /** - * This method allows user to consume the next element from the iterator - * into the cache. - * - * @param {number} count - number of elements to consume - */ - async touchNext(count = 1) { - let idx = 0; - while (idx++ < count) { - const last = this[this.length - 1]; - if (last && last.done) { - break; - } - this.push(await this.iterator.next()); + /** + * Asynchronous iterator caching the yielded elements. + * + * Elements yielded by the original iterable will be cached and available + * synchronously. Returns an async generator object implementing the + * iterator protocol over the elements of the original (async or sync) + * iterable. + */ + [Symbol.asyncIterator]() { + const cached = this; + let cur = 0; + + return { + async next() { + if (cached.length <= cur) { + cached.push(await cached.iterator.next()); } - // Return the last cached {value, done} object to allow the calling - // code to decide if it needs to call touchNext again. - return this[this.length - 1]; + return cached[cur++]; + } + }; + } + + /** + * This method allows user to consume the next element from the iterator + * into the cache. + * + * @param {number} count - number of elements to consume + */ + async touchNext(count = 1) { + let idx = 0; + while (idx++ < count) { + const last = this[this.length - 1]; + if (last && last.done) { + break; + } + this.push(await this.iterator.next()); } + // Return the last cached {value, done} object to allow the calling + // code to decide if it needs to call touchNext again. + return this[this.length - 1]; + } } -/* eslint no-console: ["error", { allow: ["warn", "error"] }] */ +/** + * The default localization strategy for Gecko. It comabines locales + * available in L10nRegistry, with locales requested by the user to + * generate the iterator over FluentBundles. + * + * In the future, we may want to allow certain modules to override this + * with a different negotitation strategy to allow for the module to + * be localized into a different language - for example DevTools. + */ +function defaultGenerateBundles(resourceIds) { + const appLocales = Services.locale.appLocalesAsBCP47; + return L10nRegistry.generateContexts(appLocales, resourceIds); +} /** * The `Localization` class is a central high-level API for vanilla @@ -143,16 +162,21 @@ * * @returns {Localization} */ - constructor(resourceIds = [], generateBundles) { + constructor(resourceIds = [], generateBundles = defaultGenerateBundles) { this.resourceIds = resourceIds; this.generateBundles = generateBundles; this.bundles = CachedAsyncIterable.from( this.generateBundles(this.resourceIds)); } - addResourceIds(resourceIds) { + /** + * @param {Array} resourceIds - List of resource IDs + * @param {bool} eager - whether the I/O for new context should + * begin eagerly + */ + addResourceIds(resourceIds, eager = false) { this.resourceIds.push(...resourceIds); - this.onChange(); + this.onChange(eager); return this.resourceIds.length; } @@ -184,9 +208,12 @@ break; } - if (typeof console !== "undefined") { + if (AppConstants.NIGHTLY_BUILD || Cu.isInAutomation) { const locale = bundle.locales[0]; const ids = Array.from(missingIds).join(", "); + if (Cu.isInAutomation) { + throw new Error(`Missing translations in ${locale}: ${ids}`); + } console.warn(`Missing translations in ${locale}: ${ids}`); } } @@ -274,21 +301,64 @@ return val; } - handleEvent() { - this.onChange(); + /** + * Register weak observers on events that will trigger cache invalidation + */ + registerObservers() { + Services.obs.addObserver(this, "intl:app-locales-changed", true); + Services.prefs.addObserver("intl.l10n.pseudo", this, true); + } + + /** + * Default observer handler method. + * + * @param {String} subject + * @param {String} topic + * @param {Object} data + */ + observe(subject, topic, data) { + switch (topic) { + case "intl:app-locales-changed": + this.onChange(); + break; + case "nsPref:changed": + switch (data) { + case "intl.l10n.pseudo": + this.onChange(); + } + break; + default: + break; + } } /** * This method should be called when there's a reason to believe * that language negotiation or available resources changed. + * + * @param {bool} eager - whether the I/O for new context should begin eagerly */ - onChange() { + onChange(eager = false) { this.bundles = CachedAsyncIterable.from( this.generateBundles(this.resourceIds)); - this.bundles.touchNext(2); + if (eager) { + // If the first app locale is the same as last fallback + // it means that we have all resources in this locale, and + // we want to eagerly fetch just that one. + // Otherwise, we're in a scenario where the first locale may + // be partial and we want to eagerly fetch a fallback as well. + const appLocale = Services.locale.appLocaleAsBCP47; + const lastFallback = Services.locale.lastFallbackLocale; + const prefetchCount = appLocale === lastFallback ? 1 : 2; + this.bundles.touchNext(prefetchCount); + } } } +Localization.prototype.QueryInterface = ChromeUtils.generateQI([ + Ci.nsISupportsWeakReference +]); + /** * Format the value of a message into a string. * @@ -380,7 +450,7 @@ * See `Localization.formatWithFallback` for more info on how this is used. * * @param {Function} method - * @param {FluentBundle} bundle + * @param {FluentBundle} bundle * @param {Array} keys * @param {{Array<{value: string, attributes: Object}>}} translations * @@ -408,44 +478,5 @@ return missingIds; } -/* global Components */ -/* eslint no-unused-vars: 0 */ - -const Cu = Components.utils; -const Cc = Components.classes; -const Ci = Components.interfaces; - -const { L10nRegistry } = - Cu.import("resource://gre/modules/L10nRegistry.jsm", {}); -const ObserverService = - Cc["@mozilla.org/observer-service;1"].getService(Ci.nsIObserverService); -const { Services } = - Cu.import("resource://gre/modules/Services.jsm", {}); - -/** - * The default localization strategy for Gecko. It comabines locales - * available in L10nRegistry, with locales requested by the user to - * generate the iterator over FluentBundles. - * - * In the future, we may want to allow certain modules to override this - * with a different negotitation strategy to allow for the module to - * be localized into a different language - for example DevTools. - */ -function defaultGenerateBundles(resourceIds) { - const requestedLocales = Services.locale.getRequestedLocales(); - const availableLocales = L10nRegistry.getAvailableLocales(); - const defaultLocale = Services.locale.defaultLocale; - const locales = Services.locale.negotiateLanguages( - requestedLocales, availableLocales, defaultLocale, - ); - return L10nRegistry.generateContexts(locales, resourceIds); -} - -class GeckoLocalization extends Localization { - constructor(resourceIds, generateBundles = defaultGenerateBundles) { - super(resourceIds, generateBundles); - } -} - -this.Localization = GeckoLocalization; -this.EXPORTED_SYMBOLS = ["Localization"]; +this.Localization = Localization; +var EXPORTED_SYMBOLS = ["Localization"]; --- ./dist/DOMLocalization.jsm 2018-10-19 08:40:37.000392886 -0600 +++ /home/zbraniecki/projects/mozilla-unified/intl/l10n/DOMLocalization.jsm 2018-10-19 21:38:25.963726161 -0600 @@ -15,13 +15,12 @@ * limitations under the License. */ +/* fluent-dom@fa25466f (October 12, 2018) */ -/* fluent-dom@0.4.0 */ - -import Localization from '../../fluent-dom/src/localization.js'; - -/* eslint no-console: ["error", {allow: ["warn"]}] */ -/* global console */ +const { Localization } = + ChromeUtils.import("resource://gre/modules/Localization.jsm", {}); +const { Services } = + ChromeUtils.import("resource://gre/modules/Services.jsm", {}); // Match the opening angle bracket (<) in HTML tags, and HTML entities like // &, &, &. @@ -61,11 +60,12 @@ }, "http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul": { global: [ - "accesskey", "aria-label", "aria-valuetext", "aria-moz-hint", "label" - ], + "accesskey", "aria-label", "aria-valuetext", "aria-moz-hint", "label", + "title", "tooltiptext"], + description: ["value"], key: ["key", "keycode"], + label: ["value"], textbox: ["placeholder"], - toolbarbutton: ["tooltiptext"], } }; @@ -96,6 +96,7 @@ const templateElement = element.ownerDocument.createElementNS( "http://www.w3.org/1999/xhtml", "template" ); + // eslint-disable-next-line no-unsanitized/property templateElement.innerHTML = value; overlayChildNodes(templateElement.content, element); } @@ -349,6 +350,46 @@ return toElement; } +/** + * Sanitizes a translation before passing them to Node.localize API. + * + * It returns `false` if the translation contains DOM Overlays and should + * not go into Node.localize. + * + * Note: There's a third item of work that JS DOM Overlays do - removal + * of attributes from the previous translation. + * This is not trivial to implement for Node.localize scenario, so + * at the moment it is not supported. + * + * @param {{ + * localName: string, + * namespaceURI: string, + * type: string || null + * l10nId: string, + * l10nArgs: Array || null, + * l10nAttrs: string ||null, + * }} l10nItems + * @param {{value: string, attrs: Object}} translations + * @returns boolean + * @private + */ +function sanitizeTranslationForNodeLocalize(l10nItem, translation) { + if (reOverlay.test(translation.value)) { + return false; + } + + if (translation.attributes) { + const explicitlyAllowed = l10nItem.l10nAttrs === null ? null : + l10nItem.l10nAttrs.split(",").map(i => i.trim()); + for (const [j, {name}] of translation.attributes.entries()) { + if (!isAttrNameLocalizable(name, l10nItem, explicitlyAllowed)) { + translation.attributes.splice(j, 1); + } + } + } + return true; +} + const L10NID_ATTR_NAME = "data-l10n-id"; const L10NARGS_ATTR_NAME = "data-l10n-args"; @@ -390,8 +431,8 @@ }; } - onChange() { - super.onChange(); + onChange(eager = false) { + super.onChange(eager); this.translateRoots(); } @@ -478,18 +519,17 @@ } if (this.windowElement) { - if (this.windowElement !== newRoot.ownerDocument.defaultView) { + if (this.windowElement !== newRoot.ownerGlobal) { throw new Error(`Cannot connect a root: DOMLocalization already has a root from a different window.`); } } else { - this.windowElement = newRoot.ownerDocument.defaultView; + this.windowElement = newRoot.ownerGlobal; this.mutationObserver = new this.windowElement.MutationObserver( mutations => this.translateMutations(mutations) ); } - this.roots.add(newRoot); this.mutationObserver.observe(newRoot, this.observerConfig); } @@ -532,7 +572,20 @@ translateRoots() { const roots = Array.from(this.roots); return Promise.all( - roots.map(root => this.translateFragment(root)) + roots.map(async root => { + // We want to first retranslate the UI, and + // then (potentially) flip the directionality. + // + // This means that the DOM alternations and directionality + // are set in the same microtask. + await this.translateFragment(root); + let primaryLocale = Services.locale.appLocaleAsBCP47; + let direction = Services.locale.isAppLocaleRTL ? "rtl" : "ltr"; + root.setAttribute("lang", primaryLocale); + root.setAttribute(root.namespaceURI === + "http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" + ? "localedir" : "dir", direction); + }) ); } @@ -599,7 +652,10 @@ if (this.pendingElements.size > 0) { if (this.pendingrAF === null) { this.pendingrAF = this.windowElement.requestAnimationFrame(() => { - this.translateElements(Array.from(this.pendingElements)); + // We need to filter for elements that lost their l10n-id while + // waiting for the animation frame. + this.translateElements(Array.from(this.pendingElements) + .filter(elem => elem.hasAttribute("data-l10n-id"))); this.pendingElements.clear(); this.pendingrAF = null; }); @@ -621,6 +677,63 @@ * @returns {Promise} */ translateFragment(frag) { + if (frag.localize) { + // This is a temporary fast-path offered by Gecko to workaround performance + // issues coming from Fluent and XBL+Stylo performing unnecesary + // operations during startup. + // For details see bug 1441037, bug 1442262, and bug 1363862. + + // A sparse array which will store translations separated out from + // all translations that is needed for DOM Overlay. + const overlayTranslations = []; + + const getTranslationsForItems = async l10nItems => { + const keys = l10nItems.map( + l10nItem => ({id: l10nItem.l10nId, args: l10nItem.l10nArgs})); + const translations = await this.formatMessages(keys); + + // Here we want to separate out elements that require DOM Overlays. + // Those elements will have to be translated using our JS + // implementation, while everything else is going to use the fast-path. + for (const [i, translation] of translations.entries()) { + if (translation === undefined) { + continue; + } + + const hasOnlyText = + sanitizeTranslationForNodeLocalize(l10nItems[i], translation); + if (!hasOnlyText) { + // Removing from translations to make Node.localize skip it. + // We will translate it below using JS DOM Overlays. + overlayTranslations[i] = translations[i]; + translations[i] = undefined; + } + } + + // We pause translation observing here because Node.localize + // will translate the whole DOM next, using the `translations`. + // + // The observer will be resumed after DOM Overlays are localized + // in the next microtask. + this.pauseObserving(); + return translations; + }; + + return frag.localize(getTranslationsForItems.bind(this)) + .then(untranslatedElements => { + for (let i = 0; i < overlayTranslations.length; i++) { + if (overlayTranslations[i] !== undefined && + untranslatedElements[i] !== undefined) { + translateElement(untranslatedElements[i], overlayTranslations[i]); + } + } + this.resumeObserving(); + }) + .catch(e => { + this.resumeObserving(); + throw e; + }); + } return this.translateElements(this.getTranslatables(frag)); } @@ -700,37 +813,5 @@ } } -/* global L10nRegistry, Services */ - -/** - * The default localization strategy for Gecko. It comabines locales - * available in L10nRegistry, with locales requested by the user to - * generate the iterator over FluentBundles. - * - * In the future, we may want to allow certain modules to override this - * with a different negotitation strategy to allow for the module to - * be localized into a different language - for example DevTools. - */ -function defaultGenerateBundles(resourceIds) { - const requestedLocales = Services.locale.getRequestedLocales(); - const availableLocales = L10nRegistry.getAvailableLocales(); - const defaultLocale = Services.locale.defaultLocale; - const locales = Services.locale.negotiateLanguages( - requestedLocales, availableLocales, defaultLocale, - ); - return L10nRegistry.generateContexts(locales, resourceIds); -} - - -class GeckoDOMLocalization extends DOMLocalization { - constructor( - windowElement, - resourceIds, - generateBundles = defaultGenerateBundles - ) { - super(windowElement, resourceIds, generateBundles); - } -} - -this.DOMLocalization = GeckoDOMLocalization; -this.EXPORTED_SYMBOLS = ["DOMLocalization"]; +this.DOMLocalization = DOMLocalization; +var EXPORTED_SYMBOLS = ["DOMLocalization"];