diff --git a/intl/l10n/Fluent.jsm b/intl/l10n/Fluent.jsm index 7fbb882a721b..4b51be5c283a 100644 --- a/intl/l10n/Fluent.jsm +++ b/intl/l10n/Fluent.jsm @@ -16,7 +16,7 @@ */ -/* fluent@fa25466f (October 12, 2018) */ +/* fluent@0.10.0 */ /* global Intl */ @@ -195,20 +195,7 @@ const FSI = "\u2068"; const PDI = "\u2069"; -/** - * Helper for matching a variant key to the given selector. - * - * Used in SelectExpressions and VariantExpressions. - * - * @param {FluentBundle} bundle - * Resolver environment object. - * @param {FluentType} key - * The key of the currently considered variant. - * @param {FluentType} selector - * The selector based om which the correct variant should be chosen. - * @returns {FluentType} - * @private - */ +// Helper: match a variant key to the given selector. function match(bundle, selector, key) { if (key === selector) { // Both are strings. @@ -233,23 +220,10 @@ function match(bundle, selector, key) { return false; } -/** - * Helper for choosing the default value from a set of members. - * - * Used in SelectExpressions and Type. - * - * @param {Object} env - * Resolver environment object. - * @param {Object} members - * Hash map of variants from which the default value is to be selected. - * @param {Number} star - * The index of the default variant. - * @returns {FluentType} - * @private - */ -function DefaultMember(env, members, star) { - if (members[star]) { - return members[star]; +// Helper: resolve the default variant from a list of variants. +function getDefault(env, variants, star) { + if (variants[star]) { + return Type(env, variants[star]); } const { errors } = env; @@ -257,176 +231,34 @@ function DefaultMember(env, members, star) { return new FluentNone(); } +// Helper: resolve arguments to a call expression. +function getArguments(env, args) { + const positional = []; + const named = {}; -/** - * Resolve a reference to another message. - * - * @param {Object} env - * Resolver environment object. - * @param {Object} id - * The identifier of the message to be resolved. - * @param {String} id.name - * The name of the identifier. - * @returns {FluentType} - * @private - */ -function MessageReference(env, {name}) { - const { bundle, errors } = env; - const message = name.startsWith("-") - ? bundle._terms.get(name) - : bundle._messages.get(name); - - if (!message) { - const err = name.startsWith("-") - ? new ReferenceError(`Unknown term: ${name}`) - : new ReferenceError(`Unknown message: ${name}`); - errors.push(err); - return new FluentNone(name); - } - - return message; -} - -/** - * Resolve a variant expression to the variant object. - * - * @param {Object} env - * Resolver environment object. - * @param {Object} expr - * An expression to be resolved. - * @param {Object} expr.ref - * An Identifier of a message for which the variant is resolved. - * @param {Object} expr.id.name - * Name a message for which the variant is resolved. - * @param {Object} expr.key - * Variant key to be resolved. - * @returns {FluentType} - * @private - */ -function VariantExpression(env, {ref, selector}) { - const message = MessageReference(env, ref); - if (message instanceof FluentNone) { - return message; - } - - const { bundle, errors } = env; - const sel = Type(env, selector); - const value = message.value || message; - - function isVariantList(node) { - return Array.isArray(node) && - node[0].type === "select" && - node[0].selector === null; - } - - if (isVariantList(value)) { - // Match the specified key against keys of each variant, in order. - for (const variant of value[0].variants) { - const key = Type(env, variant.key); - if (match(env.bundle, sel, key)) { - return variant; + if (args) { + for (const arg of args) { + if (arg.type === "narg") { + named[arg.name] = Type(env, arg.value); + } else { + positional.push(Type(env, arg)); } } } - errors.push( - new ReferenceError(`Unknown variant: ${sel.toString(bundle)}`)); - return Type(env, message); + return [positional, named]; } - -/** - * Resolve an attribute expression to the attribute object. - * - * @param {Object} env - * Resolver environment object. - * @param {Object} expr - * An expression to be resolved. - * @param {String} expr.ref - * An ID of a message for which the attribute is resolved. - * @param {String} expr.name - * Name of the attribute to be resolved. - * @returns {FluentType} - * @private - */ -function AttributeExpression(env, {ref, name}) { - const message = MessageReference(env, ref); - if (message instanceof FluentNone) { - return message; - } - - if (message.attrs) { - // Match the specified name against keys of each attribute. - for (const attrName in message.attrs) { - if (name === attrName) { - return message.attrs[name]; - } - } - } - - const { errors } = env; - errors.push(new ReferenceError(`Unknown attribute: ${name}`)); - return Type(env, message); -} - -/** - * Resolve a select expression to the member object. - * - * @param {Object} env - * Resolver environment object. - * @param {Object} expr - * An expression to be resolved. - * @param {String} expr.selector - * Selector expression - * @param {Array} expr.variants - * List of variants for the select expression. - * @param {Number} expr.star - * Index of the default variant. - * @returns {FluentType} - * @private - */ -function SelectExpression(env, {selector, variants, star}) { - if (selector === null) { - return DefaultMember(env, variants, star); - } - - let sel = Type(env, selector); - if (sel instanceof FluentNone) { - return DefaultMember(env, variants, star); - } - - // Match the selector against keys of each variant, in order. - for (const variant of variants) { - const key = Type(env, variant.key); - if (match(env.bundle, sel, key)) { - return variant; - } - } - - return DefaultMember(env, variants, star); -} - - -/** - * Resolve expression to a Fluent type. - * - * JavaScript strings are a special case. Since they natively have the - * `toString` method they can be used as if they were a Fluent type without - * paying the cost of creating a instance of one. - * - * @param {Object} env - * Resolver environment object. - * @param {Object} expr - * An expression object to be resolved into a Fluent type. - * @returns {FluentType} - * @private - */ +// Resolve an expression to a Fluent type. function Type(env, expr) { - // A fast-path for strings which are the most common case, and for - // `FluentNone` which doesn't require any additional logic. + // A fast-path for strings which are the most common case. Since they + // natively have the `toString` method they can be used as if they were + // a FluentType instance without incurring the cost of creating one. if (typeof expr === "string") { return env.bundle._transform(expr); } + + // A fast-path for `FluentNone` which doesn't require any additional logic. if (expr instanceof FluentNone) { return expr; } @@ -437,32 +269,21 @@ function Type(env, expr) { return Pattern(env, expr); } - switch (expr.type) { + case "str": + return expr.value; case "num": return new FluentNumber(expr.value); case "var": return VariableReference(env, expr); - case "func": - return FunctionReference(env, expr); - case "call": - return CallExpression(env, expr); - case "ref": { - const message = MessageReference(env, expr); - return Type(env, message); - } - case "getattr": { - const attr = AttributeExpression(env, expr); - return Type(env, attr); - } - case "getvar": { - const variant = VariantExpression(env, expr); - return Type(env, variant); - } - case "select": { - const member = SelectExpression(env, expr); - return Type(env, member); - } + case "term": + return TermReference({...env, args: {}}, expr); + case "ref": + return expr.args + ? FunctionReference(env, expr) + : MessageReference(env, expr); + case "select": + return SelectExpression(env, expr); case undefined: { // If it's a node with a value, resolve the value. if (expr.value !== null && expr.value !== undefined) { @@ -478,24 +299,13 @@ function Type(env, expr) { } } -/** - * Resolve a reference to a variable. - * - * @param {Object} env - * Resolver environment object. - * @param {Object} expr - * An expression to be resolved. - * @param {String} expr.name - * Name of an argument to be returned. - * @returns {FluentType} - * @private - */ +// Resolve a reference to a variable. function VariableReference(env, {name}) { const { args, errors } = env; if (!args || !args.hasOwnProperty(name)) { errors.push(new ReferenceError(`Unknown variable: ${name}`)); - return new FluentNone(name); + return new FluentNone(`$${name}`); } const arg = args[name]; @@ -519,26 +329,80 @@ function VariableReference(env, {name}) { errors.push( new TypeError(`Unsupported variable type: ${name}, ${typeof arg}`) ); - return new FluentNone(name); + return new FluentNone(`$${name}`); } } -/** - * Resolve a reference to a function. - * - * @param {Object} env - * Resolver environment object. - * @param {Object} expr - * An expression to be resolved. - * @param {String} expr.name - * Name of the function to be returned. - * @returns {Function} - * @private - */ -function FunctionReference(env, {name}) { - // Some functions are built-in. Others may be provided by the runtime via +// Resolve a reference to another message. +function MessageReference(env, {name, attr}) { + const {bundle, errors} = env; + const message = bundle._messages.get(name); + if (!message) { + const err = new ReferenceError(`Unknown message: ${name}`); + errors.push(err); + return new FluentNone(name); + } + + if (attr) { + const attribute = message.attrs && message.attrs[attr]; + if (attribute) { + return Type(env, attribute); + } + errors.push(new ReferenceError(`Unknown attribute: ${attr}`)); + return Type(env, message); + } + + return Type(env, message); +} + +// Resolve a call to a Term with key-value arguments. +function TermReference(env, {name, attr, selector, args}) { + const {bundle, errors} = env; + + const id = `-${name}`; + const term = bundle._terms.get(id); + if (!term) { + const err = new ReferenceError(`Unknown term: ${id}`); + errors.push(err); + return new FluentNone(id); + } + + // Every TermReference has its own args. + const [, keyargs] = getArguments(env, args); + const local = {...env, args: keyargs}; + + if (attr) { + const attribute = term.attrs && term.attrs[attr]; + if (attribute) { + return Type(local, attribute); + } + errors.push(new ReferenceError(`Unknown attribute: ${attr}`)); + return Type(local, term); + } + + const variantList = getVariantList(term); + if (selector && variantList) { + return SelectExpression(local, {...variantList, selector}); + } + + return Type(local, term); +} + +// Helper: convert a value into a variant list, if possible. +function getVariantList(term) { + const value = term.value || term; + return Array.isArray(value) + && value[0].type === "select" + && value[0].selector === null + ? value[0] + : null; +} + +// Resolve a call to a Function with positional and key-value arguments. +function FunctionReference(env, {name, args}) { + // Some functions are built-in. Others may be provided by the runtime via // the `FluentBundle` constructor. - const { bundle: { _functions }, errors } = env; + const {bundle: {_functions}, errors} = env; const func = _functions[name] || builtins[name]; if (!func) { @@ -551,59 +415,39 @@ function FunctionReference(env, {name}) { return new FluentNone(`${name}()`); } - return func; -} - -/** - * Resolve a call to a Function with positional and key-value arguments. - * - * @param {Object} env - * Resolver environment object. - * @param {Object} expr - * An expression to be resolved. - * @param {Object} expr.callee - * FTL Function object. - * @param {Array} expr.args - * FTL Function argument list. - * @returns {FluentType} - * @private - */ -function CallExpression(env, {callee, args}) { - const func = FunctionReference(env, callee); - - if (func instanceof FluentNone) { - return func; - } - - const posargs = []; - const keyargs = {}; - - for (const arg of args) { - if (arg.type === "narg") { - keyargs[arg.name] = Type(env, arg.value); - } else { - posargs.push(Type(env, arg)); - } - } - try { - return func(posargs, keyargs); + return func(...getArguments(env, args)); } catch (e) { // XXX Report errors. return new FluentNone(); } } -/** - * Resolve a pattern (a complex string with placeables). - * - * @param {Object} env - * Resolver environment object. - * @param {Array} ptn - * Array of pattern elements. - * @returns {Array} - * @private - */ +// Resolve a select expression to the member object. +function SelectExpression(env, {selector, variants, star}) { + if (selector === null) { + return getDefault(env, variants, star); + } + + let sel = Type(env, selector); + if (sel instanceof FluentNone) { + const variant = getDefault(env, variants, star); + return Type(env, variant); + } + + // Match the selector against keys of each variant, in order. + for (const variant of variants) { + const key = Type(env, variant.key); + if (match(env.bundle, sel, key)) { + return Type(env, variant); + } + } + + const variant = getDefault(env, variants, star); + return Type(env, variant); +} + +// Resolve a pattern (a complex string with placeables). function Pattern(env, ptn) { const { bundle, dirty, errors } = env; @@ -679,45 +523,44 @@ class FluentError extends Error {} // This regex is used to iterate through the beginnings of messages and terms. // With the /m flag, the ^ matches at the beginning of every line. -const RE_MESSAGE_START = /^(-?[a-zA-Z][a-zA-Z0-9_-]*) *= */mg; +const RE_MESSAGE_START = /^(-?[a-zA-Z][\w-]*) *= */mg; // Both Attributes and Variants are parsed in while loops. These regexes are // used to break out of them. -const RE_ATTRIBUTE_START = /\.([a-zA-Z][a-zA-Z0-9_-]*) *= */y; -// [^] matches all characters, including newlines. -// XXX Use /s (dotall) when it's widely supported. -const RE_VARIANT_START = /\*?\[[^]*?] */y; +const RE_ATTRIBUTE_START = /\.([a-zA-Z][\w-]*) *= */y; +const RE_VARIANT_START = /\*?\[/y; -const RE_IDENTIFIER = /(-?[a-zA-Z][a-zA-Z0-9_-]*)/y; const RE_NUMBER_LITERAL = /(-?[0-9]+(\.[0-9]+)?)/y; +const RE_IDENTIFIER = /([a-zA-Z][\w-]*)/y; +const RE_REFERENCE = /([$-])?([a-zA-Z][\w-]*)(?:\.([a-zA-Z][\w-]*))?/y; // A "run" is a sequence of text or string literal characters which don't -// require any special handling. For TextElements such special characters are: -// { (starts a placeable), \ (starts an escape sequence), and line breaks which -// require additional logic to check if the next line is indented. For -// StringLiterals they are: \ (starts an escape sequence), " (ends the -// literal), and line breaks which are not allowed in StringLiterals. Also note -// that string runs may be empty, but text runs may not. -const RE_TEXT_RUN = /([^\\{\n\r]+)/y; +// require any special handling. For TextElements such special characters are: { +// (starts a placeable), and line breaks which require additional logic to check +// if the next line is indented. For StringLiterals they are: \ (starts an +// escape sequence), " (ends the literal), and line breaks which are not allowed +// in StringLiterals. Note that string runs may be empty; text runs may not. +const RE_TEXT_RUN = /([^{}\n\r]+)/y; const RE_STRING_RUN = /([^\\"\n\r]*)/y; // Escape sequences. -const RE_UNICODE_ESCAPE = /\\u([a-fA-F0-9]{4})/y; const RE_STRING_ESCAPE = /\\([\\"])/y; -const RE_TEXT_ESCAPE = /\\([\\{])/y; +const RE_UNICODE_ESCAPE = /\\u([a-fA-F0-9]{4})|\\U([a-fA-F0-9]{6})/y; -// Used for trimming TextElements and indents. With the /m flag, the $ matches -// the end of every line. -const RE_TRAILING_SPACES = / +$/mg; -// CRLFs are normalized to LF. -const RE_CRLF = /\r\n/g; +// Used for trimming TextElements and indents. +const RE_LEADING_NEWLINES = /^\n+/; +const RE_TRAILING_SPACES = / +$/; +// Used in makeIndent to strip spaces from blank lines and normalize CRLF to LF. +const RE_BLANK_LINES = / *\r?\n/g; +// Used in makeIndent to measure the indentation. +const RE_INDENT = /( *)$/; // Common tokens. const TOKEN_BRACE_OPEN = /{\s*/y; const TOKEN_BRACE_CLOSE = /\s*}/y; const TOKEN_BRACKET_OPEN = /\[\s*/y; -const TOKEN_BRACKET_CLOSE = /\s*]/y; -const TOKEN_PAREN_OPEN = /\(\s*/y; +const TOKEN_BRACKET_CLOSE = /\s*] */y; +const TOKEN_PAREN_OPEN = /\s*\(\s*/y; const TOKEN_ARROW = /\s*->\s*/y; const TOKEN_COLON = /\s*:\s*/y; // Note the optional comma. As a deviation from the Fluent EBNF, the parser @@ -811,7 +654,7 @@ class FluentResource extends Map { return false; } - // Execute a regex, advance the cursor, and return the capture group. + // Execute a regex, advance the cursor, and return all capture groups. function match(re) { re.lastIndex = cursor; let result = re.exec(source); @@ -819,7 +662,12 @@ class FluentResource extends Map { throw new FluentError(`Expected ${re.toString()}`); } cursor = re.lastIndex; - return result[1]; + return result; + } + + // Execute a regex, advance the cursor, and return the capture group. + function match1(re) { + return match(re)[1]; } function parseMessage() { @@ -827,6 +675,9 @@ class FluentResource extends Map { let attrs = parseAttributes(); if (attrs === null) { + if (value === null) { + throw new FluentError("Expected message value or attributes"); + } return value; } @@ -835,67 +686,62 @@ class FluentResource extends Map { function parseAttributes() { let attrs = {}; - let hasAttributes = false; while (test(RE_ATTRIBUTE_START)) { - if (!hasAttributes) { - hasAttributes = true; + let name = match1(RE_ATTRIBUTE_START); + let value = parsePattern(); + if (value === null) { + throw new FluentError("Expected attribute value"); } - - let name = match(RE_ATTRIBUTE_START); - attrs[name] = parsePattern(); + attrs[name] = value; } - return hasAttributes ? attrs : null; + return Object.keys(attrs).length > 0 ? attrs : null; } function parsePattern() { // First try to parse any simple text on the same line as the id. if (test(RE_TEXT_RUN)) { - var first = match(RE_TEXT_RUN); + var first = match1(RE_TEXT_RUN); } - // If there's a backslash escape or a placeable on the first line, fall - // back to parsing a complex pattern. - switch (source[cursor]) { - case "{": - case "\\": - return first - // Re-use the text parsed above, if possible. - ? parsePatternElements(first) - : parsePatternElements(); + // If there's a placeable on the first line, parse a complex pattern. + if (source[cursor] === "{" || source[cursor] === "}") { + // Re-use the text parsed above, if possible. + return parsePatternElements(first ? [first] : [], Infinity); } // RE_TEXT_VALUE stops at newlines. Only continue parsing the pattern if // what comes after the newline is indented. let indent = parseIndent(); if (indent) { - return first + if (first) { // If there's text on the first line, the blank block is part of the - // translation content. - ? parsePatternElements(first, trim(indent)) - // Otherwise, we're dealing with a block pattern. The blank block is - // the leading whitespace; discard it. - : parsePatternElements(); + // translation content in its entirety. + return parsePatternElements([first, indent], indent.length); + } + // Otherwise, we're dealing with a block pattern, i.e. a pattern which + // starts on a new line. Discrad the leading newlines but keep the + // inline indent; it will be used by the dedentation logic. + indent.value = trim(indent.value, RE_LEADING_NEWLINES); + return parsePatternElements([indent], indent.length); } if (first) { // It was just a simple inline text after all. - return trim(first); + return trim(first, RE_TRAILING_SPACES); } return null; } // Parse a complex pattern as an array of elements. - function parsePatternElements(...elements) { + function parsePatternElements(elements = [], commonIndent) { let placeableCount = 0; - let needsTrimming = false; while (true) { if (test(RE_TEXT_RUN)) { - elements.push(match(RE_TEXT_RUN)); - needsTrimming = true; + elements.push(match1(RE_TEXT_RUN)); continue; } @@ -904,35 +750,43 @@ class FluentResource extends Map { throw new FluentError("Too many placeables"); } elements.push(parsePlaceable()); - needsTrimming = false; continue; } + if (source[cursor] === "}") { + throw new FluentError("Unbalanced closing brace"); + } + let indent = parseIndent(); if (indent) { - elements.push(trim(indent)); - needsTrimming = false; - continue; - } - - if (source[cursor] === "\\") { - elements.push(parseEscapeSequence(RE_TEXT_ESCAPE)); - needsTrimming = false; + elements.push(indent); + commonIndent = Math.min(commonIndent, indent.length); continue; } break; } - if (needsTrimming) { - // Trim the trailing whitespace of the last element if it's a - // TextElement. Use a flag rather than a typeof check to tell - // TextElements and StringLiterals apart (both are strings). - let lastIndex = elements.length - 1; - elements[lastIndex] = trim(elements[lastIndex]); + let lastIndex = elements.length - 1; + // Trim the trailing spaces in the last element if it's a TextElement. + if (typeof elements[lastIndex] === "string") { + elements[lastIndex] = trim(elements[lastIndex], RE_TRAILING_SPACES); } - return elements; + let baked = []; + for (let element of elements) { + if (element.type === "indent") { + // Dedent indented lines by the maximum common indent. + element = element.value.slice(0, element.value.length - commonIndent); + } else if (element.type === "str") { + // Optimize StringLiterals into their value. + element = element.value; + } + if (element) { + baked.push(element); + } + } + return baked; } function parsePlaceable() { @@ -965,28 +819,20 @@ class FluentResource extends Map { return parsePlaceable(); } - if (consumeChar("$")) { - return {type: "var", name: match(RE_IDENTIFIER)}; - } - - if (test(RE_IDENTIFIER)) { - let ref = {type: "ref", name: match(RE_IDENTIFIER)}; - - if (consumeChar(".")) { - let name = match(RE_IDENTIFIER); - return {type: "getattr", ref, name}; - } + if (test(RE_REFERENCE)) { + let [, sigil, name, attr = null] = match(RE_REFERENCE); + let type = {"$": "var", "-": "term"}[sigil] || "ref"; if (source[cursor] === "[") { - return {type: "getvar", ref, selector: parseVariantKey()}; + // DEPRECATED VariantExpressions will be removed before 1.0. + return {type, name, selector: parseVariantKey()}; } if (consumeToken(TOKEN_PAREN_OPEN)) { - let callee = {...ref, type: "func"}; - return {type: "call", callee, args: parseArguments()}; + return {type, name, attr, args: parseArguments()}; } - return ref; + return {type, name, attr, args: null}; } return parseLiteral(); @@ -1035,18 +881,29 @@ class FluentResource extends Map { } let key = parseVariantKey(); - cursor = RE_VARIANT_START.lastIndex; - variants[count++] = {key, value: parsePattern()}; + let value = parsePattern(); + if (value === null) { + throw new FluentError("Expected variant value"); + } + variants[count++] = {key, value}; } - return count > 0 ? {variants, star} : null; + if (count === 0) { + return null; + } + + if (star === undefined) { + throw new FluentError("Expected default variant"); + } + + return {variants, star}; } function parseVariantKey() { consumeToken(TOKEN_BRACKET_OPEN, FluentError); let key = test(RE_NUMBER_LITERAL) ? parseNumberLiteral() - : match(RE_IDENTIFIER); + : match1(RE_IDENTIFIER); consumeToken(TOKEN_BRACKET_CLOSE, FluentError); return key; } @@ -1064,22 +921,22 @@ class FluentResource extends Map { } function parseNumberLiteral() { - return {type: "num", value: match(RE_NUMBER_LITERAL)}; + return {type: "num", value: match1(RE_NUMBER_LITERAL)}; } function parseStringLiteral() { consumeChar("\"", FluentError); let value = ""; while (true) { - value += match(RE_STRING_RUN); + value += match1(RE_STRING_RUN); if (source[cursor] === "\\") { - value += parseEscapeSequence(RE_STRING_ESCAPE); + value += parseEscapeSequence(); continue; } if (consumeChar("\"")) { - return value; + return {type: "str", value}; } // We've reached an EOL of EOF. @@ -1088,14 +945,20 @@ class FluentResource extends Map { } // Unescape known escape sequences. - function parseEscapeSequence(reSpecialized) { - if (test(RE_UNICODE_ESCAPE)) { - let sequence = match(RE_UNICODE_ESCAPE); - return String.fromCodePoint(parseInt(sequence, 16)); + function parseEscapeSequence() { + if (test(RE_STRING_ESCAPE)) { + return match1(RE_STRING_ESCAPE); } - if (test(reSpecialized)) { - return match(reSpecialized); + if (test(RE_UNICODE_ESCAPE)) { + let [, codepoint4, codepoint6] = match(RE_UNICODE_ESCAPE); + let codepoint = parseInt(codepoint4 || codepoint6, 16); + return codepoint <= 0xD7FF || 0xE000 <= codepoint + // It's a Unicode scalar value. + ? String.fromCodePoint(codepoint) + // Lonely surrogates can cause trouble when the parsing result is + // saved using UTF-8. Use U+FFFD REPLACEMENT CHARACTER instead. + : "�"; } throw new FluentError("Unknown escape sequence"); @@ -1119,7 +982,7 @@ class FluentResource extends Map { case "{": // Placeables don't require indentation (in EBNF: block-placeable). // Continue the Pattern. - return source.slice(start, cursor).replace(RE_CRLF, "\n"); + return makeIndent(source.slice(start, cursor)); } // If the first character on the line is not one of the special characters @@ -1128,7 +991,7 @@ class FluentResource extends Map { if (source[cursor - 1] === " ") { // It's an indented text character (in EBNF: indented-char). Continue // the Pattern. - return source.slice(start, cursor).replace(RE_CRLF, "\n"); + return makeIndent(source.slice(start, cursor)); } // A not-indented text character is likely the identifier of the next @@ -1136,9 +999,16 @@ class FluentResource extends Map { return false; } - // Trim spaces trailing on every line of text. - function trim(text) { - return text.replace(RE_TRAILING_SPACES, ""); + // Trim blanks in text according to the given regex. + function trim(text, re) { + return text.replace(re, ""); + } + + // Normalize a blank block and extract the indent details. + function makeIndent(blank) { + let value = blank.replace(RE_BLANK_LINES, "\n"); + let length = RE_INDENT.exec(blank)[1].length; + return {type: "indent", value, length}; } } } diff --git a/intl/l10n/fluent.js.patch b/intl/l10n/fluent.js.patch index 88b91c567ccd..80d7c8c9847a 100644 --- a/intl/l10n/fluent.js.patch +++ b/intl/l10n/fluent.js.patch @@ -1,297 +1,270 @@ ---- ./dist/Fluent.jsm 2018-10-19 08:40:36.557032837 -0600 -+++ /home/zbraniecki/projects/mozilla-unified/intl/l10n/Fluent.jsm 2018-10-19 21:22:35.174315857 -0600 -@@ -16,7 +16,7 @@ +diff --git a/intl/l10n/DOMLocalization.jsm b/intl/l10n/DOMLocalization.jsm +--- a/intl/l10n/DOMLocalization.jsm ++++ b/intl/l10n/DOMLocalization.jsm +@@ -15,12 +15,13 @@ + * limitations under the License. */ +-/* fluent-dom@fa25466f (October 12, 2018) */ ++ ++/* fluent-dom@0.4.0 */ --/* fluent-dom@0.4.0 */ -+/* fluent@fa25466f (October 12, 2018) */ +-const { Localization } = +- ChromeUtils.import("resource://gre/modules/Localization.jsm", {}); +-const { Services } = +- ChromeUtils.import("resource://gre/modules/Services.jsm", {}); ++import Localization from '../../fluent-dom/src/localization.js'; ++ ++/* eslint no-console: ["error", {allow: ["warn"]}] */ ++/* global console */ - /* global Intl */ + // Match the opening angle bracket (<) in HTML tags, and HTML entities like + // &, &, &. +@@ -38,7 +39,7 @@ const TEXT_LEVEL_ELEMENTS = { + "http://www.w3.org/1999/xhtml": [ + "em", "strong", "small", "s", "cite", "q", "dfn", "abbr", "data", + "time", "code", "var", "samp", "kbd", "sub", "sup", "i", "b", "u", +- "mark", "bdi", "bdo", "span", "br", "wbr", ++ "mark", "bdi", "bdo", "span", "br", "wbr" + ], + }; -@@ -139,7 +139,53 @@ - return unwrapped; +@@ -56,17 +57,16 @@ const LOCALIZABLE_ATTRIBUTES = { + track: ["label"], + img: ["alt"], + textarea: ["placeholder"], +- th: ["abbr"], ++ th: ["abbr"] + }, + "http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul": { + global: [ +- "accesskey", "aria-label", "aria-valuetext", "aria-moz-hint", "label", +- "title", "tooltiptext"], +- description: ["value"], ++ "accesskey", "aria-label", "aria-valuetext", "aria-moz-hint", "label" ++ ], + key: ["key", "keycode"], +- label: ["value"], + textbox: ["placeholder"], +- }, ++ toolbarbutton: ["tooltiptext"], ++ } + }; + + +@@ -96,7 +96,6 @@ function translateElement(element, trans + const templateElement = element.ownerDocument.createElementNS( + "http://www.w3.org/1999/xhtml", "template" + ); +- // eslint-disable-next-line no-unsanitized/property + templateElement.innerHTML = value; + overlayChildNodes(templateElement.content, element); + } +@@ -350,46 +349,6 @@ function shallowPopulateUsing(fromElemen + return toElement; } --/* global Intl */ -+/** -+ * @overview -+ * -+ * The role of the Fluent resolver is to format a translation object to an -+ * instance of `FluentType` or an array of instances. -+ * -+ * Translations can contain references to other messages or variables, -+ * conditional logic in form of select expressions, traits which describe their -+ * grammatical features, and can use Fluent builtins which make use of the -+ * `Intl` formatters to format numbers, dates, lists and more into the -+ * bundle's language. See the documentation of the Fluent syntax for more -+ * information. -+ * -+ * In case of errors the resolver will try to salvage as much of the -+ * translation as possible. In rare situations where the resolver didn't know -+ * how to recover from an error it will return an instance of `FluentNone`. -+ * -+ * `MessageReference`, `VariantExpression`, `AttributeExpression` and -+ * `SelectExpression` resolve to raw Runtime Entries objects and the result of -+ * the resolution needs to be passed into `Type` to get their real value. -+ * This is useful for composing expressions. Consider: -+ * -+ * brand-name[nominative] -+ * -+ * which is a `VariantExpression` with properties `id: MessageReference` and -+ * `key: Keyword`. If `MessageReference` was resolved eagerly, it would -+ * instantly resolve to the value of the `brand-name` message. Instead, we -+ * want to get the message object and look for its `nominative` variant. -+ * -+ * All other expressions (except for `FunctionReference` which is only used in -+ * `CallExpression`) resolve to an instance of `FluentType`. The caller should -+ * use the `toString` method to convert the instance to a native value. -+ * -+ * -+ * All functions in this file pass around a special object called `env`. -+ * This object stores a set of elements used by all resolve functions: -+ * -+ * * {FluentBundle} bundle -+ * bundle for which the given resolution is happening -+ * * {Object} args -+ * list of developer provided arguments that can be used -+ * * {Array} errors -+ * list of errors collected while resolving -+ * * {WeakSet} dirty -+ * Set of patterns already encountered during this resolution. -+ * This is used to prevent cyclic resolutions. -+ */ +-/** +- * Sanitizes a translation before passing them to Node.localize API. +- * +- * It returns `false` if the translation contains DOM Overlays and should +- * not go into Node.localize. +- * +- * Note: There's a third item of work that JS DOM Overlays do - removal +- * of attributes from the previous translation. +- * This is not trivial to implement for Node.localize scenario, so +- * at the moment it is not supported. +- * +- * @param {{ +- * localName: string, +- * namespaceURI: string, +- * type: string || null +- * l10nId: string, +- * l10nArgs: Array || null, +- * l10nAttrs: string ||null, +- * }} l10nItems +- * @param {{value: string, attrs: Object}} translations +- * @returns boolean +- * @private +- */ +-function sanitizeTranslationForNodeLocalize(l10nItem, translation) { +- if (reOverlay.test(translation.value)) { +- return false; +- } +- +- if (translation.attributes) { +- const explicitlyAllowed = l10nItem.l10nAttrs === null ? null : +- l10nItem.l10nAttrs.split(",").map(i => i.trim()); +- for (const [j, {name}] of translation.attributes.entries()) { +- if (!isAttrNameLocalizable(name, l10nItem, explicitlyAllowed)) { +- translation.attributes.splice(j, 1); +- } +- } +- } +- return true; +-} +- + const L10NID_ATTR_NAME = "data-l10n-id"; + const L10NARGS_ATTR_NAME = "data-l10n-args"; - // Prevent expansion of too long placeables. - const MAX_PLACEABLE_LENGTH = 2500; -@@ -1319,14 +1365,6 @@ +@@ -427,12 +386,12 @@ class DOMLocalization extends Localizati + characterData: false, + childList: true, + subtree: true, +- attributeFilter: [L10NID_ATTR_NAME, L10NARGS_ATTR_NAME], ++ attributeFilter: [L10NID_ATTR_NAME, L10NARGS_ATTR_NAME] + }; + } + +- onChange(eager = false) { +- super.onChange(eager); ++ onChange() { ++ super.onChange(); + this.translateRoots(); + } + +@@ -497,7 +456,7 @@ class DOMLocalization extends Localizati + getAttributes(element) { + return { + id: element.getAttribute(L10NID_ATTR_NAME), +- args: JSON.parse(element.getAttribute(L10NARGS_ATTR_NAME) || null), ++ args: JSON.parse(element.getAttribute(L10NARGS_ATTR_NAME) || null) + }; + } + +@@ -519,17 +478,18 @@ class DOMLocalization extends Localizati + } + + if (this.windowElement) { +- if (this.windowElement !== newRoot.ownerGlobal) { ++ if (this.windowElement !== newRoot.ownerDocument.defaultView) { + throw new Error(`Cannot connect a root: + DOMLocalization already has a root from a different window.`); + } + } else { +- this.windowElement = newRoot.ownerGlobal; ++ this.windowElement = newRoot.ownerDocument.defaultView; + this.mutationObserver = new this.windowElement.MutationObserver( + mutations => this.translateMutations(mutations) + ); + } + ++ + this.roots.add(newRoot); + this.mutationObserver.observe(newRoot, this.observerConfig); + } +@@ -572,20 +532,7 @@ class DOMLocalization extends Localizati + translateRoots() { + const roots = Array.from(this.roots); + return Promise.all( +- roots.map(async root => { +- // We want to first retranslate the UI, and +- // then (potentially) flip the directionality. +- // +- // This means that the DOM alternations and directionality +- // are set in the same microtask. +- await this.translateFragment(root); +- let primaryLocale = Services.locale.appLocaleAsBCP47; +- let direction = Services.locale.isAppLocaleRTL ? "rtl" : "ltr"; +- root.setAttribute("lang", primaryLocale); +- root.setAttribute(root.namespaceURI === +- "http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" +- ? "localedir" : "dir", direction); +- }) ++ roots.map(root => this.translateFragment(root)) + ); + } + +@@ -652,10 +599,7 @@ class DOMLocalization extends Localizati + if (this.pendingElements.size > 0) { + if (this.pendingrAF === null) { + this.pendingrAF = this.windowElement.requestAnimationFrame(() => { +- // We need to filter for elements that lost their l10n-id while +- // waiting for the animation frame. +- this.translateElements(Array.from(this.pendingElements) +- .filter(elem => elem.hasAttribute("data-l10n-id"))); ++ this.translateElements(Array.from(this.pendingElements)); + this.pendingElements.clear(); + this.pendingrAF = null; + }); +@@ -677,63 +621,6 @@ class DOMLocalization extends Localizati + * @returns {Promise} + */ + translateFragment(frag) { +- if (frag.localize) { +- // This is a temporary fast-path offered by Gecko to workaround performance +- // issues coming from Fluent and XBL+Stylo performing unnecesary +- // operations during startup. +- // For details see bug 1441037, bug 1442262, and bug 1363862. +- +- // A sparse array which will store translations separated out from +- // all translations that is needed for DOM Overlay. +- const overlayTranslations = []; +- +- const getTranslationsForItems = async l10nItems => { +- const keys = l10nItems.map( +- l10nItem => ({id: l10nItem.l10nId, args: l10nItem.l10nArgs})); +- const translations = await this.formatMessages(keys); +- +- // Here we want to separate out elements that require DOM Overlays. +- // Those elements will have to be translated using our JS +- // implementation, while everything else is going to use the fast-path. +- for (const [i, translation] of translations.entries()) { +- if (translation === undefined) { +- continue; +- } +- +- const hasOnlyText = +- sanitizeTranslationForNodeLocalize(l10nItems[i], translation); +- if (!hasOnlyText) { +- // Removing from translations to make Node.localize skip it. +- // We will translate it below using JS DOM Overlays. +- overlayTranslations[i] = translations[i]; +- translations[i] = undefined; +- } +- } +- +- // We pause translation observing here because Node.localize +- // will translate the whole DOM next, using the `translations`. +- // +- // The observer will be resumed after DOM Overlays are localized +- // in the next microtask. +- this.pauseObserving(); +- return translations; +- }; +- +- return frag.localize(getTranslationsForItems.bind(this)) +- .then(untranslatedElements => { +- for (let i = 0; i < overlayTranslations.length; i++) { +- if (overlayTranslations[i] !== undefined && +- untranslatedElements[i] !== undefined) { +- translateElement(untranslatedElements[i], overlayTranslations[i]); +- } +- } +- this.resumeObserving(); +- }) +- .catch(e => { +- this.resumeObserving(); +- throw e; +- }); +- } + return this.translateElements(this.getTranslatables(frag)); + } + +@@ -808,10 +695,42 @@ class DOMLocalization extends Localizati + getKeysForElement(element) { + return { + id: element.getAttribute(L10NID_ATTR_NAME), +- args: JSON.parse(element.getAttribute(L10NARGS_ATTR_NAME) || null), ++ args: JSON.parse(element.getAttribute(L10NARGS_ATTR_NAME) || null) + }; } } --/* -- * @module fluent -- * @overview -- * -- * `fluent` is a JavaScript implementation of Project Fluent, a localization -- * framework designed to unleash the expressive power of the natural language. -- * -- */ -- - this.FluentBundle = FluentBundle; --this.EXPORTED_SYMBOLS = ["FluentBundle"]; -+this.FluentResource = FluentResource; -+var EXPORTED_SYMBOLS = ["FluentBundle", "FluentResource"]; ---- ./dist/Localization.jsm 2018-10-19 08:40:36.773712561 -0600 -+++ /home/zbraniecki/projects/mozilla-unified/intl/l10n/Localization.jsm 2018-10-19 21:20:57.295233460 -0600 -@@ -16,27 +16,34 @@ - */ - - --/* fluent-dom@0.4.0 */ -+/* fluent-dom@fa25466f (October 12, 2018) */ +-this.DOMLocalization = DOMLocalization; +-var EXPORTED_SYMBOLS = ["DOMLocalization"]; ++/* global L10nRegistry, Services */ + -+/* eslint no-console: ["error", { allow: ["warn", "error"] }] */ -+/* global console */ -+ -+const { L10nRegistry } = ChromeUtils.import("resource://gre/modules/L10nRegistry.jsm", {}); -+const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm", {}); -+const { AppConstants } = ChromeUtils.import("resource://gre/modules/AppConstants.jsm", {}); - - /* - * Base CachedIterable class. - */ - class CachedIterable extends Array { -- /** -- * Create a `CachedIterable` instance from an iterable or, if another -- * instance of `CachedIterable` is passed, return it without any -- * modifications. -- * -- * @param {Iterable} iterable -- * @returns {CachedIterable} -- */ -- static from(iterable) { -- if (iterable instanceof this) { -- return iterable; -- } -- -- return new this(iterable); -+ /** -+ * Create a `CachedIterable` instance from an iterable or, if another -+ * instance of `CachedIterable` is passed, return it without any -+ * modifications. -+ * -+ * @param {Iterable} iterable -+ * @returns {CachedIterable} -+ */ -+ static from(iterable) { -+ if (iterable instanceof this) { -+ return iterable; - } -+ -+ return new this(iterable); -+ } - } - - /* -@@ -46,88 +53,100 @@ - * iterable. - */ - class CachedAsyncIterable extends CachedIterable { -- /** -- * Create an `CachedAsyncIterable` instance. -- * -- * @param {Iterable} iterable -- * @returns {CachedAsyncIterable} -- */ -- constructor(iterable) { -- super(); -- -- if (Symbol.asyncIterator in Object(iterable)) { -- this.iterator = iterable[Symbol.asyncIterator](); -- } else if (Symbol.iterator in Object(iterable)) { -- this.iterator = iterable[Symbol.iterator](); -- } else { -- throw new TypeError("Argument must implement the iteration protocol."); -- } -- } -+ /** -+ * Create an `CachedAsyncIterable` instance. -+ * -+ * @param {Iterable} iterable -+ * @returns {CachedAsyncIterable} -+ */ -+ constructor(iterable) { -+ super(); - -- /** -- * Synchronous iterator over the cached elements. -- * -- * Return a generator object implementing the iterator protocol over the -- * cached elements of the original (async or sync) iterable. -- */ -- [Symbol.iterator]() { -- const cached = this; -- let cur = 0; -- -- return { -- next() { -- if (cached.length === cur) { -- return {value: undefined, done: true}; -- } -- return cached[cur++]; -- } -- }; -+ if (Symbol.asyncIterator in Object(iterable)) { -+ this.iterator = iterable[Symbol.asyncIterator](); -+ } else if (Symbol.iterator in Object(iterable)) { -+ this.iterator = iterable[Symbol.iterator](); -+ } else { -+ throw new TypeError("Argument must implement the iteration protocol."); - } -+ } - -- /** -- * Asynchronous iterator caching the yielded elements. -- * -- * Elements yielded by the original iterable will be cached and available -- * synchronously. Returns an async generator object implementing the -- * iterator protocol over the elements of the original (async or sync) -- * iterable. -- */ -- [Symbol.asyncIterator]() { -- const cached = this; -- let cur = 0; -- -- return { -- async next() { -- if (cached.length <= cur) { -- cached.push(await cached.iterator.next()); -- } -- return cached[cur++]; -- } -- }; -- } -+ /** -+ * Synchronous iterator over the cached elements. -+ * -+ * Return a generator object implementing the iterator protocol over the -+ * cached elements of the original (async or sync) iterable. -+ */ -+ [Symbol.iterator]() { -+ const cached = this; -+ let cur = 0; -+ -+ return { -+ next() { -+ if (cached.length === cur) { -+ return {value: undefined, done: true}; -+ } -+ return cached[cur++]; -+ } -+ }; -+ } - -- /** -- * This method allows user to consume the next element from the iterator -- * into the cache. -- * -- * @param {number} count - number of elements to consume -- */ -- async touchNext(count = 1) { -- let idx = 0; -- while (idx++ < count) { -- const last = this[this.length - 1]; -- if (last && last.done) { -- break; -- } -- this.push(await this.iterator.next()); -+ /** -+ * Asynchronous iterator caching the yielded elements. -+ * -+ * Elements yielded by the original iterable will be cached and available -+ * synchronously. Returns an async generator object implementing the -+ * iterator protocol over the elements of the original (async or sync) -+ * iterable. -+ */ -+ [Symbol.asyncIterator]() { -+ const cached = this; -+ let cur = 0; -+ -+ return { -+ async next() { -+ if (cached.length <= cur) { -+ cached.push(await cached.iterator.next()); - } -- // Return the last cached {value, done} object to allow the calling -- // code to decide if it needs to call touchNext again. -- return this[this.length - 1]; -+ return cached[cur++]; -+ } -+ }; -+ } -+ -+ /** -+ * This method allows user to consume the next element from the iterator -+ * into the cache. -+ * -+ * @param {number} count - number of elements to consume -+ */ -+ async touchNext(count = 1) { -+ let idx = 0; -+ while (idx++ < count) { -+ const last = this[this.length - 1]; -+ if (last && last.done) { -+ break; -+ } -+ this.push(await this.iterator.next()); - } -+ // Return the last cached {value, done} object to allow the calling -+ // code to decide if it needs to call touchNext again. -+ return this[this.length - 1]; -+ } - } - --/* eslint no-console: ["error", { allow: ["warn", "error"] }] */ +/** + * The default localization strategy for Gecko. It comabines locales + * available in L10nRegistry, with locales requested by the user to @@ -302,435 +275,1240 @@ + * be localized into a different language - for example DevTools. + */ +function defaultGenerateBundles(resourceIds) { -+ const appLocales = Services.locale.appLocalesAsBCP47; -+ return L10nRegistry.generateContexts(appLocales, resourceIds); ++ const requestedLocales = Services.locale.getRequestedLocales(); ++ const availableLocales = L10nRegistry.getAvailableLocales(); ++ const defaultLocale = Services.locale.defaultLocale; ++ const locales = Services.locale.negotiateLanguages( ++ requestedLocales, availableLocales, defaultLocale, ++ ); ++ return L10nRegistry.generateContexts(locales, resourceIds); +} ++ ++ ++class GeckoDOMLocalization extends DOMLocalization { ++ constructor( ++ windowElement, ++ resourceIds, ++ generateBundles = defaultGenerateBundles ++ ) { ++ super(windowElement, resourceIds, generateBundles); ++ } ++} ++ ++this.DOMLocalization = GeckoDOMLocalization; ++this.EXPORTED_SYMBOLS = ["DOMLocalization"]; +diff --git a/intl/l10n/Fluent.jsm b/intl/l10n/Fluent.jsm +--- a/intl/l10n/Fluent.jsm ++++ b/intl/l10n/Fluent.jsm +@@ -16,7 +16,7 @@ + */ + + +-/* fluent@0.10.0 */ ++/* fluent-dom@0.4.0 */ + + /* global Intl */ + +@@ -139,53 +139,7 @@ function values(opts) { + return unwrapped; + } + +-/** +- * @overview +- * +- * The role of the Fluent resolver is to format a translation object to an +- * instance of `FluentType` or an array of instances. +- * +- * Translations can contain references to other messages or variables, +- * conditional logic in form of select expressions, traits which describe their +- * grammatical features, and can use Fluent builtins which make use of the +- * `Intl` formatters to format numbers, dates, lists and more into the +- * bundle's language. See the documentation of the Fluent syntax for more +- * information. +- * +- * In case of errors the resolver will try to salvage as much of the +- * translation as possible. In rare situations where the resolver didn't know +- * how to recover from an error it will return an instance of `FluentNone`. +- * +- * `MessageReference`, `VariantExpression`, `AttributeExpression` and +- * `SelectExpression` resolve to raw Runtime Entries objects and the result of +- * the resolution needs to be passed into `Type` to get their real value. +- * This is useful for composing expressions. Consider: +- * +- * brand-name[nominative] +- * +- * which is a `VariantExpression` with properties `id: MessageReference` and +- * `key: Keyword`. If `MessageReference` was resolved eagerly, it would +- * instantly resolve to the value of the `brand-name` message. Instead, we +- * want to get the message object and look for its `nominative` variant. +- * +- * All other expressions (except for `FunctionReference` which is only used in +- * `CallExpression`) resolve to an instance of `FluentType`. The caller should +- * use the `toString` method to convert the instance to a native value. +- * +- * +- * All functions in this file pass around a special object called `env`. +- * This object stores a set of elements used by all resolve functions: +- * +- * * {FluentBundle} bundle +- * bundle for which the given resolution is happening +- * * {Object} args +- * list of developer provided arguments that can be used +- * * {Array} errors +- * list of errors collected while resolving +- * * {WeakSet} dirty +- * Set of patterns already encountered during this resolution. +- * This is used to prevent cyclic resolutions. +- */ ++/* global Intl */ + + // Prevent expansion of too long placeables. + const MAX_PLACEABLE_LENGTH = 2500; +@@ -514,7 +468,7 @@ function Pattern(env, ptn) { + */ + function resolve(bundle, args, message, errors = []) { + const env = { +- bundle, args, errors, dirty: new WeakSet(), ++ bundle, args, errors, dirty: new WeakSet() + }; + return Type(env, message).toString(bundle); + } +@@ -1064,7 +1018,7 @@ class FluentBundle { + constructor(locales, { + functions = {}, + useIsolating = true, +- transform = v => v, ++ transform = v => v + } = {}) { + this.locales = Array.isArray(locales) ? locales : [locales]; + +@@ -1235,6 +1189,14 @@ class FluentBundle { + } + } + ++/* ++ * @module fluent ++ * @overview ++ * ++ * `fluent` is a JavaScript implementation of Project Fluent, a localization ++ * framework designed to unleash the expressive power of the natural language. ++ * ++ */ ++ + this.FluentBundle = FluentBundle; +-this.FluentResource = FluentResource; +-var EXPORTED_SYMBOLS = ["FluentBundle", "FluentResource"]; ++this.EXPORTED_SYMBOLS = ["FluentBundle"]; +diff --git a/intl/l10n/Localization.jsm b/intl/l10n/Localization.jsm +--- a/intl/l10n/Localization.jsm ++++ b/intl/l10n/Localization.jsm +@@ -16,34 +16,27 @@ + */ + + +-/* fluent-dom@fa25466f (October 12, 2018) */ +- +-/* eslint no-console: ["error", { allow: ["warn", "error"] }] */ +-/* global console */ +- +-const { L10nRegistry } = ChromeUtils.import("resource://gre/modules/L10nRegistry.jsm", {}); +-const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm", {}); +-const { AppConstants } = ChromeUtils.import("resource://gre/modules/AppConstants.jsm", {}); ++/* fluent-dom@0.4.0 */ + + /* + * Base CachedIterable class. + */ + class CachedIterable extends Array { +- /** +- * Create a `CachedIterable` instance from an iterable or, if another +- * instance of `CachedIterable` is passed, return it without any +- * modifications. +- * +- * @param {Iterable} iterable +- * @returns {CachedIterable} +- */ +- static from(iterable) { +- if (iterable instanceof this) { +- return iterable; ++ /** ++ * Create a `CachedIterable` instance from an iterable or, if another ++ * instance of `CachedIterable` is passed, return it without any ++ * modifications. ++ * ++ * @param {Iterable} iterable ++ * @returns {CachedIterable} ++ */ ++ static from(iterable) { ++ if (iterable instanceof this) { ++ return iterable; ++ } ++ ++ return new this(iterable); + } +- +- return new this(iterable); +- } + } + + /* +@@ -53,80 +46,88 @@ class CachedIterable extends Array { + * iterable. + */ + class CachedAsyncIterable extends CachedIterable { +- /** +- * Create an `CachedAsyncIterable` instance. +- * +- * @param {Iterable} iterable +- * @returns {CachedAsyncIterable} +- */ +- constructor(iterable) { +- super(); ++ /** ++ * Create an `CachedAsyncIterable` instance. ++ * ++ * @param {Iterable} iterable ++ * @returns {CachedAsyncIterable} ++ */ ++ constructor(iterable) { ++ super(); ++ ++ if (Symbol.asyncIterator in Object(iterable)) { ++ this.iterator = iterable[Symbol.asyncIterator](); ++ } else if (Symbol.iterator in Object(iterable)) { ++ this.iterator = iterable[Symbol.iterator](); ++ } else { ++ throw new TypeError("Argument must implement the iteration protocol."); ++ } ++ } + +- if (Symbol.asyncIterator in Object(iterable)) { +- this.iterator = iterable[Symbol.asyncIterator](); +- } else if (Symbol.iterator in Object(iterable)) { +- this.iterator = iterable[Symbol.iterator](); +- } else { +- throw new TypeError("Argument must implement the iteration protocol."); +- } +- } ++ /** ++ * Synchronous iterator over the cached elements. ++ * ++ * Return a generator object implementing the iterator protocol over the ++ * cached elements of the original (async or sync) iterable. ++ */ ++ [Symbol.iterator]() { ++ const cached = this; ++ let cur = 0; + +- /** +- * Asynchronous iterator caching the yielded elements. +- * +- * Elements yielded by the original iterable will be cached and available +- * synchronously. Returns an async generator object implementing the +- * iterator protocol over the elements of the original (async or sync) +- * iterable. +- */ +- [Symbol.asyncIterator]() { +- const cached = this; +- let cur = 0; ++ return { ++ next() { ++ if (cached.length === cur) { ++ return {value: undefined, done: true}; ++ } ++ return cached[cur++]; ++ } ++ }; ++ } + +- return { +- async next() { +- if (cached.length <= cur) { +- cached.push(cached.iterator.next()); +- } +- return cached[cur++]; +- }, +- }; +- } ++ /** ++ * Asynchronous iterator caching the yielded elements. ++ * ++ * Elements yielded by the original iterable will be cached and available ++ * synchronously. Returns an async generator object implementing the ++ * iterator protocol over the elements of the original (async or sync) ++ * iterable. ++ */ ++ [Symbol.asyncIterator]() { ++ const cached = this; ++ let cur = 0; + +- /** +- * This method allows user to consume the next element from the iterator +- * into the cache. +- * +- * @param {number} count - number of elements to consume +- */ +- async touchNext(count = 1) { +- let idx = 0; +- while (idx++ < count) { +- const last = this[this.length - 1]; +- if (last && (await last).done) { +- break; +- } +- this.push(this.iterator.next()); ++ return { ++ async next() { ++ if (cached.length <= cur) { ++ cached.push(await cached.iterator.next()); ++ } ++ return cached[cur++]; ++ } ++ }; + } +- // Return the last cached {value, done} object to allow the calling +- // code to decide if it needs to call touchNext again. +- return this[this.length - 1]; +- } ++ ++ /** ++ * This method allows user to consume the next element from the iterator ++ * into the cache. ++ * ++ * @param {number} count - number of elements to consume ++ */ ++ async touchNext(count = 1) { ++ let idx = 0; ++ while (idx++ < count) { ++ const last = this[this.length - 1]; ++ if (last && last.done) { ++ break; ++ } ++ this.push(await this.iterator.next()); ++ } ++ // Return the last cached {value, done} object to allow the calling ++ // code to decide if it needs to call touchNext again. ++ return this[this.length - 1]; ++ } + } + +-/** +- * The default localization strategy for Gecko. It comabines locales +- * available in L10nRegistry, with locales requested by the user to +- * generate the iterator over FluentBundles. +- * +- * In the future, we may want to allow certain modules to override this +- * with a different negotitation strategy to allow for the module to +- * be localized into a different language - for example DevTools. +- */ +-function defaultGenerateBundles(resourceIds) { +- const appLocales = Services.locale.appLocalesAsBCP47; +- return L10nRegistry.generateBundles(appLocales, resourceIds); +-} ++/* eslint no-console: ["error", { allow: ["warn", "error"] }] */ /** * The `Localization` class is a central high-level API for vanilla -@@ -143,16 +162,21 @@ +@@ -142,21 +143,16 @@ class Localization { * * @returns {Localization} */ -- constructor(resourceIds = [], generateBundles) { -+ constructor(resourceIds = [], generateBundles = defaultGenerateBundles) { +- constructor(resourceIds = [], generateBundles = defaultGenerateBundles) { ++ constructor(resourceIds = [], generateBundles) { this.resourceIds = resourceIds; this.generateBundles = generateBundles; this.bundles = CachedAsyncIterable.from( this.generateBundles(this.resourceIds)); } -- addResourceIds(resourceIds) { -+ /** -+ * @param {Array} resourceIds - List of resource IDs -+ * @param {bool} eager - whether the I/O for new context should -+ * begin eagerly -+ */ -+ addResourceIds(resourceIds, eager = false) { +- /** +- * @param {Array} resourceIds - List of resource IDs +- * @param {bool} eager - whether the I/O for new context should +- * begin eagerly +- */ +- addResourceIds(resourceIds, eager = false) { ++ addResourceIds(resourceIds) { this.resourceIds.push(...resourceIds); -- this.onChange(); -+ this.onChange(eager); +- this.onChange(eager); ++ this.onChange(); return this.resourceIds.length; } -@@ -184,9 +208,12 @@ +@@ -188,12 +184,9 @@ class Localization { break; } -- if (typeof console !== "undefined") { -+ if (AppConstants.NIGHTLY_BUILD || Cu.isInAutomation) { +- if (AppConstants.NIGHTLY_BUILD || Cu.isInAutomation) { ++ if (typeof console !== "undefined") { const locale = bundle.locales[0]; const ids = Array.from(missingIds).join(", "); -+ if (Cu.isInAutomation) { -+ throw new Error(`Missing translations in ${locale}: ${ids}`); -+ } +- if (Cu.isInAutomation) { +- throw new Error(`Missing translations in ${locale}: ${ids}`); +- } console.warn(`Missing translations in ${locale}: ${ids}`); } } -@@ -274,21 +301,64 @@ +@@ -281,64 +274,21 @@ class Localization { return val; } -- handleEvent() { -- this.onChange(); -+ /** -+ * Register weak observers on events that will trigger cache invalidation -+ */ -+ registerObservers() { -+ Services.obs.addObserver(this, "intl:app-locales-changed", true); -+ Services.prefs.addObserver("intl.l10n.pseudo", this, true); -+ } -+ -+ /** -+ * Default observer handler method. -+ * -+ * @param {String} subject -+ * @param {String} topic -+ * @param {Object} data -+ */ -+ observe(subject, topic, data) { -+ switch (topic) { -+ case "intl:app-locales-changed": -+ this.onChange(); -+ break; -+ case "nsPref:changed": -+ switch (data) { -+ case "intl.l10n.pseudo": -+ this.onChange(); -+ } -+ break; -+ default: -+ break; -+ } +- /** +- * Register weak observers on events that will trigger cache invalidation +- */ +- registerObservers() { +- Services.obs.addObserver(this, "intl:app-locales-changed", true); +- Services.prefs.addObserver("intl.l10n.pseudo", this, true); +- } +- +- /** +- * Default observer handler method. +- * +- * @param {String} subject +- * @param {String} topic +- * @param {Object} data +- */ +- observe(subject, topic, data) { +- switch (topic) { +- case "intl:app-locales-changed": +- this.onChange(); +- break; +- case "nsPref:changed": +- switch (data) { +- case "intl.l10n.pseudo": +- this.onChange(); +- } +- break; +- default: +- break; +- } ++ handleEvent() { ++ this.onChange(); } /** * This method should be called when there's a reason to believe * that language negotiation or available resources changed. -+ * -+ * @param {bool} eager - whether the I/O for new context should begin eagerly +- * +- * @param {bool} eager - whether the I/O for new context should begin eagerly */ -- onChange() { -+ onChange(eager = false) { +- onChange(eager = false) { ++ onChange() { this.bundles = CachedAsyncIterable.from( this.generateBundles(this.resourceIds)); -- this.bundles.touchNext(2); -+ if (eager) { -+ // If the first app locale is the same as last fallback -+ // it means that we have all resources in this locale, and -+ // we want to eagerly fetch just that one. -+ // Otherwise, we're in a scenario where the first locale may -+ // be partial and we want to eagerly fetch a fallback as well. -+ const appLocale = Services.locale.appLocaleAsBCP47; -+ const lastFallback = Services.locale.lastFallbackLocale; -+ const prefetchCount = appLocale === lastFallback ? 1 : 2; -+ this.bundles.touchNext(prefetchCount); -+ } +- if (eager) { +- // If the first app locale is the same as last fallback +- // it means that we have all resources in this locale, and +- // we want to eagerly fetch just that one. +- // Otherwise, we're in a scenario where the first locale may +- // be partial and we want to eagerly fetch a fallback as well. +- const appLocale = Services.locale.appLocaleAsBCP47; +- const lastFallback = Services.locale.lastFallbackLocale; +- const prefetchCount = appLocale === lastFallback ? 1 : 2; +- this.bundles.touchNext(prefetchCount); +- } ++ this.bundles.touchNext(2); } } -+Localization.prototype.QueryInterface = ChromeUtils.generateQI([ -+ Ci.nsISupportsWeakReference -+]); -+ +-Localization.prototype.QueryInterface = ChromeUtils.generateQI([ +- Ci.nsISupportsWeakReference, +-]); +- /** * Format the value of a message into a string. * -@@ -380,7 +450,7 @@ +@@ -430,7 +380,7 @@ function messageFromBundle(bundle, error * See `Localization.formatWithFallback` for more info on how this is used. * * @param {Function} method -- * @param {FluentBundle} bundle -+ * @param {FluentBundle} bundle +- * @param {FluentBundle} bundle ++ * @param {FluentBundle} bundle * @param {Array} keys * @param {{Array<{value: string, attributes: Object}>}} translations * -@@ -408,44 +478,5 @@ +@@ -458,5 +408,44 @@ function keysFromBundle(method, bundle, return missingIds; } --/* global Components */ --/* eslint no-unused-vars: 0 */ -- --const Cu = Components.utils; --const Cc = Components.classes; --const Ci = Components.interfaces; -- --const { L10nRegistry } = -- Cu.import("resource://gre/modules/L10nRegistry.jsm", {}); --const ObserverService = -- Cc["@mozilla.org/observer-service;1"].getService(Ci.nsIObserverService); --const { Services } = -- Cu.import("resource://gre/modules/Services.jsm", {}); -- --/** -- * The default localization strategy for Gecko. It comabines locales -- * available in L10nRegistry, with locales requested by the user to -- * generate the iterator over FluentBundles. -- * -- * In the future, we may want to allow certain modules to override this -- * with a different negotitation strategy to allow for the module to -- * be localized into a different language - for example DevTools. -- */ --function defaultGenerateBundles(resourceIds) { -- const requestedLocales = Services.locale.getRequestedLocales(); -- const availableLocales = L10nRegistry.getAvailableLocales(); -- const defaultLocale = Services.locale.defaultLocale; -- const locales = Services.locale.negotiateLanguages( -- requestedLocales, availableLocales, defaultLocale, -- ); -- return L10nRegistry.generateContexts(locales, resourceIds); --} -- --class GeckoLocalization extends Localization { -- constructor(resourceIds, generateBundles = defaultGenerateBundles) { -- super(resourceIds, generateBundles); -- } --} -- --this.Localization = GeckoLocalization; --this.EXPORTED_SYMBOLS = ["Localization"]; -+this.Localization = Localization; -+var EXPORTED_SYMBOLS = ["Localization"]; ---- ./dist/DOMLocalization.jsm 2018-10-19 08:40:37.000392886 -0600 -+++ /home/zbraniecki/projects/mozilla-unified/intl/l10n/DOMLocalization.jsm 2018-10-19 21:38:25.963726161 -0600 -@@ -15,13 +15,12 @@ - * limitations under the License. - */ - -+/* fluent-dom@fa25466f (October 12, 2018) */ - --/* fluent-dom@0.4.0 */ -- --import Localization from '../../fluent-dom/src/localization.js'; -- --/* eslint no-console: ["error", {allow: ["warn"]}] */ --/* global console */ -+const { Localization } = -+ ChromeUtils.import("resource://gre/modules/Localization.jsm", {}); -+const { Services } = -+ ChromeUtils.import("resource://gre/modules/Services.jsm", {}); - - // Match the opening angle bracket (<) in HTML tags, and HTML entities like - // &, &, &. -@@ -61,11 +60,12 @@ - }, - "http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul": { - global: [ -- "accesskey", "aria-label", "aria-valuetext", "aria-moz-hint", "label" -- ], -+ "accesskey", "aria-label", "aria-valuetext", "aria-moz-hint", "label", -+ "title", "tooltiptext"], -+ description: ["value"], - key: ["key", "keycode"], -+ label: ["value"], - textbox: ["placeholder"], -- toolbarbutton: ["tooltiptext"], - } - }; - -@@ -96,6 +96,7 @@ - const templateElement = element.ownerDocument.createElementNS( - "http://www.w3.org/1999/xhtml", "template" - ); -+ // eslint-disable-next-line no-unsanitized/property - templateElement.innerHTML = value; - overlayChildNodes(templateElement.content, element); - } -@@ -349,6 +350,46 @@ - return toElement; - } - -+/** -+ * Sanitizes a translation before passing them to Node.localize API. -+ * -+ * It returns `false` if the translation contains DOM Overlays and should -+ * not go into Node.localize. -+ * -+ * Note: There's a third item of work that JS DOM Overlays do - removal -+ * of attributes from the previous translation. -+ * This is not trivial to implement for Node.localize scenario, so -+ * at the moment it is not supported. -+ * -+ * @param {{ -+ * localName: string, -+ * namespaceURI: string, -+ * type: string || null -+ * l10nId: string, -+ * l10nArgs: Array || null, -+ * l10nAttrs: string ||null, -+ * }} l10nItems -+ * @param {{value: string, attrs: Object}} translations -+ * @returns boolean -+ * @private -+ */ -+function sanitizeTranslationForNodeLocalize(l10nItem, translation) { -+ if (reOverlay.test(translation.value)) { -+ return false; -+ } +-this.Localization = Localization; +-var EXPORTED_SYMBOLS = ["Localization"]; ++/* global Components */ ++/* eslint no-unused-vars: 0 */ + -+ if (translation.attributes) { -+ const explicitlyAllowed = l10nItem.l10nAttrs === null ? null : -+ l10nItem.l10nAttrs.split(",").map(i => i.trim()); -+ for (const [j, {name}] of translation.attributes.entries()) { -+ if (!isAttrNameLocalizable(name, l10nItem, explicitlyAllowed)) { -+ translation.attributes.splice(j, 1); -+ } -+ } -+ } -+ return true; ++const Cu = Components.utils; ++const Cc = Components.classes; ++const Ci = Components.interfaces; ++ ++const { L10nRegistry } = ++ Cu.import("resource://gre/modules/L10nRegistry.jsm", {}); ++const ObserverService = ++ Cc["@mozilla.org/observer-service;1"].getService(Ci.nsIObserverService); ++const { Services } = ++ Cu.import("resource://gre/modules/Services.jsm", {}); ++ ++/** ++ * The default localization strategy for Gecko. It comabines locales ++ * available in L10nRegistry, with locales requested by the user to ++ * generate the iterator over FluentBundles. ++ * ++ * In the future, we may want to allow certain modules to override this ++ * with a different negotitation strategy to allow for the module to ++ * be localized into a different language - for example DevTools. ++ */ ++function defaultGenerateBundles(resourceIds) { ++ const requestedLocales = Services.locale.getRequestedLocales(); ++ const availableLocales = L10nRegistry.getAvailableLocales(); ++ const defaultLocale = Services.locale.defaultLocale; ++ const locales = Services.locale.negotiateLanguages( ++ requestedLocales, availableLocales, defaultLocale, ++ ); ++ return L10nRegistry.generateContexts(locales, resourceIds); +} + - const L10NID_ATTR_NAME = "data-l10n-id"; - const L10NARGS_ATTR_NAME = "data-l10n-args"; - -@@ -390,8 +431,8 @@ - }; - } - -- onChange() { -- super.onChange(); -+ onChange(eager = false) { -+ super.onChange(eager); - this.translateRoots(); - } - -@@ -478,18 +519,17 @@ - } - - if (this.windowElement) { -- if (this.windowElement !== newRoot.ownerDocument.defaultView) { -+ if (this.windowElement !== newRoot.ownerGlobal) { - throw new Error(`Cannot connect a root: - DOMLocalization already has a root from a different window.`); - } - } else { -- this.windowElement = newRoot.ownerDocument.defaultView; -+ this.windowElement = newRoot.ownerGlobal; - this.mutationObserver = new this.windowElement.MutationObserver( - mutations => this.translateMutations(mutations) - ); - } - -- - this.roots.add(newRoot); - this.mutationObserver.observe(newRoot, this.observerConfig); - } -@@ -532,7 +572,20 @@ - translateRoots() { - const roots = Array.from(this.roots); - return Promise.all( -- roots.map(root => this.translateFragment(root)) -+ roots.map(async root => { -+ // We want to first retranslate the UI, and -+ // then (potentially) flip the directionality. -+ // -+ // This means that the DOM alternations and directionality -+ // are set in the same microtask. -+ await this.translateFragment(root); -+ let primaryLocale = Services.locale.appLocaleAsBCP47; -+ let direction = Services.locale.isAppLocaleRTL ? "rtl" : "ltr"; -+ root.setAttribute("lang", primaryLocale); -+ root.setAttribute(root.namespaceURI === -+ "http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" -+ ? "localedir" : "dir", direction); -+ }) - ); - } - -@@ -599,7 +652,10 @@ - if (this.pendingElements.size > 0) { - if (this.pendingrAF === null) { - this.pendingrAF = this.windowElement.requestAnimationFrame(() => { -- this.translateElements(Array.from(this.pendingElements)); -+ // We need to filter for elements that lost their l10n-id while -+ // waiting for the animation frame. -+ this.translateElements(Array.from(this.pendingElements) -+ .filter(elem => elem.hasAttribute("data-l10n-id"))); - this.pendingElements.clear(); - this.pendingrAF = null; - }); -@@ -621,6 +677,63 @@ - * @returns {Promise} - */ - translateFragment(frag) { -+ if (frag.localize) { -+ // This is a temporary fast-path offered by Gecko to workaround performance -+ // issues coming from Fluent and XBL+Stylo performing unnecesary -+ // operations during startup. -+ // For details see bug 1441037, bug 1442262, and bug 1363862. ++class GeckoLocalization extends Localization { ++ constructor(resourceIds, generateBundles = defaultGenerateBundles) { ++ super(resourceIds, generateBundles); ++ } ++} + -+ // A sparse array which will store translations separated out from -+ // all translations that is needed for DOM Overlay. -+ const overlayTranslations = []; -+ -+ const getTranslationsForItems = async l10nItems => { -+ const keys = l10nItems.map( -+ l10nItem => ({id: l10nItem.l10nId, args: l10nItem.l10nArgs})); -+ const translations = await this.formatMessages(keys); -+ -+ // Here we want to separate out elements that require DOM Overlays. -+ // Those elements will have to be translated using our JS -+ // implementation, while everything else is going to use the fast-path. -+ for (const [i, translation] of translations.entries()) { -+ if (translation === undefined) { -+ continue; -+ } -+ -+ const hasOnlyText = -+ sanitizeTranslationForNodeLocalize(l10nItems[i], translation); -+ if (!hasOnlyText) { -+ // Removing from translations to make Node.localize skip it. -+ // We will translate it below using JS DOM Overlays. -+ overlayTranslations[i] = translations[i]; -+ translations[i] = undefined; -+ } -+ } -+ -+ // We pause translation observing here because Node.localize -+ // will translate the whole DOM next, using the `translations`. -+ // -+ // The observer will be resumed after DOM Overlays are localized -+ // in the next microtask. -+ this.pauseObserving(); -+ return translations; -+ }; -+ -+ return frag.localize(getTranslationsForItems.bind(this)) -+ .then(untranslatedElements => { -+ for (let i = 0; i < overlayTranslations.length; i++) { -+ if (overlayTranslations[i] !== undefined && -+ untranslatedElements[i] !== undefined) { -+ translateElement(untranslatedElements[i], overlayTranslations[i]); -+ } -+ } -+ this.resumeObserving(); -+ }) -+ .catch(e => { -+ this.resumeObserving(); -+ throw e; -+ }); -+ } - return this.translateElements(this.getTranslatables(frag)); - } - -@@ -700,37 +813,5 @@ - } - } - --/* global L10nRegistry, Services */ -- --/** -- * The default localization strategy for Gecko. It comabines locales -- * available in L10nRegistry, with locales requested by the user to -- * generate the iterator over FluentBundles. -- * -- * In the future, we may want to allow certain modules to override this -- * with a different negotitation strategy to allow for the module to -- * be localized into a different language - for example DevTools. -- */ --function defaultGenerateBundles(resourceIds) { -- const requestedLocales = Services.locale.getRequestedLocales(); -- const availableLocales = L10nRegistry.getAvailableLocales(); -- const defaultLocale = Services.locale.defaultLocale; -- const locales = Services.locale.negotiateLanguages( -- requestedLocales, availableLocales, defaultLocale, -- ); -- return L10nRegistry.generateContexts(locales, resourceIds); --} -- -- --class GeckoDOMLocalization extends DOMLocalization { -- constructor( -- windowElement, -- resourceIds, -- generateBundles = defaultGenerateBundles -- ) { -- super(windowElement, resourceIds, generateBundles); -- } --} -- --this.DOMLocalization = GeckoDOMLocalization; --this.EXPORTED_SYMBOLS = ["DOMLocalization"]; -+this.DOMLocalization = DOMLocalization; -+var EXPORTED_SYMBOLS = ["DOMLocalization"]; ++this.Localization = GeckoLocalization; ++this.EXPORTED_SYMBOLS = ["Localization"]; +diff --git a/intl/l10n/fluent.js.patch b/intl/l10n/fluent.js.patch +--- a/intl/l10n/fluent.js.patch ++++ b/intl/l10n/fluent.js.patch +@@ -1,736 +0,0 @@ +---- ./dist/Fluent.jsm 2018-10-19 08:40:36.557032837 -0600 +-+++ /home/zbraniecki/projects/mozilla-unified/intl/l10n/Fluent.jsm 2018-10-19 21:22:35.174315857 -0600 +-@@ -16,7 +16,7 @@ +- */ +- +- +--/* fluent-dom@0.4.0 */ +-+/* fluent@fa25466f (October 12, 2018) */ +- +- /* global Intl */ +- +-@@ -139,7 +139,53 @@ +- return unwrapped; +- } +- +--/* global Intl */ +-+/** +-+ * @overview +-+ * +-+ * The role of the Fluent resolver is to format a translation object to an +-+ * instance of `FluentType` or an array of instances. +-+ * +-+ * Translations can contain references to other messages or variables, +-+ * conditional logic in form of select expressions, traits which describe their +-+ * grammatical features, and can use Fluent builtins which make use of the +-+ * `Intl` formatters to format numbers, dates, lists and more into the +-+ * bundle's language. See the documentation of the Fluent syntax for more +-+ * information. +-+ * +-+ * In case of errors the resolver will try to salvage as much of the +-+ * translation as possible. In rare situations where the resolver didn't know +-+ * how to recover from an error it will return an instance of `FluentNone`. +-+ * +-+ * `MessageReference`, `VariantExpression`, `AttributeExpression` and +-+ * `SelectExpression` resolve to raw Runtime Entries objects and the result of +-+ * the resolution needs to be passed into `Type` to get their real value. +-+ * This is useful for composing expressions. Consider: +-+ * +-+ * brand-name[nominative] +-+ * +-+ * which is a `VariantExpression` with properties `id: MessageReference` and +-+ * `key: Keyword`. If `MessageReference` was resolved eagerly, it would +-+ * instantly resolve to the value of the `brand-name` message. Instead, we +-+ * want to get the message object and look for its `nominative` variant. +-+ * +-+ * All other expressions (except for `FunctionReference` which is only used in +-+ * `CallExpression`) resolve to an instance of `FluentType`. The caller should +-+ * use the `toString` method to convert the instance to a native value. +-+ * +-+ * +-+ * All functions in this file pass around a special object called `env`. +-+ * This object stores a set of elements used by all resolve functions: +-+ * +-+ * * {FluentBundle} bundle +-+ * bundle for which the given resolution is happening +-+ * * {Object} args +-+ * list of developer provided arguments that can be used +-+ * * {Array} errors +-+ * list of errors collected while resolving +-+ * * {WeakSet} dirty +-+ * Set of patterns already encountered during this resolution. +-+ * This is used to prevent cyclic resolutions. +-+ */ +- +- // Prevent expansion of too long placeables. +- const MAX_PLACEABLE_LENGTH = 2500; +-@@ -1319,14 +1365,6 @@ +- } +- } +- +--/* +-- * @module fluent +-- * @overview +-- * +-- * `fluent` is a JavaScript implementation of Project Fluent, a localization +-- * framework designed to unleash the expressive power of the natural language. +-- * +-- */ +-- +- this.FluentBundle = FluentBundle; +--this.EXPORTED_SYMBOLS = ["FluentBundle"]; +-+this.FluentResource = FluentResource; +-+var EXPORTED_SYMBOLS = ["FluentBundle", "FluentResource"]; +---- ./dist/Localization.jsm 2018-10-19 08:40:36.773712561 -0600 +-+++ /home/zbraniecki/projects/mozilla-unified/intl/l10n/Localization.jsm 2018-10-19 21:20:57.295233460 -0600 +-@@ -16,27 +16,34 @@ +- */ +- +- +--/* fluent-dom@0.4.0 */ +-+/* fluent-dom@fa25466f (October 12, 2018) */ +-+ +-+/* eslint no-console: ["error", { allow: ["warn", "error"] }] */ +-+/* global console */ +-+ +-+const { L10nRegistry } = ChromeUtils.import("resource://gre/modules/L10nRegistry.jsm", {}); +-+const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm", {}); +-+const { AppConstants } = ChromeUtils.import("resource://gre/modules/AppConstants.jsm", {}); +- +- /* +- * Base CachedIterable class. +- */ +- class CachedIterable extends Array { +-- /** +-- * Create a `CachedIterable` instance from an iterable or, if another +-- * instance of `CachedIterable` is passed, return it without any +-- * modifications. +-- * +-- * @param {Iterable} iterable +-- * @returns {CachedIterable} +-- */ +-- static from(iterable) { +-- if (iterable instanceof this) { +-- return iterable; +-- } +-- +-- return new this(iterable); +-+ /** +-+ * Create a `CachedIterable` instance from an iterable or, if another +-+ * instance of `CachedIterable` is passed, return it without any +-+ * modifications. +-+ * +-+ * @param {Iterable} iterable +-+ * @returns {CachedIterable} +-+ */ +-+ static from(iterable) { +-+ if (iterable instanceof this) { +-+ return iterable; +- } +-+ +-+ return new this(iterable); +-+ } +- } +- +- /* +-@@ -46,88 +53,100 @@ +- * iterable. +- */ +- class CachedAsyncIterable extends CachedIterable { +-- /** +-- * Create an `CachedAsyncIterable` instance. +-- * +-- * @param {Iterable} iterable +-- * @returns {CachedAsyncIterable} +-- */ +-- constructor(iterable) { +-- super(); +-- +-- if (Symbol.asyncIterator in Object(iterable)) { +-- this.iterator = iterable[Symbol.asyncIterator](); +-- } else if (Symbol.iterator in Object(iterable)) { +-- this.iterator = iterable[Symbol.iterator](); +-- } else { +-- throw new TypeError("Argument must implement the iteration protocol."); +-- } +-- } +-+ /** +-+ * Create an `CachedAsyncIterable` instance. +-+ * +-+ * @param {Iterable} iterable +-+ * @returns {CachedAsyncIterable} +-+ */ +-+ constructor(iterable) { +-+ super(); +- +-- /** +-- * Synchronous iterator over the cached elements. +-- * +-- * Return a generator object implementing the iterator protocol over the +-- * cached elements of the original (async or sync) iterable. +-- */ +-- [Symbol.iterator]() { +-- const cached = this; +-- let cur = 0; +-- +-- return { +-- next() { +-- if (cached.length === cur) { +-- return {value: undefined, done: true}; +-- } +-- return cached[cur++]; +-- } +-- }; +-+ if (Symbol.asyncIterator in Object(iterable)) { +-+ this.iterator = iterable[Symbol.asyncIterator](); +-+ } else if (Symbol.iterator in Object(iterable)) { +-+ this.iterator = iterable[Symbol.iterator](); +-+ } else { +-+ throw new TypeError("Argument must implement the iteration protocol."); +- } +-+ } +- +-- /** +-- * Asynchronous iterator caching the yielded elements. +-- * +-- * Elements yielded by the original iterable will be cached and available +-- * synchronously. Returns an async generator object implementing the +-- * iterator protocol over the elements of the original (async or sync) +-- * iterable. +-- */ +-- [Symbol.asyncIterator]() { +-- const cached = this; +-- let cur = 0; +-- +-- return { +-- async next() { +-- if (cached.length <= cur) { +-- cached.push(await cached.iterator.next()); +-- } +-- return cached[cur++]; +-- } +-- }; +-- } +-+ /** +-+ * Synchronous iterator over the cached elements. +-+ * +-+ * Return a generator object implementing the iterator protocol over the +-+ * cached elements of the original (async or sync) iterable. +-+ */ +-+ [Symbol.iterator]() { +-+ const cached = this; +-+ let cur = 0; +-+ +-+ return { +-+ next() { +-+ if (cached.length === cur) { +-+ return {value: undefined, done: true}; +-+ } +-+ return cached[cur++]; +-+ } +-+ }; +-+ } +- +-- /** +-- * This method allows user to consume the next element from the iterator +-- * into the cache. +-- * +-- * @param {number} count - number of elements to consume +-- */ +-- async touchNext(count = 1) { +-- let idx = 0; +-- while (idx++ < count) { +-- const last = this[this.length - 1]; +-- if (last && last.done) { +-- break; +-- } +-- this.push(await this.iterator.next()); +-+ /** +-+ * Asynchronous iterator caching the yielded elements. +-+ * +-+ * Elements yielded by the original iterable will be cached and available +-+ * synchronously. Returns an async generator object implementing the +-+ * iterator protocol over the elements of the original (async or sync) +-+ * iterable. +-+ */ +-+ [Symbol.asyncIterator]() { +-+ const cached = this; +-+ let cur = 0; +-+ +-+ return { +-+ async next() { +-+ if (cached.length <= cur) { +-+ cached.push(await cached.iterator.next()); +- } +-- // Return the last cached {value, done} object to allow the calling +-- // code to decide if it needs to call touchNext again. +-- return this[this.length - 1]; +-+ return cached[cur++]; +-+ } +-+ }; +-+ } +-+ +-+ /** +-+ * This method allows user to consume the next element from the iterator +-+ * into the cache. +-+ * +-+ * @param {number} count - number of elements to consume +-+ */ +-+ async touchNext(count = 1) { +-+ let idx = 0; +-+ while (idx++ < count) { +-+ const last = this[this.length - 1]; +-+ if (last && last.done) { +-+ break; +-+ } +-+ this.push(await this.iterator.next()); +- } +-+ // Return the last cached {value, done} object to allow the calling +-+ // code to decide if it needs to call touchNext again. +-+ return this[this.length - 1]; +-+ } +- } +- +--/* eslint no-console: ["error", { allow: ["warn", "error"] }] */ +-+/** +-+ * The default localization strategy for Gecko. It comabines locales +-+ * available in L10nRegistry, with locales requested by the user to +-+ * generate the iterator over FluentBundles. +-+ * +-+ * In the future, we may want to allow certain modules to override this +-+ * with a different negotitation strategy to allow for the module to +-+ * be localized into a different language - for example DevTools. +-+ */ +-+function defaultGenerateBundles(resourceIds) { +-+ const appLocales = Services.locale.appLocalesAsBCP47; +-+ return L10nRegistry.generateContexts(appLocales, resourceIds); +-+} +- +- /** +- * The `Localization` class is a central high-level API for vanilla +-@@ -143,16 +162,21 @@ +- * +- * @returns {Localization} +- */ +-- constructor(resourceIds = [], generateBundles) { +-+ constructor(resourceIds = [], generateBundles = defaultGenerateBundles) { +- this.resourceIds = resourceIds; +- this.generateBundles = generateBundles; +- this.bundles = CachedAsyncIterable.from( +- this.generateBundles(this.resourceIds)); +- } +- +-- addResourceIds(resourceIds) { +-+ /** +-+ * @param {Array} resourceIds - List of resource IDs +-+ * @param {bool} eager - whether the I/O for new context should +-+ * begin eagerly +-+ */ +-+ addResourceIds(resourceIds, eager = false) { +- this.resourceIds.push(...resourceIds); +-- this.onChange(); +-+ this.onChange(eager); +- return this.resourceIds.length; +- } +- +-@@ -184,9 +208,12 @@ +- break; +- } +- +-- if (typeof console !== "undefined") { +-+ if (AppConstants.NIGHTLY_BUILD || Cu.isInAutomation) { +- const locale = bundle.locales[0]; +- const ids = Array.from(missingIds).join(", "); +-+ if (Cu.isInAutomation) { +-+ throw new Error(`Missing translations in ${locale}: ${ids}`); +-+ } +- console.warn(`Missing translations in ${locale}: ${ids}`); +- } +- } +-@@ -274,21 +301,64 @@ +- return val; +- } +- +-- handleEvent() { +-- this.onChange(); +-+ /** +-+ * Register weak observers on events that will trigger cache invalidation +-+ */ +-+ registerObservers() { +-+ Services.obs.addObserver(this, "intl:app-locales-changed", true); +-+ Services.prefs.addObserver("intl.l10n.pseudo", this, true); +-+ } +-+ +-+ /** +-+ * Default observer handler method. +-+ * +-+ * @param {String} subject +-+ * @param {String} topic +-+ * @param {Object} data +-+ */ +-+ observe(subject, topic, data) { +-+ switch (topic) { +-+ case "intl:app-locales-changed": +-+ this.onChange(); +-+ break; +-+ case "nsPref:changed": +-+ switch (data) { +-+ case "intl.l10n.pseudo": +-+ this.onChange(); +-+ } +-+ break; +-+ default: +-+ break; +-+ } +- } +- +- /** +- * This method should be called when there's a reason to believe +- * that language negotiation or available resources changed. +-+ * +-+ * @param {bool} eager - whether the I/O for new context should begin eagerly +- */ +-- onChange() { +-+ onChange(eager = false) { +- this.bundles = CachedAsyncIterable.from( +- this.generateBundles(this.resourceIds)); +-- this.bundles.touchNext(2); +-+ if (eager) { +-+ // If the first app locale is the same as last fallback +-+ // it means that we have all resources in this locale, and +-+ // we want to eagerly fetch just that one. +-+ // Otherwise, we're in a scenario where the first locale may +-+ // be partial and we want to eagerly fetch a fallback as well. +-+ const appLocale = Services.locale.appLocaleAsBCP47; +-+ const lastFallback = Services.locale.lastFallbackLocale; +-+ const prefetchCount = appLocale === lastFallback ? 1 : 2; +-+ this.bundles.touchNext(prefetchCount); +-+ } +- } +- } +- +-+Localization.prototype.QueryInterface = ChromeUtils.generateQI([ +-+ Ci.nsISupportsWeakReference +-+]); +-+ +- /** +- * Format the value of a message into a string. +- * +-@@ -380,7 +450,7 @@ +- * See `Localization.formatWithFallback` for more info on how this is used. +- * +- * @param {Function} method +-- * @param {FluentBundle} bundle +-+ * @param {FluentBundle} bundle +- * @param {Array} keys +- * @param {{Array<{value: string, attributes: Object}>}} translations +- * +-@@ -408,44 +478,5 @@ +- return missingIds; +- } +- +--/* global Components */ +--/* eslint no-unused-vars: 0 */ +-- +--const Cu = Components.utils; +--const Cc = Components.classes; +--const Ci = Components.interfaces; +-- +--const { L10nRegistry } = +-- Cu.import("resource://gre/modules/L10nRegistry.jsm", {}); +--const ObserverService = +-- Cc["@mozilla.org/observer-service;1"].getService(Ci.nsIObserverService); +--const { Services } = +-- Cu.import("resource://gre/modules/Services.jsm", {}); +-- +--/** +-- * The default localization strategy for Gecko. It comabines locales +-- * available in L10nRegistry, with locales requested by the user to +-- * generate the iterator over FluentBundles. +-- * +-- * In the future, we may want to allow certain modules to override this +-- * with a different negotitation strategy to allow for the module to +-- * be localized into a different language - for example DevTools. +-- */ +--function defaultGenerateBundles(resourceIds) { +-- const requestedLocales = Services.locale.getRequestedLocales(); +-- const availableLocales = L10nRegistry.getAvailableLocales(); +-- const defaultLocale = Services.locale.defaultLocale; +-- const locales = Services.locale.negotiateLanguages( +-- requestedLocales, availableLocales, defaultLocale, +-- ); +-- return L10nRegistry.generateContexts(locales, resourceIds); +--} +-- +--class GeckoLocalization extends Localization { +-- constructor(resourceIds, generateBundles = defaultGenerateBundles) { +-- super(resourceIds, generateBundles); +-- } +--} +-- +--this.Localization = GeckoLocalization; +--this.EXPORTED_SYMBOLS = ["Localization"]; +-+this.Localization = Localization; +-+var EXPORTED_SYMBOLS = ["Localization"]; +---- ./dist/DOMLocalization.jsm 2018-10-19 08:40:37.000392886 -0600 +-+++ /home/zbraniecki/projects/mozilla-unified/intl/l10n/DOMLocalization.jsm 2018-10-19 21:38:25.963726161 -0600 +-@@ -15,13 +15,12 @@ +- * limitations under the License. +- */ +- +-+/* fluent-dom@fa25466f (October 12, 2018) */ +- +--/* fluent-dom@0.4.0 */ +-- +--import Localization from '../../fluent-dom/src/localization.js'; +-- +--/* eslint no-console: ["error", {allow: ["warn"]}] */ +--/* global console */ +-+const { Localization } = +-+ ChromeUtils.import("resource://gre/modules/Localization.jsm", {}); +-+const { Services } = +-+ ChromeUtils.import("resource://gre/modules/Services.jsm", {}); +- +- // Match the opening angle bracket (<) in HTML tags, and HTML entities like +- // &, &, &. +-@@ -61,11 +60,12 @@ +- }, +- "http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul": { +- global: [ +-- "accesskey", "aria-label", "aria-valuetext", "aria-moz-hint", "label" +-- ], +-+ "accesskey", "aria-label", "aria-valuetext", "aria-moz-hint", "label", +-+ "title", "tooltiptext"], +-+ description: ["value"], +- key: ["key", "keycode"], +-+ label: ["value"], +- textbox: ["placeholder"], +-- toolbarbutton: ["tooltiptext"], +- } +- }; +- +-@@ -96,6 +96,7 @@ +- const templateElement = element.ownerDocument.createElementNS( +- "http://www.w3.org/1999/xhtml", "template" +- ); +-+ // eslint-disable-next-line no-unsanitized/property +- templateElement.innerHTML = value; +- overlayChildNodes(templateElement.content, element); +- } +-@@ -349,6 +350,46 @@ +- return toElement; +- } +- +-+/** +-+ * Sanitizes a translation before passing them to Node.localize API. +-+ * +-+ * It returns `false` if the translation contains DOM Overlays and should +-+ * not go into Node.localize. +-+ * +-+ * Note: There's a third item of work that JS DOM Overlays do - removal +-+ * of attributes from the previous translation. +-+ * This is not trivial to implement for Node.localize scenario, so +-+ * at the moment it is not supported. +-+ * +-+ * @param {{ +-+ * localName: string, +-+ * namespaceURI: string, +-+ * type: string || null +-+ * l10nId: string, +-+ * l10nArgs: Array || null, +-+ * l10nAttrs: string ||null, +-+ * }} l10nItems +-+ * @param {{value: string, attrs: Object}} translations +-+ * @returns boolean +-+ * @private +-+ */ +-+function sanitizeTranslationForNodeLocalize(l10nItem, translation) { +-+ if (reOverlay.test(translation.value)) { +-+ return false; +-+ } +-+ +-+ if (translation.attributes) { +-+ const explicitlyAllowed = l10nItem.l10nAttrs === null ? null : +-+ l10nItem.l10nAttrs.split(",").map(i => i.trim()); +-+ for (const [j, {name}] of translation.attributes.entries()) { +-+ if (!isAttrNameLocalizable(name, l10nItem, explicitlyAllowed)) { +-+ translation.attributes.splice(j, 1); +-+ } +-+ } +-+ } +-+ return true; +-+} +-+ +- const L10NID_ATTR_NAME = "data-l10n-id"; +- const L10NARGS_ATTR_NAME = "data-l10n-args"; +- +-@@ -390,8 +431,8 @@ +- }; +- } +- +-- onChange() { +-- super.onChange(); +-+ onChange(eager = false) { +-+ super.onChange(eager); +- this.translateRoots(); +- } +- +-@@ -478,18 +519,17 @@ +- } +- +- if (this.windowElement) { +-- if (this.windowElement !== newRoot.ownerDocument.defaultView) { +-+ if (this.windowElement !== newRoot.ownerGlobal) { +- throw new Error(`Cannot connect a root: +- DOMLocalization already has a root from a different window.`); +- } +- } else { +-- this.windowElement = newRoot.ownerDocument.defaultView; +-+ this.windowElement = newRoot.ownerGlobal; +- this.mutationObserver = new this.windowElement.MutationObserver( +- mutations => this.translateMutations(mutations) +- ); +- } +- +-- +- this.roots.add(newRoot); +- this.mutationObserver.observe(newRoot, this.observerConfig); +- } +-@@ -532,7 +572,20 @@ +- translateRoots() { +- const roots = Array.from(this.roots); +- return Promise.all( +-- roots.map(root => this.translateFragment(root)) +-+ roots.map(async root => { +-+ // We want to first retranslate the UI, and +-+ // then (potentially) flip the directionality. +-+ // +-+ // This means that the DOM alternations and directionality +-+ // are set in the same microtask. +-+ await this.translateFragment(root); +-+ let primaryLocale = Services.locale.appLocaleAsBCP47; +-+ let direction = Services.locale.isAppLocaleRTL ? "rtl" : "ltr"; +-+ root.setAttribute("lang", primaryLocale); +-+ root.setAttribute(root.namespaceURI === +-+ "http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul" +-+ ? "localedir" : "dir", direction); +-+ }) +- ); +- } +- +-@@ -599,7 +652,10 @@ +- if (this.pendingElements.size > 0) { +- if (this.pendingrAF === null) { +- this.pendingrAF = this.windowElement.requestAnimationFrame(() => { +-- this.translateElements(Array.from(this.pendingElements)); +-+ // We need to filter for elements that lost their l10n-id while +-+ // waiting for the animation frame. +-+ this.translateElements(Array.from(this.pendingElements) +-+ .filter(elem => elem.hasAttribute("data-l10n-id"))); +- this.pendingElements.clear(); +- this.pendingrAF = null; +- }); +-@@ -621,6 +677,63 @@ +- * @returns {Promise} +- */ +- translateFragment(frag) { +-+ if (frag.localize) { +-+ // This is a temporary fast-path offered by Gecko to workaround performance +-+ // issues coming from Fluent and XBL+Stylo performing unnecesary +-+ // operations during startup. +-+ // For details see bug 1441037, bug 1442262, and bug 1363862. +-+ +-+ // A sparse array which will store translations separated out from +-+ // all translations that is needed for DOM Overlay. +-+ const overlayTranslations = []; +-+ +-+ const getTranslationsForItems = async l10nItems => { +-+ const keys = l10nItems.map( +-+ l10nItem => ({id: l10nItem.l10nId, args: l10nItem.l10nArgs})); +-+ const translations = await this.formatMessages(keys); +-+ +-+ // Here we want to separate out elements that require DOM Overlays. +-+ // Those elements will have to be translated using our JS +-+ // implementation, while everything else is going to use the fast-path. +-+ for (const [i, translation] of translations.entries()) { +-+ if (translation === undefined) { +-+ continue; +-+ } +-+ +-+ const hasOnlyText = +-+ sanitizeTranslationForNodeLocalize(l10nItems[i], translation); +-+ if (!hasOnlyText) { +-+ // Removing from translations to make Node.localize skip it. +-+ // We will translate it below using JS DOM Overlays. +-+ overlayTranslations[i] = translations[i]; +-+ translations[i] = undefined; +-+ } +-+ } +-+ +-+ // We pause translation observing here because Node.localize +-+ // will translate the whole DOM next, using the `translations`. +-+ // +-+ // The observer will be resumed after DOM Overlays are localized +-+ // in the next microtask. +-+ this.pauseObserving(); +-+ return translations; +-+ }; +-+ +-+ return frag.localize(getTranslationsForItems.bind(this)) +-+ .then(untranslatedElements => { +-+ for (let i = 0; i < overlayTranslations.length; i++) { +-+ if (overlayTranslations[i] !== undefined && +-+ untranslatedElements[i] !== undefined) { +-+ translateElement(untranslatedElements[i], overlayTranslations[i]); +-+ } +-+ } +-+ this.resumeObserving(); +-+ }) +-+ .catch(e => { +-+ this.resumeObserving(); +-+ throw e; +-+ }); +-+ } +- return this.translateElements(this.getTranslatables(frag)); +- } +- +-@@ -700,37 +813,5 @@ +- } +- } +- +--/* global L10nRegistry, Services */ +-- +--/** +-- * The default localization strategy for Gecko. It comabines locales +-- * available in L10nRegistry, with locales requested by the user to +-- * generate the iterator over FluentBundles. +-- * +-- * In the future, we may want to allow certain modules to override this +-- * with a different negotitation strategy to allow for the module to +-- * be localized into a different language - for example DevTools. +-- */ +--function defaultGenerateBundles(resourceIds) { +-- const requestedLocales = Services.locale.getRequestedLocales(); +-- const availableLocales = L10nRegistry.getAvailableLocales(); +-- const defaultLocale = Services.locale.defaultLocale; +-- const locales = Services.locale.negotiateLanguages( +-- requestedLocales, availableLocales, defaultLocale, +-- ); +-- return L10nRegistry.generateContexts(locales, resourceIds); +--} +-- +-- +--class GeckoDOMLocalization extends DOMLocalization { +-- constructor( +-- windowElement, +-- resourceIds, +-- generateBundles = defaultGenerateBundles +-- ) { +-- super(windowElement, resourceIds, generateBundles); +-- } +--} +-- +--this.DOMLocalization = GeckoDOMLocalization; +--this.EXPORTED_SYMBOLS = ["DOMLocalization"]; +-+this.DOMLocalization = DOMLocalization; +-+var EXPORTED_SYMBOLS = ["DOMLocalization"];