bug 1370176, generic configuration for l10n, r=flod,gps

The current way to configure compare-locales has a lot of assumptions that make our l10n system really stubborn. The generic configuration is independent of python, and uses toml files for configuration. They're still modular, but there's only one file format. See http://moz-l10n-config.readthedocs.io/en/latest/fileformat.html for the specification. Also fixes a few nits in filter.py, where we compared the entity key as bool, which is false if we pass in ''. Explicitly compare as "entity is None" to be precise about when we're checking files. MozReview-Commit-ID: 5TmfobaImF4 --HG-- extra : rebase_source : 84e56eb2076e74f79677df9e0368811579c1f173
2017-06-05 10:28:33 +02:00 · 2017-06-05 10:28:33 +02:00 · c92e5dfc1e
--- a/browser/locales/filter.py
+++ b/browser/locales/filter.py
@ -18,7 +18,7 @@ def test(mod, path, entity = None):
  if mod not in ("browser", "extensions/spellcheck"):
    # we only have exceptions for browser and extensions/spellcheck
    return "error"
-  if not entity:
+  if entity is None:
    # the only files to ignore are spell checkers
    if mod == "extensions/spellcheck":
      return "ignore"
--- a/browser/locales/l10n.toml
+++ b/browser/locales/l10n.toml
@ -0,0 +1,169 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+basepath = "../.."
+
+locales = [
+    "ach",
+    "af",
+    "an",
+    "ar",
+    "as",
+    "ast",
+    "az",
+    "be",
+    "bg",
+    "bn-BD",
+    "bn-IN",
+    "br",
+    "bs",
+    "ca",
+    "cak",
+    "cs",
+    "cy",
+    "da",
+    "de",
+    "dsb",
+    "el",
+    "en-GB",
+    "en-ZA",
+    "eo",
+    "es-AR",
+    "es-CL",
+    "es-ES",
+    "es-MX",
+    "et",
+    "eu",
+    "fa",
+    "ff",
+    "fi",
+    "fr",
+    "fy-NL",
+    "ga-IE",
+    "gd",
+    "gl",
+    "gn",
+    "gu-IN",
+    "he",
+    "hi-IN",
+    "hr",
+    "hsb",
+    "hu",
+    "hy-AM",
+    "id",
+    "is",
+    "it",
+    "ja",
+    "ja-JP-mac",
+    "ka",
+    "kab",
+    "kk",
+    "km",
+    "kn",
+    "ko",
+    "lij",
+    "lo",
+    "lt",
+    "ltg",
+    "lv",
+    "mai",
+    "mk",
+    "ml",
+    "mr",
+    "ms",
+    "my",
+    "nb-NO",
+    "ne-NP",
+    "nl",
+    "nn-NO",
+    "or",
+    "pa-IN",
+    "pl",
+    "pt-BR",
+    "pt-PT",
+    "rm",
+    "ro",
+    "ru",
+    "si",
+    "sk",
+    "sl",
+    "son",
+    "sq",
+    "sr",
+    "sv-SE",
+    "ta",
+    "te",
+    "th",
+    "tl",
+    "tr",
+    "uk",
+    "ur",
+    "uz",
+    "vi",
+    "xh",
+    "zh-CN",
+    "zh-TW",
+    ]
+
+[env]
+    l = "{l10n_base}/{locale}/"
+
+
+[[paths]]
+    reference = "browser/locales/en-US/**"
+    l10n = "{l}browser/**"
+
+[[paths]]
+    reference = "browser/branding/official/locales/en-US/**"
+    l10n = "{l}browser/branding/official/**"
+
+[[paths]]
+    reference = "browser/extensions/onboarding/locales/en-US/**"
+    l10n = "{l}browser/extensions/onboarding/**"
+
+[[paths]]
+    reference = "browser/extensions/webcompat-reporter/locales/en-US/**"
+    l10n = "{l}browser/extensions/webcompat-reporter/**"
+
+[[paths]]
+    reference = "services/sync/locales/en-US/**"
+    l10n = "{l}services/sync/**"
+
+
+[[includes]]
+    path = "toolkit/locales/l10n.toml"
+
+[[includes]]
+    path = "devtools/client/locales/l10n.toml"
+
+# Filters
+# The filters below are evaluated one after the other, in the given order.
+# Enter a combination of path as in the localization, key, and an action,
+# to change the default behavior of compare-locales and l10n merge.
+#
+# For browser/locales/en-US/chrome/browser/foo.properties,
+# path would be {l}browser/chrome/browser/foo.properties
+# key: the key/id of the entity
+# If key isn't specified, the complete file can be missing.
+[[filters]]
+    path = "{l}browser/defines.inc"
+    key = "MOZ_LANGPACK_CONTRIBUTORS"
+    action = "ignore"
+
+[[filters]]
+    path = [
+        "{l}browser/defines.inc",
+        "{l}browser/firefox-l10n.js",
+    ]
+    action = "ignore"
+
+[[filters]]
+    path = "{l}browser/chrome/browser-region/region.properties"
+    key = [
+        "re:^browser\\.search\\.order\\.[1-9]$",
+        "re:^browser\\.contentHandlers\\.types\\.[0-5]\\..*$",
+        "re:^gecko\\.handlerService\\.schemes\\..*$",
+        "re:^gecko\\.handlerService\\.defaultHandlersVersion$"
+    ]
+    action = "ignore"
--- a/devtools/client/locales/l10n.toml
+++ b/devtools/client/locales/l10n.toml
@ -0,0 +1,12 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+basepath = "../../.."
+
+[env]
+    l = "{l10n_base}/{locale}/"
+
+[[paths]]
+    reference = "devtools/client/locales/en-US/**"
+    l10n = "{l}devtools/client/**"
--- a/devtools/shared/locales/l10n.toml
+++ b/devtools/shared/locales/l10n.toml
@ -0,0 +1,12 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+basepath = "../../.."
+
+[env]
+    l = "{l10n_base}/{locale}/"
+
+[[paths]]
+    reference = "devtools/shared/locales/en-US/**"
+    l10n = "{l}devtools/shared/**"
--- a/mobile/android/locales/filter.py
+++ b/mobile/android/locales/filter.py
@ -53,7 +53,7 @@ def test(mod, path, entity = None):
    # we only have exceptions for mobile*
    return "error"
  if mod == "mobile/android":
-    if not entity:
+    if entity is None:
      if (re.match(r"mobile-l10n.js", path) or
          re.match(r"defines.inc", path)):
        return "ignore"
--- a/mobile/android/locales/jar.mn
+++ b/mobile/android/locales/jar.mn
@ -37,7 +37,7 @@
 % resource search-plugins chrome://browser/locale/searchplugins/

 # overrides for toolkit l10n, also for en-US
-# keep this file list in sync with filter.py
+# keep this file list in sync with l10n.toml and filter.py
 relativesrcdir toolkit/locales:
  locale/@AB_CD@/browser/overrides/about.dtd                       (%chrome/global/about.dtd)
  locale/@AB_CD@/browser/overrides/aboutAbout.dtd                  (%chrome/global/aboutAbout.dtd)
--- a/mobile/android/locales/l10n.toml
+++ b/mobile/android/locales/l10n.toml
@ -0,0 +1,246 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+basepath = "../../.."
+
+locales = [
+    "an",
+    "ar",
+    "as",
+    "ast",
+    "az",
+    "be",
+    "bg",
+    "bn-BD",
+    "bn-IN",
+    "br",
+    "ca",
+    "cak",
+    "cs",
+    "cy",
+    "da",
+    "de",
+    "dsb",
+    "el",
+    "en-GB",
+    "en-ZA",
+    "eo",
+    "es-AR",
+    "es-CL",
+    "es-ES",
+    "es-MX",
+    "et",
+    "eu",
+    "fa",
+    "ff",
+    "fi",
+    "fr",
+    "fy-NL",
+    "ga-IE",
+    "gd",
+    "gl",
+    "gn",
+    "gu-IN",
+    "he",
+    "hi-IN",
+    "hr",
+    "hsb",
+    "hu",
+    "hy-AM",
+    "id",
+    "is",
+    "it",
+    "ja",
+    "ka",
+    "kab",
+    "kk",
+    "kn",
+    "ko",
+    "lo",
+    "lt",
+    "lv",
+    "mai",
+    "ml",
+    "mr",
+    "ms",
+    "my",
+    "nb-NO",
+    "ne-NP",
+    "nl",
+    "nn-NO",
+    "or",
+    "pa-IN",
+    "pl",
+    "pt-BR",
+    "pt-PT",
+    "rm",
+    "ro",
+    "ru",
+    "sk",
+    "sl",
+    "son",
+    "sq",
+    "sr",
+    "sv-SE",
+    "ta",
+    "te",
+    "th",
+    "tr",
+    "trs",
+    "uk",
+    "ur",
+    "uz",
+    "wo",
+    "xh",
+    "zam",
+    "zh-CN",
+    "zh-TW",
+    ]
+
+[build]
+exclude-multi-locale = [
+    "be",
+    "bn-BD",
+    "ne-NP",
+    "trs",
+    "wo",
+    "zam",
+]
+
+[env]
+    l = "{l10n_base}/{locale}/"
+
+
+[[paths]]
+    reference = "mobile/locales/en-US/**"
+    l10n = "{l}mobile/**"
+
+[[paths]]
+    reference = "mobile/android/locales/en-US/**"
+    l10n = "{l}mobile/android/**"
+
+[[paths]]
+    reference = "mobile/android/base/locales/en-US/**"
+    l10n = "{l}mobile/android/base/**"
+    test = [
+        "android-dtd",
+    ]
+
+# hand-picked paths from toolkit, keep in sync with jar.mn
+[[paths]]
+    reference = "dom/locales/en-US/chrome/global.dtd"
+    l10n = "{l}dom/chrome/global.dtd"
+
+[[paths]]
+    reference = "dom/locales/en-US/chrome/accessibility/AccessFu.properties"
+    l10n = "{l}dom/chrome/accessibility/AccessFu.properties"
+
+[[paths]]
+    reference = "dom/locales/en-US/chrome/dom/dom.properties"
+    l10n = "{l}dom/chrome/dom/dom.properties"
+
+[[paths]]
+    reference = "dom/locales/en-US/chrome/plugins.properties"
+    l10n = "{l}dom/chrome/plugins.properties"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/about.dtd"
+    l10n = "{l}toolkit/chrome/global/about.dtd"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/aboutAbout.dtd"
+    l10n = "{l}toolkit/chrome/global/aboutAbout.dtd"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/aboutReader.properties"
+    l10n = "{l}toolkit/chrome/global/aboutReader.properties"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/aboutRights.dtd"
+    l10n = "{l}toolkit/chrome/global/aboutRights.dtd"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/charsetMenu.properties"
+    l10n = "{l}toolkit/chrome/global/charsetMenu.properties"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/commonDialogs.properties"
+    l10n = "{l}toolkit/chrome/global/commonDialogs.properties"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/intl.properties"
+    l10n = "{l}toolkit/chrome/global/intl.properties"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/intl.css"
+    l10n = "{l}toolkit/chrome/global/intl.css"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/passwordmgr/passwordmgr.properties"
+    l10n = "{l}toolkit/chrome/passwordmgr/passwordmgr.properties"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/search/search.properties"
+    l10n = "{l}toolkit/chrome/search/search.properties"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/pluginproblem/pluginproblem.dtd"
+    l10n = "{l}toolkit/chrome/pluginproblem/pluginproblem.dtd"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/aboutSupport.dtd"
+    l10n = "{l}toolkit/chrome/global/aboutSupport.dtd"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/aboutSupport.properties"
+    l10n = "{l}toolkit/chrome/global/aboutSupport.properties"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/crashreporter/crashes.dtd"
+    l10n = "{l}toolkit/crashreporter/crashes.dtd"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/crashreporter/crashes.properties"
+    l10n = "{l}toolkit/crashreporter/crashes.properties"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/mozilla.dtd"
+    l10n = "{l}toolkit/chrome/global/mozilla.dtd"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/aboutTelemetry.dtd"
+    l10n = "{l}toolkit/chrome/global/aboutTelemetry.dtd"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/aboutTelemetry.properties"
+    l10n = "{l}toolkit/chrome/global/aboutTelemetry.properties"
+
+[[paths]]
+    reference = "toolkit/locales/en-US/chrome/global/aboutWebrtc.properties"
+    l10n = "{l}toolkit/chrome/global/aboutWebrtc.properties"
+
+
+[[filters]]
+    path = [
+        "{l}mobile/android/mobile-l10n.js",
+        "{l}mobile/android/defines.inc",
+    ]
+    action = "ignore"
+
+[[filters]]
+    path = "{l}mobile/android/defines.inc"
+    key = "MOZ_LANGPACK_CONTRIBUTORS"
+    action = "ignore"
+
+[[filters]]
+    path = "{l}mobile/chrome/region.properties"
+    key = [
+        "re:^browser\\.search\\.order\\.[1-9]$",
+        "re:^browser\\.search\\.[a-zA-Z]+\\.US",
+        "re:^browser\\.contentHandlers\\.types\\.[0-5]\\..*$",
+        "re:^gecko\\.handlerService\\.schemes\\..+$",
+        "re:^gecko\\.handlerService\\.defaultHandlersVersion$",
+        "re:^browser\\.suggestedsites\\..+$",
+    ]
+    action = "ignore"
--- a/moz.build
+++ b/moz.build
@ -40,6 +40,10 @@ with Files('GNUmakefile'):
 with Files('*gradle*'):
    BUG_COMPONENT = ('Firefox for Android', 'Build Config & IDE Support')

+with Files('**/l10n.toml'):
+    BUG_COMPONENT = ('Core', 'Localization')
+    FINAL = True
+
 with Files('README.txt'):
    BUG_COMPONENT = ('Core', 'General')

--- a/toolkit/locales/l10n.toml
+++ b/toolkit/locales/l10n.toml
@ -0,0 +1,29 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+basepath = "../.."
+
+
+[env]
+    l = "{l10n_base}/{locale}/"
+
+
+[[paths]]
+    reference = "toolkit/locales/en-US/**"
+    l10n = "{l}toolkit/**"
+
+[[paths]]
+    reference = "dom/locales/en-US/**"
+    l10n = "{l}dom/**"
+
+[[paths]]
+    reference = "netwerk/locales/en-US/**"
+    l10n = "{l}netwerk/**"
+
+[[paths]]
+    reference = "security/manager/locales/en-US/**"
+    l10n = "{l}security/manager/**"
+
+[[includes]]
+    path = "devtools/shared/locales/l10n.toml"
--- a/tools/compare-locales/docs/index.rst
+++ b/tools/compare-locales/docs/index.rst
@ -20,7 +20,24 @@ Exposing strings
 Localizers only handle a few file formats in well-known locations in the
 source tree.

-The locations are in directories like
+The locations are specified by TOML files. They're part of the bigger
+localization ecosystem at Mozilla, and `the documentation about the
+file format <http://moz-l10n-config.readthedocs.io/en/latest/fileformat.html>`_
+explains how to set them up, and what the entries mean. In short, you find
+
+.. code-block:: toml
+
+    [[paths]]
+        reference = browser/locales/en-US/**
+        l10n = {l}browser/**
+
+to add a directory for all localizations. Changes to these files are best
+submitted for review by :Pike or :flod.
+
+These configuration files are the future, and right now, we still have
+support for the previous way to configuring l10n, which is described below.
+
+The locations are commonly in directories like

    :file:`browser/`\ ``locales/en-US/``\ :file:`subdir/file.ext`

@ -88,7 +105,7 @@ Properties
 ini
    Used by the crashreporter and updater, avoid if possible.
 foo.defines
-    Used during builds, for example to create file:`install.rdf` for
+    Used during builds, for example to create :file:`install.rdf` for
    language packs.

 Adding new formats involves changing various different tools, and is strongly