Bug 1295157 - create devtools l10n migration script for dtd to properties migration;r=Pike

For devtools-html, devtools will progressively stop using DTD files and will use only properties
files. To avoid retranslating already localized strings in every locale, this script can be used
by localization teams to automatically migrate strings moved in the scope of the devtools-html
project.

MozReview-Commit-ID: KNDfCoPXOM9

--HG--
extra : rebase_source : 48fbf2ae3daab89891262e460c979b6c141a2caa
This commit is contained in:
Julian Descottes 2016-09-08 23:45:42 +02:00
Родитель 4a7f6008a3
Коммит 336f71fd5e
6 изменённых файлов: 397 добавлений и 0 удалений

Просмотреть файл

@ -0,0 +1,16 @@
devtools-l10n-migration script
==============================
For devtools.html, devtools will no longer rely on DTD files. This migration
script is aimed at localizers to automate the migration of strings from DTD to
properties files.
How to run this script?
To migrate all configuration files:
python migrate/main.py path/to/your/l10n/repo/ -c migrate/conf/
To migrate only one configuration file:
python migrate/main.py path/to/your/l10n/repo/ -c migrate/conf/bug1294186
All configuration files should be named after the bug where specific devtools strings were migrated.

Просмотреть файл

Просмотреть файл

@ -0,0 +1,22 @@
font-inspector.properties:fontinspector.seeAll.tooltip = font-inspector.dtd:showAllFonts
font-inspector.properties:fontinspector.seeAll = font-inspector.dtd:showAllFontsUsed
font-inspector.properties:fontinspector.usedAs = font-inspector.dtd:usedAs
font-inspector.properties:fontinspector.system = font-inspector.dtd:system
font-inspector.properties:fontinspector.remote = font-inspector.dtd:remote
font-inspector.properties:fontinspector.previewText = font-inspector.dtd:previewHint
inspector.properties:inspector.eyedropper.label = inspector.dtd:inspectorEyeDropper.label
inspector.properties:inspector.breadcrumbs.label = inspector.dtd:inspectorBreadcrumbsGroup
boxmodel.properties:boxmodel.title = layoutview.dtd:layoutViewTitle
boxmodel.properties:boxmodel.margin = layoutview.dtd:margin.tooltip
boxmodel.properties:boxmodel.padding = layoutview.dtd:padding.tooltip
boxmodel.properties:boxmodel.border = layoutview.dtd:border.tooltip
boxmodel.properties:boxmodel.content = layoutview.dtd:content.tooltip
boxmodel.properties:boxmodel.geometryButton.tooltip = layoutview.dtd:geometry.button.tooltip
inspector.properties:inspector.browserStyles.label = styleinspector.dtd:browserStylesLabel
inspector.properties:inspector.filterStyles.placeholder = styleinspector.dtd:filterStylesPlaceholder
inspector.properties:inspector.addRule.tooltip = styleinspector.dtd:addRuleButtonTooltip
inspector.properties:inspector.togglePseudo.tooltip = styleinspector.dtd:togglePseudoClassPanel
inspector.properties:inspector.noProperties = styleinspector.dtd:noPropertiesFound

Просмотреть файл

@ -0,0 +1,98 @@
netmonitor.properties:netmonitor.perfNotice1 = netmonitor.dtd:netmonitorUI.perfNotice1
netmonitor.properties:netmonitor.perfNotice2 = netmonitor.dtd:netmonitorUI.perfNotice2
netmonitor.properties:netmonitor.perfNotice3 = netmonitor.dtd:netmonitorUI.perfNotice3
netmonitor.properties:netmonitor.reloadNotice1 = netmonitor.dtd:netmonitorUI.reloadNotice1
netmonitor.properties:netmonitor.reloadNotice2 = netmonitor.dtd:netmonitorUI.reloadNotice2
netmonitor.properties:netmonitor.reloadNotice3 = netmonitor.dtd:netmonitorUI.reloadNotice3
netmonitor.properties:netmonitor.toolbar.status3 = netmonitor.dtd:netmonitorUI.toolbar.status3
netmonitor.properties:netmonitor.toolbar.method = netmonitor.dtd:netmonitorUI.toolbar.method
netmonitor.properties:netmonitor.toolbar.file = netmonitor.dtd:netmonitorUI.toolbar.file
netmonitor.properties:netmonitor.toolbar.domain = netmonitor.dtd:netmonitorUI.toolbar.domain
netmonitor.properties:netmonitor.toolbar.cause = netmonitor.dtd:netmonitorUI.toolbar.cause
netmonitor.properties:netmonitor.toolbar.type = netmonitor.dtd:netmonitorUI.toolbar.type
netmonitor.properties:netmonitor.toolbar.transferred = netmonitor.dtd:netmonitorUI.toolbar.transferred
netmonitor.properties:netmonitor.toolbar.size = netmonitor.dtd:netmonitorUI.toolbar.size
netmonitor.properties:netmonitor.toolbar.waterfall = netmonitor.dtd:netmonitorUI.toolbar.waterfall
netmonitor.properties:netmonitor.tab.headers = netmonitor.dtd:netmonitorUI.tab.headers
netmonitor.properties:netmonitor.tab.cookies = netmonitor.dtd:netmonitorUI.tab.cookies
netmonitor.properties:netmonitor.tab.params = netmonitor.dtd:netmonitorUI.tab.params
netmonitor.properties:netmonitor.tab.response = netmonitor.dtd:netmonitorUI.tab.response
netmonitor.properties:netmonitor.tab.timings = netmonitor.dtd:netmonitorUI.tab.timings
netmonitor.properties:netmonitor.tab.preview = netmonitor.dtd:netmonitorUI.tab.preview
netmonitor.properties:netmonitor.tab.security = netmonitor.dtd:netmonitorUI.tab.security
netmonitor.properties:netmonitor.footer.filterAll = netmonitor.dtd:netmonitorUI.footer.filterAll
netmonitor.properties:netmonitor.footer.filterHTML = netmonitor.dtd:netmonitorUI.footer.filterHTML
netmonitor.properties:netmonitor.footer.filterCSS = netmonitor.dtd:netmonitorUI.footer.filterCSS
netmonitor.properties:netmonitor.footer.filterJS = netmonitor.dtd:netmonitorUI.footer.filterJS
netmonitor.properties:netmonitor.footer.filterXHR = netmonitor.dtd:netmonitorUI.footer.filterXHR
netmonitor.properties:netmonitor.footer.filterFonts = netmonitor.dtd:netmonitorUI.footer.filterFonts
netmonitor.properties:netmonitor.footer.filterImages = netmonitor.dtd:netmonitorUI.footer.filterImages
netmonitor.properties:netmonitor.footer.filterMedia = netmonitor.dtd:netmonitorUI.footer.filterMedia
netmonitor.properties:netmonitor.footer.filterFlash = netmonitor.dtd:netmonitorUI.footer.filterFlash
netmonitor.properties:netmonitor.footer.filterWS = netmonitor.dtd:netmonitorUI.footer.filterWS
netmonitor.properties:netmonitor.footer.filterOther = netmonitor.dtd:netmonitorUI.footer.filterOther
netmonitor.properties:netmonitor.footer.filterFreetext.label = netmonitor.dtd:netmonitorUI.footer.filterFreetext.label
netmonitor.properties:netmonitor.footer.filterFreetext.key = netmonitor.dtd:netmonitorUI.footer.filterFreetext.key
netmonitor.properties:netmonitor.footer.clear = netmonitor.dtd:netmonitorUI.footer.clear
netmonitor.properties:netmonitor.footer.perf = netmonitor.dtd:netmonitorUI.footer.perf
netmonitor.properties:netmonitor.panesButton.tooltip = netmonitor.dtd:netmonitorUI.panesButton.tooltip
netmonitor.properties:netmonitor.summary.url = netmonitor.dtd:netmonitorUI.summary.url
netmonitor.properties:netmonitor.summary.method = netmonitor.dtd:netmonitorUI.summary.method
netmonitor.properties:netmonitor.summary.address = netmonitor.dtd:netmonitorUI.summary.address
netmonitor.properties:netmonitor.summary.status = netmonitor.dtd:netmonitorUI.summary.status
netmonitor.properties:netmonitor.summary.version = netmonitor.dtd:netmonitorUI.summary.version
netmonitor.properties:netmonitor.summary.editAndResend = netmonitor.dtd:netmonitorUI.summary.editAndResend
netmonitor.properties:netmonitor.summary.rawHeaders = netmonitor.dtd:netmonitorUI.summary.rawHeaders
netmonitor.properties:netmonitor.summary.rawHeaders.requestHeaders = netmonitor.dtd:netmonitorUI.summary.rawHeaders.requestHeaders
netmonitor.properties:netmonitor.summary.rawHeaders.responseHeaders = netmonitor.dtd:netmonitorUI.summary.rawHeaders.responseHeaders
netmonitor.properties:netmonitor.summary.size = netmonitor.dtd:netmonitorUI.summary.size
netmonitor.properties:netmonitor.response.name = netmonitor.dtd:netmonitorUI.response.name
netmonitor.properties:netmonitor.response.dimensions = netmonitor.dtd:netmonitorUI.response.dimensions
netmonitor.properties:netmonitor.response.mime = netmonitor.dtd:netmonitorUI.response.mime
netmonitor.properties:netmonitor.timings.blocked = netmonitor.dtd:netmonitorUI.timings.blocked
netmonitor.properties:netmonitor.timings.dns = netmonitor.dtd:netmonitorUI.timings.dns
netmonitor.properties:netmonitor.timings.connect = netmonitor.dtd:netmonitorUI.timings.connect
netmonitor.properties:netmonitor.timings.send = netmonitor.dtd:netmonitorUI.timings.send
netmonitor.properties:netmonitor.timings.wait = netmonitor.dtd:netmonitorUI.timings.wait
netmonitor.properties:netmonitor.timings.receive = netmonitor.dtd:netmonitorUI.timings.receive
netmonitor.properties:netmonitor.security.warning.cipher = netmonitor.dtd:netmonitorUI.security.warning.cipher
netmonitor.properties:netmonitor.security.error = netmonitor.dtd:netmonitorUI.security.error
netmonitor.properties:netmonitor.security.protocolVersion = netmonitor.dtd:netmonitorUI.security.protocolVersion
netmonitor.properties:netmonitor.security.cipherSuite = netmonitor.dtd:netmonitorUI.security.cipherSuite
netmonitor.properties:netmonitor.security.hsts = netmonitor.dtd:netmonitorUI.security.hsts
netmonitor.properties:netmonitor.security.hpkp = netmonitor.dtd:netmonitorUI.security.hpkp
netmonitor.properties:netmonitor.security.connection = netmonitor.dtd:netmonitorUI.security.connection
netmonitor.properties:netmonitor.security.certificate = netmonitor.dtd:netmonitorUI.security.certificate
netmonitor.properties:netmonitor.context.copyUrl = netmonitor.dtd:netmonitorUI.context.copyUrl
netmonitor.properties:netmonitor.context.copyUrl.accesskey = netmonitor.dtd:netmonitorUI.context.copyUrl.accesskey
netmonitor.properties:netmonitor.context.copyUrlParams = netmonitor.dtd:netmonitorUI.context.copyUrlParams
netmonitor.properties:netmonitor.context.copyUrlParams.accesskey = netmonitor.dtd:netmonitorUI.context.copyUrlParams.accesskey
netmonitor.properties:netmonitor.context.copyPostData = netmonitor.dtd:netmonitorUI.context.copyPostData
netmonitor.properties:netmonitor.context.copyPostData.accesskey = netmonitor.dtd:netmonitorUI.context.copyPostData.accesskey
netmonitor.properties:netmonitor.context.copyAsCurl = netmonitor.dtd:netmonitorUI.context.copyAsCurl
netmonitor.properties:netmonitor.context.copyAsCurl.accesskey = netmonitor.dtd:netmonitorUI.context.copyAsCurl.accesskey
netmonitor.properties:netmonitor.context.copyRequestHeaders = netmonitor.dtd:netmonitorUI.context.copyRequestHeaders
netmonitor.properties:netmonitor.context.copyRequestHeaders.accesskey = netmonitor.dtd:netmonitorUI.context.copyRequestHeaders.accesskey
netmonitor.properties:netmonitor.context.copyResponseHeaders = netmonitor.dtd:netmonitorUI.context.copyResponseHeaders
netmonitor.properties:netmonitor.context.copyResponseHeaders.accesskey = netmonitor.dtd:netmonitorUI.context.copyResponseHeaders.accesskey
netmonitor.properties:netmonitor.context.copyResponse = netmonitor.dtd:netmonitorUI.context.copyResponse
netmonitor.properties:netmonitor.context.copyResponse.accesskey = netmonitor.dtd:netmonitorUI.context.copyResponse.accesskey
netmonitor.properties:netmonitor.context.copyImageAsDataUri = netmonitor.dtd:netmonitorUI.context.copyImageAsDataUri
netmonitor.properties:netmonitor.context.copyImageAsDataUri.accesskey = netmonitor.dtd:netmonitorUI.context.copyImageAsDataUri.accesskey
netmonitor.properties:netmonitor.context.copyAllAsHar = netmonitor.dtd:netmonitorUI.context.copyAllAsHar
netmonitor.properties:netmonitor.context.copyAllAsHar.accesskey = netmonitor.dtd:netmonitorUI.context.copyAllAsHar.accesskey
netmonitor.properties:netmonitor.context.saveAllAsHar = netmonitor.dtd:netmonitorUI.context.saveAllAsHar
netmonitor.properties:netmonitor.context.saveAllAsHar.accesskey = netmonitor.dtd:netmonitorUI.context.saveAllAsHar.accesskey
netmonitor.properties:netmonitor.context.editAndResend = netmonitor.dtd:netmonitorUI.summary.editAndResend
netmonitor.properties:netmonitor.context.editAndResend.accesskey = netmonitor.dtd:netmonitorUI.summary.editAndResend.accesskey
netmonitor.properties:netmonitor.context.newTab = netmonitor.dtd:netmonitorUI.context.newTab
netmonitor.properties:netmonitor.context.newTab.accesskey = netmonitor.dtd:netmonitorUI.context.newTab.accesskey
netmonitor.properties:netmonitor.context.perfTools = netmonitor.dtd:netmonitorUI.context.perfTools
netmonitor.properties:netmonitor.context.perfTools.accesskey = netmonitor.dtd:netmonitorUI.context.perfTools.accesskey
netmonitor.properties:netmonitor.custom.newRequest = netmonitor.dtd:netmonitorUI.custom.newRequest
netmonitor.properties:netmonitor.custom.query = netmonitor.dtd:netmonitorUI.custom.query
netmonitor.properties:netmonitor.custom.headers = netmonitor.dtd:netmonitorUI.custom.headers
netmonitor.properties:netmonitor.custom.postData = netmonitor.dtd:netmonitorUI.custom.postData
netmonitor.properties:netmonitor.custom.send = netmonitor.dtd:netmonitorUI.custom.send
netmonitor.properties:netmonitor.custom.cancel = netmonitor.dtd:netmonitorUI.custom.cancel
netmonitor.properties:netmonitor.backButton = netmonitor.dtd:netmonitorUI.backButton

Просмотреть файл

@ -0,0 +1,261 @@
import argparse
import glob
import HTMLParser
import logging
import os
import re
import sys
import urllib2
# Import compare-locales parser from parent folder.
script_path = os.path.dirname(os.path.realpath(__file__))
compare_locales_path = os.path.join(script_path, '../../../compare-locales')
sys.path.insert(0, compare_locales_path)
from compare_locales import parser
# Configure logging format and level
logging.basicConfig(format=' [%(levelname)s] %(message)s', level=logging.INFO)
# License header to use when creating new properties files.
DEFAULT_HEADER = ('# This Source Code Form is subject to the terms of the '
'Mozilla Public\n# License, v. 2.0. If a copy of the MPL '
'was not distributed with this\n# file, You can obtain '
'one at http://mozilla.org/MPL/2.0/.\n')
# Base url to retrieve properties files on central, that will be parsed for
# localization notes.
CENTRAL_BASE_URL = ('https://hg.mozilla.org/'
'mozilla-central/raw-file/tip/'
'devtools/client/locales/en-US/')
# HTML parser to translate HTML entities in dtd files.
HTML_PARSER = HTMLParser.HTMLParser()
# Cache to store properties files retrieved over the network.
central_prop_cache = {}
# Cache the parsed entities from the existing DTD files.
dtd_entities_cache = {}
# Retrieve the content of the current version of a properties file for the
# provided filename, from devtools/client on mozilla central. Will return an
# empty array if the file can't be retrieved or read.
def get_central_prop_content(prop_filename):
if prop_filename in central_prop_cache:
return central_prop_cache[prop_filename]
url = CENTRAL_BASE_URL + prop_filename
logging.info('loading localization file from central: {%s}' % url)
try:
central_prop_cache[prop_filename] = urllib2.urlopen(url).readlines()
except:
logging.error('failed to load properties file from central: {%s}'
% url)
central_prop_cache[prop_filename] = []
return central_prop_cache[prop_filename]
# Retrieve the current en-US localization notes for the provided prop_name.
def get_localization_note(prop_name, prop_filename):
prop_content = get_central_prop_content(prop_filename)
comment_buffer = []
for i, line in enumerate(prop_content):
# Remove line breaks.
line = line.strip('\n').strip('\r')
if line.startswith('#'):
# Comment line, add to the current comment buffer.
comment_buffer.append(line)
elif re.search('(^|\n)' + re.escape(prop_name) + '\s*=', line):
# Property found, the current comment buffer is the localization
# note.
break;
else:
# No match, not a comment, reinitialize the comment buffer.
comment_buffer = []
return '\n'.join(comment_buffer)
# Retrieve the parsed DTD entities for a provided path. Results are cached by
# dtd path.
def get_dtd_entities(dtd_path):
if dtd_path in dtd_entities_cache:
return dtd_entities_cache[dtd_path]
dtd_parser = parser.getParser('.dtd')
dtd_parser.readFile(dtd_path)
dtd_entities_cache[dtd_path] = dtd_parser.parse()
return dtd_entities_cache[dtd_path]
# Extract the value of an entity in a dtd file.
def get_translation_from_dtd(dtd_path, entity_name):
entities, map = get_dtd_entities(dtd_path)
if entity_name not in map:
# Bail out if translation is missing.
return
key = map[entity_name]
entity = entities[key]
translation = HTML_PARSER.unescape(entity.val)
return translation.encode('utf-8')
# Extract the header and file wide comments for the provided properties file
# filename.
def get_properties_header(prop_filename):
prop_content = get_central_prop_content(prop_filename)
# if the file content is empty, return the default license header.
if len(prop_content) == 0:
return DEFAULT_HEADER
header_buffer = []
for i, line in enumerate(prop_content):
# remove line breaks.
line = line.strip('\n').strip('\r')
# regexp matching keys extracted form parser.py.
is_entity_line = re.search('^(\s*)'
'((?:[#!].*?\n\s*)*)'
'([^#!\s\n][^=:\n]*?)\s*[:=][ \t]*', line)
is_loc_note = re.search('^(\s*)'
'\#\s*LOCALIZATION NOTE\s*\([^)]+\)', line)
if is_entity_line or is_loc_note:
# header finished, break the loop.
break
else:
# header line, add to the current buffer.
header_buffer.append(line)
# concatenate the current buffer and return.
return '\n'.join(header_buffer)
# Create a new properties file at the provided path.
def create_properties_file(prop_path):
logging.info('creating new *.properties file: {%s}' % prop_path)
prop_filename = os.path.basename(prop_path)
header = get_properties_header(prop_filename)
prop_file = open(prop_path, 'w+')
prop_file.write(header)
prop_file.close()
# Migrate a single string entry for a dtd to a properties file.
def migrate_string(dtd_path, prop_path, dtd_name, prop_name):
if not os.path.isfile(dtd_path):
logging.error('dtd file can not be found at: {%s}' % dtd_path)
return
translation = get_translation_from_dtd(dtd_path, dtd_name)
if not translation:
logging.error('translation could not be found for: {%s} in {%s}'
% (dtd_name, dtd_path))
return
# Create properties file if missing.
if not os.path.isfile(prop_path):
create_properties_file(prop_path)
if not os.path.isfile(prop_path):
logging.error('could not create new properties file at: {%s}'
% prop_path)
return
prop_line = prop_name + '=' + translation + '\n'
# Skip the string if it already exists in the destination file.
prop_file_content = open(prop_path, 'r').read()
if prop_line in prop_file_content:
logging.warning('string already migrated, skipping: {%s}' % prop_name)
return
# Skip the string and log an error if an existing entry is found, but with
# a different value.
if re.search('(^|\n)' + re.escape(prop_name) + '\s*=', prop_file_content):
logging.error('existing string found, skipping: {%s}' % prop_name)
return
prop_filename = os.path.basename(prop_path)
logging.info('migrating {%s} in {%s}' % (prop_name, prop_filename))
with open(prop_path, 'a') as prop_file:
localization_note = get_localization_note(prop_name, prop_filename)
if len(localization_note):
prop_file.write('\n' + localization_note)
else:
logging.warning('localization notes could not be found for: {%s}'
% prop_name)
prop_file.write('\n' + prop_line)
# Apply the migration instructions in the provided configuration file.
def migrate_conf(conf_path, l10n_path):
f = open(conf_path, 'r')
lines = f.readlines()
f.close()
for i, line in enumerate(lines):
# Remove line breaks.
line = line.strip('\n').strip('\r')
# Skip invalid lines.
if ' = ' not in line:
continue
# Expected syntax: ${prop_path}:${prop_name} = ${dtd_path}:${dtd_name}.
prop_info, dtd_info = line.split(' = ')
prop_path, prop_name = prop_info.split(':')
dtd_path, dtd_name = dtd_info.split(':')
dtd_path = os.path.join(l10n_path, dtd_path)
prop_path = os.path.join(l10n_path, prop_path)
migrate_string(dtd_path, prop_path, dtd_name, prop_name)
def main():
# Read command line arguments.
arg_parser = argparse.ArgumentParser(
description='Migrate devtools localized strings.')
arg_parser.add_argument('path', type=str, help='path to l10n repository')
arg_parser.add_argument('-c', '--config', type=str,
help='path to configuration file or folder')
args = arg_parser.parse_args()
# Retrieve path to devtools localization files in l10n repository.
devtools_l10n_path = os.path.join(args.path, 'devtools/client/')
if not os.path.exists(devtools_l10n_path):
logging.error('l10n path is invalid: {%s}' % devtools_l10n_path)
exit()
logging.info('l10n path is valid: {%s}' % devtools_l10n_path)
# Retrieve configuration files to apply.
if os.path.isdir(args.config):
conf_files = glob.glob(args.config + '*')
elif os.path.isfile(args.config):
conf_files = [args.config]
else:
logging.error('config path is invalid: {%s}' % args.config)
exit()
# Perform migration for each configuration file.
for conf_file in conf_files:
logging.info('performing migration for config file: {%s}' % conf_file)
migrate_conf(conf_file, devtools_l10n_path)
if __name__ == '__main__':
main()

Просмотреть файл