mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Bug 848284, support for running l10n on try, r=catlee DONTBUILD 

--HG--
rename : testing/mozharness/configs/single_locale/mozilla-central.py => testing/mozharness/configs/single_locale/try.py
This commit is contained in:
Nick Thomas 2016-04-28 10:53:07 +12:00
Родитель 80791b18a4
Коммит de41c2626d
9 изменённых файлов: 195 добавлений и 43 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
testing/mozharness/configs/single_locale/linux.py
Просмотреть файл

@ -14,6 +14,8 @@ config = {
"LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
"L10NBASEDIR": "../../l10n",
"MOZ_MAKE_COMPLETE_MAR": "1",
'TOOLTOOL_CACHE': '/builds/tooltool_cache',
'TOOLTOOL_HOME': '/builds',
},
"ssh_key_dir": "/home/mock_mozilla/.ssh",
"log_name": "single_locale",

2
testing/mozharness/configs/single_locale/linux64.py
Просмотреть файл

@ -14,6 +14,8 @@ config = {
"LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
"L10NBASEDIR": "../../l10n",
"MOZ_MAKE_COMPLETE_MAR": "1",
'TOOLTOOL_CACHE': '/builds/tooltool_cache',
'TOOLTOOL_HOME': '/builds',
},
"ssh_key_dir": "/home/mock_mozilla/.ssh",
"log_name": "single_locale",

3
testing/mozharness/configs/single_locale/macosx64.py
Просмотреть файл

@ -18,6 +18,8 @@ config = {
"L10NBASEDIR": "../../l10n",
"MOZ_MAKE_COMPLETE_MAR": "1",
"LOCALE_MERGEDIR": "%(abs_merge_dir)s/",
'TOOLTOOL_CACHE': '/builds/tooltool_cache',
'TOOLTOOL_HOME': '/builds',
},
"ssh_key_dir": "~/.ssh",
"log_name": "single_locale",
@ -34,7 +36,6 @@ config = {
'tooltool_script': ["/builds/tooltool.py"],
'tooltool_bootstrap': "setup.sh",
'tooltool_manifest_src': 'browser/config/tooltool-manifests/macosx64/releng.manifest',
# balrog credential file:
'balrog_credentials_file': 'oauth.txt',

1
testing/mozharness/configs/single_locale/production.py
Просмотреть файл

@ -1,4 +1,5 @@
config = {
"upload_environment": "prod",
"upload_env": {
"UPLOAD_USER": "ffxbld",
# ssh_key_dir is defined per platform: it is "~/.ssh" for every platform

3
testing/mozharness/configs/single_locale/staging.py
Просмотреть файл

@ -1,10 +1,11 @@
config = {
"upload_environment": "stage",
"upload_env": {
"UPLOAD_USER": "ffxbld",
# ssh_key_dir is defined per platform: it is "~/.ssh" for every platform
# except when mock is in use, in this case, ssh_key_dir is
# /home/mock_mozilla/.ssh
"UPLOAD_SSH_KEY": "%(ssh_key_dir)s/stage-ffxbld_rsa",
"UPLOAD_SSH_KEY": "%(ssh_key_dir)s/ffxbld_rsa",
"UPLOAD_HOST": "upload.ffxbld.productdelivery.stage.mozaws.net",
"POST_UPLOAD_CMD": "post_upload.py -b %(branch)s-l10n -p %(stage_product)s -i %(buildid)s --release-to-latest --release-to-dated %(post_upload_extra)s",
"UPLOAD_TO_TEMP": "1"

47
testing/mozharness/configs/single_locale/try.py Normal file
Просмотреть файл

@ -0,0 +1,47 @@
config = {
"nightly_build": False,
"branch": "try",
"en_us_binary_url": "http://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central",
"update_channel": "nightly",
"latest_mar_dir": '/pub/firefox/nightly/latest-mozilla-central-l10n',
"update_gecko_source_to_enUS": False,
# l10n
"hg_l10n_base": "https://hg.mozilla.org/l10n-central",
# mar
"mar_tools_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-central/mar-tools/%(platform)s",
# repositories
"mozilla_dir": "try",
"hgtool_base_bundle_urls": ["https://archive.mozilla.org/pub/firefox/bundles"],
"repos": [{
"vcs": "hg",
"repo": "https://hg.mozilla.org/build/tools",
"revision": "default",
"dest": "tools",
}, {
"vcs": "hgtool",
"repo": "https://hg.mozilla.org/try",
"revision": "%(revision)s",
"dest": "try",
"clone_by_revision": True,
"clone_with_purge": True,
}, {
"vcs": "hgtool",
"repo": "https://hg.mozilla.org/build/compare-locales",
"revision": "RELEASE_AUTOMATION"
}],
# purge options
'is_automation': True,
"upload_env": {
"UPLOAD_USER": "trybld",
# ssh_key_dir is defined per platform: it is "~/.ssh" for every platform
# except when mock is in use, in this case, ssh_key_dir is
# /home/mock_mozilla/.ssh
"UPLOAD_SSH_KEY": "%(ssh_key_dir)s/trybld_dsa",
"UPLOAD_HOST": "upload.trybld.productdelivery.%(upload_environment)s.mozaws.net",
"POST_UPLOAD_CMD": "post_upload.py --who %(who)s --builddir %(branch)s-%(platform)s --tinderbox-builds-dir %(who)s-%(revision)s -p %(stage_product)s -i %(buildid)s --revision %(revision)s --release-to-try-builds %(post_upload_extra)s",
"UPLOAD_TO_TEMP": "1"
},
}

4
testing/mozharness/configs/single_locale/win32.py
Просмотреть файл

@ -14,6 +14,10 @@ config = {
"DIST": "%(abs_objdir)s",
"L10NBASEDIR": "../../l10n",
"MOZ_MAKE_COMPLETE_MAR": "1",
"PATH": 'C:\\mozilla-build\\nsis-3.0b1;'
'%s' % (os.environ.get('path')),
'TOOLTOOL_CACHE': '/c/builds/tooltool_cache',
'TOOLTOOL_HOME': '/c/builds',
},
"ssh_key_dir": "~/.ssh",
"log_name": "single_locale",

4
testing/mozharness/configs/single_locale/win64.py
Просмотреть файл

@ -14,6 +14,10 @@ config = {
"LOCALE_MERGEDIR": "%(abs_merge_dir)s",
"L10NBASEDIR": "../../l10n",
"MOZ_MAKE_COMPLETE_MAR": "1",
"PATH": 'C:\\mozilla-build\\nsis-3.0b1;'
'%s' % (os.environ.get('path')),
'TOOLTOOL_CACHE': '/c/builds/tooltool_cache',
'TOOLTOOL_HOME': '/c/builds',
},
"ssh_key_dir": "~/.ssh",
"log_name": "single_locale",

172
testing/mozharness/scripts/desktop_l10n.py
Просмотреть файл

@ -65,14 +65,15 @@ configuration_tokens = ('branch',
'update_channel',
'ssh_key_dir',
'stage_product',
)
'upload_environment',
)
# some other values such as "%(version)s", "%(buildid)s", ...
# are defined at run time and they cannot be enforced in the _pre_config_lock
# phase
runtime_config_tokens = ('buildid', 'version', 'locale', 'from_buildid',
'abs_objdir', 'abs_merge_dir', 'version',
'to_buildid', 'en_us_binary_url', 'mar_tools_url')
'abs_objdir', 'abs_merge_dir', 'revision',
'to_buildid', 'en_us_binary_url', 'mar_tools_url',
'post_upload_extra', 'who')
# DesktopSingleLocale {{{1
class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
@ -122,6 +123,13 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
"dest": "tag_override",
"type": "string",
"help": "Override the tags set for all repos"}
], [
['--revision', ],
{"action": "store",
"dest": "revision",
"type": "string",
"help": "Override the gecko revision to use (otherwise use buildbot supplied"
" value, or en-US revision) "}
], [
['--user-repo-override', ],
{"action": "store",
@ -209,6 +217,7 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
self.bootstrap_env = None
self.upload_env = None
self.revision = None
self.enUS_revision = None
self.version = None
self.upload_urls = {}
self.locales_property = {}
@ -334,8 +343,9 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
for element in config_option:
new_list.append(self.__detokenise_element(element, token, value))
return tuple(new_list)
# everything else, bool, number, ...
return None
else:
# everything else, bool, number, ...
return config_option
# Helper methods {{{2
def query_bootstrap_env(self):
@ -377,16 +387,48 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
self.bootstrap_env = bootstrap_env
return self.bootstrap_env
def _query_who(self):
""" looks for who triggered the build with a change.
This is used for things like try builds where the upload dir is
associated with who pushed to try. First it will look in self.config
and failing that, will poll buildbot_config
If nothing is found, it will default to returning "nobody@example.com"
"""
if self.config.get('who'):
return self.config['who']
self.read_buildbot_config()
try:
return self.buildbot_config['sourcestamp']['changes'][0]['who']
except (KeyError, IndexError):
# KeyError: "sourcestamp" or "changes" or "who" not in buildbot_config
# IndexError: buildbot_config['sourcestamp']['changes'] is empty
pass
try:
return str(self.buildbot_config['properties']['who'])
except KeyError:
pass
return "nobody@example.com"
def _query_upload_env(self):
"""returns the environment used for the upload step"""
if self.upload_env:
return self.upload_env
config = self.config
buildid = self._query_buildid()
version = self.query_version()
replace_dict = {
'buildid': self._query_buildid(),
'version': self.query_version(),
'post_upload_extra': ' '.join(config.get('post_upload_extra', [])),
'upload_environment': config['upload_environment'],
}
if config['branch'] == 'try':
replace_dict.update({
'who': self._query_who(),
'revision': self._query_revision(),
})
upload_env = self.query_env(partial_env=config.get("upload_env"),
replace_dict={'buildid': buildid,
'version': version})
replace_dict=replace_dict)
upload_env['LATEST_MAR_DIR'] = config['latest_mar_dir']
# check if there are any extra option from the platform configuration
# and append them to the env
@ -394,6 +436,7 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
if 'upload_env_extra' in config:
for extra in config['upload_env_extra']:
upload_env[extra] = config['upload_env_extra'][extra]
self.upload_env = upload_env
return self.upload_env
@ -435,18 +478,51 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
return self.buildid
def _query_revision(self):
""" Get the gecko revision in this order of precedence
* cached value
* command line arg --revision (development)
* buildbot properties (try with buildbot forced build)
* buildbot change (try with buildbot scheduler)
* from the en-US build (m-c & m-a)
This will fail the last case if the build hasn't been pulled yet.
"""
if self.revision:
return self.revision
self.read_buildbot_config()
config = self.config
revision = None
if config.get("revision"):
revision = config["revision"]
elif 'revision' in self.buildbot_properties:
revision = self.buildbot_properties['revision']
elif (self.buildbot_config and
self.buildbot_config.get('sourcestamp', {}).get('revision')):
revision = self.buildbot_config['sourcestamp']['revision']
elif self.buildbot_config and self.buildbot_config.get('revision'):
revision = self.buildbot_config['revision']
elif config.get("update_gecko_source_to_enUS", True):
revision = self._query_enUS_revision()
if not revision:
self.fatal("Can't determine revision!")
self.revision = str(revision)
return self.revision
def _query_enUS_revision(self):
"""Get revision from the objdir.
Only valid after setup is run.
"""
if self.revision:
return self.revision
if self.enUS_revision:
return self.enUS_revision
r = re.compile(r"^(gecko|fx)_revision ([0-9a-f]+\+?)$")
output = self._query_make_ident_output()
for line in output.splitlines():
match = r.match(line)
if match:
self.revision = match.groups()[1]
return self.revision
self.enUS_revision = match.groups()[1]
return self.enUS_revision
def _query_make_variable(self, variable, make_args=None,
exclude_lines=PyMakeIgnoreList):
@ -557,12 +633,18 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
replace_dict = {}
if config.get("user_repo_override"):
replace_dict['user_repo_override'] = config['user_repo_override']
# this is OK so early because we get it from buildbot, or
# the command line for local dev
replace_dict['revision'] = self._query_revision()
for repository in config['repos']:
current_repo = {}
for key, value in repository.iteritems():
try:
current_repo[key] = value % replace_dict
except TypeError:
# pass through non-interpolables, like booleans
current_repo[key] = value
except KeyError:
self.error('not all the values in "{0}" can be replaced. Check your configuration'.format(value))
raise
@ -581,30 +663,32 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
self._run_make_in_config_dir()
self.make_wget_en_US()
self.make_unpack_en_US()
revision = self._query_revision()
if not revision:
self.fatal("Can't determine revision!")
# TODO do this through VCSMixin instead of hardcoding hg
# self.update(dest=dirs["abs_mozilla_dir"], revision=revision)
hg = self.query_exe("hg")
self.run_command([hg, "update", "-r", revision],
cwd=dirs["abs_mozilla_dir"],
env=self.query_bootstrap_env(),
error_list=BaseErrorList,
halt_on_failure=True, fatal_exit_code=3)
# if checkout updates CLOBBER file with a newer timestamp,
# next make -f client.mk configure will delete archives
# downloaded with make wget_en_US, so just touch CLOBBER file
_clobber_file = self._clobber_file()
if os.path.exists(_clobber_file):
self._touch_file(_clobber_file)
# and again...
# thanks to the last hg update, we can be on different firefox 'version'
# than the one on default,
self._mach_configure()
self._run_make_in_config_dir()
self.download_mar_tools()
# on try we want the source we already have, otherwise update to the
# same as the en-US binary
if self.config.get("update_gecko_source_to_enUS", True):
revision = self._query_enUS_revision()
# TODO do this through VCSMixin instead of hardcoding hg
# self.update(dest=dirs["abs_mozilla_dir"], revision=revision)
hg = self.query_exe("hg")
self.run_command([hg, "update", "-r", revision],
cwd=dirs["abs_mozilla_dir"],
env=self.query_bootstrap_env(),
error_list=BaseErrorList,
halt_on_failure=True, fatal_exit_code=3)
# if checkout updates CLOBBER file with a newer timestamp,
# next make -f client.mk configure will delete archives
# downloaded with make wget_en_US, so just touch CLOBBER file
_clobber_file = self._clobber_file()
if os.path.exists(_clobber_file):
self._touch_file(_clobber_file)
# and again...
# thanks to the last hg update, we can be on different firefox 'version'
# than the one on default,
self._mach_configure()
self._run_make_in_config_dir()
def _run_make_in_config_dir(self):
"""this step creates nsinstall, needed my make_wget_en_US()
"""
@ -706,9 +790,12 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
env = self.query_l10n_env()
dirs = self.query_abs_dirs()
buildid = self._query_buildid()
replace_dict = {
'buildid': buildid,
'branch': config['branch']
}
try:
env['POST_UPLOAD_CMD'] = config['base_post_upload_cmd'] % {'buildid': buildid,
'branch': config['branch']}
env['POST_UPLOAD_CMD'] = config['base_post_upload_cmd'] % replace_dict
except KeyError:
# no base_post_upload_cmd in configuration, just skip it
pass
@ -765,7 +852,7 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
env=env, halt_on_failure=False)
def repack_locale(self, locale):
"""wraps the logic for comapare locale, make installers and generate
"""wraps the logic for compare locale, make installers and generating
complete updates."""
if self.run_compare_locales(locale) != SUCCESS:
@ -783,7 +870,7 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
return FAILURE
# set_upload_files() should be called after make upload, to make sure
# we have all files in place (cheksums, etc)
# we have all files in place (checksums, etc)
if self.set_upload_files(locale):
self.error("failed to get list of files to upload for locale %s" % locale)
return FAILURE
@ -814,7 +901,7 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
return self.abs_dirs
def submit_to_balrog(self):
"""submit to barlog"""
"""submit to balrog"""
if not self.config.get("balrog_servers"):
self.info("balrog_servers not set; skipping balrog submission.")
return
@ -954,6 +1041,9 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin,
auth_file = self._get_tooltool_auth_file()
if auth_file and os.path.exists(auth_file):
cmd.extend(['--authentication-file', auth_file])
cache = config['bootstrap_env'].get('TOOLTOOL_CACHE')
if cache:
cmd.extend(['-c', cache])
self.info(str(cmd))
self.run_command(cmd, cwd=dirs['abs_mozilla_dir'], halt_on_failure=True)