diff --git a/testing/mozharness/configs/single_locale/linux.py b/testing/mozharness/configs/single_locale/linux.py index 37af15c3881a..dcb72ceb10c7 100644 --- a/testing/mozharness/configs/single_locale/linux.py +++ b/testing/mozharness/configs/single_locale/linux.py @@ -14,6 +14,8 @@ config = { "LOCALE_MERGEDIR": "%(abs_merge_dir)s/", "L10NBASEDIR": "../../l10n", "MOZ_MAKE_COMPLETE_MAR": "1", + 'TOOLTOOL_CACHE': '/builds/tooltool_cache', + 'TOOLTOOL_HOME': '/builds', }, "ssh_key_dir": "/home/mock_mozilla/.ssh", "log_name": "single_locale", diff --git a/testing/mozharness/configs/single_locale/linux64.py b/testing/mozharness/configs/single_locale/linux64.py index 1bcfe2065a7c..fe9f7122e2d4 100644 --- a/testing/mozharness/configs/single_locale/linux64.py +++ b/testing/mozharness/configs/single_locale/linux64.py @@ -14,6 +14,8 @@ config = { "LOCALE_MERGEDIR": "%(abs_merge_dir)s/", "L10NBASEDIR": "../../l10n", "MOZ_MAKE_COMPLETE_MAR": "1", + 'TOOLTOOL_CACHE': '/builds/tooltool_cache', + 'TOOLTOOL_HOME': '/builds', }, "ssh_key_dir": "/home/mock_mozilla/.ssh", "log_name": "single_locale", diff --git a/testing/mozharness/configs/single_locale/macosx64.py b/testing/mozharness/configs/single_locale/macosx64.py index ceafa8398735..719fca71620b 100644 --- a/testing/mozharness/configs/single_locale/macosx64.py +++ b/testing/mozharness/configs/single_locale/macosx64.py @@ -18,6 +18,8 @@ config = { "L10NBASEDIR": "../../l10n", "MOZ_MAKE_COMPLETE_MAR": "1", "LOCALE_MERGEDIR": "%(abs_merge_dir)s/", + 'TOOLTOOL_CACHE': '/builds/tooltool_cache', + 'TOOLTOOL_HOME': '/builds', }, "ssh_key_dir": "~/.ssh", "log_name": "single_locale", @@ -34,7 +36,6 @@ config = { 'tooltool_script': ["/builds/tooltool.py"], 'tooltool_bootstrap': "setup.sh", 'tooltool_manifest_src': 'browser/config/tooltool-manifests/macosx64/releng.manifest', - # balrog credential file: 'balrog_credentials_file': 'oauth.txt', diff --git a/testing/mozharness/configs/single_locale/production.py b/testing/mozharness/configs/single_locale/production.py index 8e5ce72cebf3..fe97fe361425 100644 --- a/testing/mozharness/configs/single_locale/production.py +++ b/testing/mozharness/configs/single_locale/production.py @@ -1,4 +1,5 @@ config = { + "upload_environment": "prod", "upload_env": { "UPLOAD_USER": "ffxbld", # ssh_key_dir is defined per platform: it is "~/.ssh" for every platform diff --git a/testing/mozharness/configs/single_locale/staging.py b/testing/mozharness/configs/single_locale/staging.py index b705aeed4c9b..82caa8dda6f2 100644 --- a/testing/mozharness/configs/single_locale/staging.py +++ b/testing/mozharness/configs/single_locale/staging.py @@ -1,10 +1,11 @@ config = { + "upload_environment": "stage", "upload_env": { "UPLOAD_USER": "ffxbld", # ssh_key_dir is defined per platform: it is "~/.ssh" for every platform # except when mock is in use, in this case, ssh_key_dir is # /home/mock_mozilla/.ssh - "UPLOAD_SSH_KEY": "%(ssh_key_dir)s/stage-ffxbld_rsa", + "UPLOAD_SSH_KEY": "%(ssh_key_dir)s/ffxbld_rsa", "UPLOAD_HOST": "upload.ffxbld.productdelivery.stage.mozaws.net", "POST_UPLOAD_CMD": "post_upload.py -b %(branch)s-l10n -p %(stage_product)s -i %(buildid)s --release-to-latest --release-to-dated %(post_upload_extra)s", "UPLOAD_TO_TEMP": "1" diff --git a/testing/mozharness/configs/single_locale/try.py b/testing/mozharness/configs/single_locale/try.py new file mode 100644 index 000000000000..f3dd9616f997 --- /dev/null +++ b/testing/mozharness/configs/single_locale/try.py @@ -0,0 +1,47 @@ +config = { + "nightly_build": False, + "branch": "try", + "en_us_binary_url": "http://archive.mozilla.org/pub/firefox/nightly/latest-mozilla-central", + "update_channel": "nightly", + "latest_mar_dir": '/pub/firefox/nightly/latest-mozilla-central-l10n', + "update_gecko_source_to_enUS": False, + + # l10n + "hg_l10n_base": "https://hg.mozilla.org/l10n-central", + + # mar + "mar_tools_url": "http://ftp.mozilla.org/pub/mozilla.org/firefox/nightly/latest-mozilla-central/mar-tools/%(platform)s", + + # repositories + "mozilla_dir": "try", + "hgtool_base_bundle_urls": ["https://archive.mozilla.org/pub/firefox/bundles"], + "repos": [{ + "vcs": "hg", + "repo": "https://hg.mozilla.org/build/tools", + "revision": "default", + "dest": "tools", + }, { + "vcs": "hgtool", + "repo": "https://hg.mozilla.org/try", + "revision": "%(revision)s", + "dest": "try", + "clone_by_revision": True, + "clone_with_purge": True, + }, { + "vcs": "hgtool", + "repo": "https://hg.mozilla.org/build/compare-locales", + "revision": "RELEASE_AUTOMATION" + }], + # purge options + 'is_automation': True, + "upload_env": { + "UPLOAD_USER": "trybld", + # ssh_key_dir is defined per platform: it is "~/.ssh" for every platform + # except when mock is in use, in this case, ssh_key_dir is + # /home/mock_mozilla/.ssh + "UPLOAD_SSH_KEY": "%(ssh_key_dir)s/trybld_dsa", + "UPLOAD_HOST": "upload.trybld.productdelivery.%(upload_environment)s.mozaws.net", + "POST_UPLOAD_CMD": "post_upload.py --who %(who)s --builddir %(branch)s-%(platform)s --tinderbox-builds-dir %(who)s-%(revision)s -p %(stage_product)s -i %(buildid)s --revision %(revision)s --release-to-try-builds %(post_upload_extra)s", + "UPLOAD_TO_TEMP": "1" + }, +} diff --git a/testing/mozharness/configs/single_locale/win32.py b/testing/mozharness/configs/single_locale/win32.py index 0466d50f5420..8a8546b43330 100644 --- a/testing/mozharness/configs/single_locale/win32.py +++ b/testing/mozharness/configs/single_locale/win32.py @@ -14,6 +14,10 @@ config = { "DIST": "%(abs_objdir)s", "L10NBASEDIR": "../../l10n", "MOZ_MAKE_COMPLETE_MAR": "1", + "PATH": 'C:\\mozilla-build\\nsis-3.0b1;' + '%s' % (os.environ.get('path')), + 'TOOLTOOL_CACHE': '/c/builds/tooltool_cache', + 'TOOLTOOL_HOME': '/c/builds', }, "ssh_key_dir": "~/.ssh", "log_name": "single_locale", diff --git a/testing/mozharness/configs/single_locale/win64.py b/testing/mozharness/configs/single_locale/win64.py index 483ef15315d9..8e4ccea104c8 100644 --- a/testing/mozharness/configs/single_locale/win64.py +++ b/testing/mozharness/configs/single_locale/win64.py @@ -14,6 +14,10 @@ config = { "LOCALE_MERGEDIR": "%(abs_merge_dir)s", "L10NBASEDIR": "../../l10n", "MOZ_MAKE_COMPLETE_MAR": "1", + "PATH": 'C:\\mozilla-build\\nsis-3.0b1;' + '%s' % (os.environ.get('path')), + 'TOOLTOOL_CACHE': '/c/builds/tooltool_cache', + 'TOOLTOOL_HOME': '/c/builds', }, "ssh_key_dir": "~/.ssh", "log_name": "single_locale", diff --git a/testing/mozharness/scripts/desktop_l10n.py b/testing/mozharness/scripts/desktop_l10n.py index 8129bbbd3ca6..53ab36a08760 100755 --- a/testing/mozharness/scripts/desktop_l10n.py +++ b/testing/mozharness/scripts/desktop_l10n.py @@ -65,14 +65,15 @@ configuration_tokens = ('branch', 'update_channel', 'ssh_key_dir', 'stage_product', - ) + 'upload_environment', + ) # some other values such as "%(version)s", "%(buildid)s", ... # are defined at run time and they cannot be enforced in the _pre_config_lock # phase runtime_config_tokens = ('buildid', 'version', 'locale', 'from_buildid', - 'abs_objdir', 'abs_merge_dir', 'version', - 'to_buildid', 'en_us_binary_url', 'mar_tools_url') - + 'abs_objdir', 'abs_merge_dir', 'revision', + 'to_buildid', 'en_us_binary_url', 'mar_tools_url', + 'post_upload_extra', 'who') # DesktopSingleLocale {{{1 class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, @@ -122,6 +123,13 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, "dest": "tag_override", "type": "string", "help": "Override the tags set for all repos"} + ], [ + ['--revision', ], + {"action": "store", + "dest": "revision", + "type": "string", + "help": "Override the gecko revision to use (otherwise use buildbot supplied" + " value, or en-US revision) "} ], [ ['--user-repo-override', ], {"action": "store", @@ -209,6 +217,7 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, self.bootstrap_env = None self.upload_env = None self.revision = None + self.enUS_revision = None self.version = None self.upload_urls = {} self.locales_property = {} @@ -334,8 +343,9 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, for element in config_option: new_list.append(self.__detokenise_element(element, token, value)) return tuple(new_list) - # everything else, bool, number, ... - return None + else: + # everything else, bool, number, ... + return config_option # Helper methods {{{2 def query_bootstrap_env(self): @@ -377,16 +387,48 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, self.bootstrap_env = bootstrap_env return self.bootstrap_env + def _query_who(self): + """ looks for who triggered the build with a change. + + This is used for things like try builds where the upload dir is + associated with who pushed to try. First it will look in self.config + and failing that, will poll buildbot_config + If nothing is found, it will default to returning "nobody@example.com" + """ + if self.config.get('who'): + return self.config['who'] + self.read_buildbot_config() + try: + return self.buildbot_config['sourcestamp']['changes'][0]['who'] + except (KeyError, IndexError): + # KeyError: "sourcestamp" or "changes" or "who" not in buildbot_config + # IndexError: buildbot_config['sourcestamp']['changes'] is empty + pass + try: + return str(self.buildbot_config['properties']['who']) + except KeyError: + pass + return "nobody@example.com" + def _query_upload_env(self): """returns the environment used for the upload step""" if self.upload_env: return self.upload_env config = self.config - buildid = self._query_buildid() - version = self.query_version() + + replace_dict = { + 'buildid': self._query_buildid(), + 'version': self.query_version(), + 'post_upload_extra': ' '.join(config.get('post_upload_extra', [])), + 'upload_environment': config['upload_environment'], + } + if config['branch'] == 'try': + replace_dict.update({ + 'who': self._query_who(), + 'revision': self._query_revision(), + }) upload_env = self.query_env(partial_env=config.get("upload_env"), - replace_dict={'buildid': buildid, - 'version': version}) + replace_dict=replace_dict) upload_env['LATEST_MAR_DIR'] = config['latest_mar_dir'] # check if there are any extra option from the platform configuration # and append them to the env @@ -394,6 +436,7 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, if 'upload_env_extra' in config: for extra in config['upload_env_extra']: upload_env[extra] = config['upload_env_extra'][extra] + self.upload_env = upload_env return self.upload_env @@ -435,18 +478,51 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, return self.buildid def _query_revision(self): + """ Get the gecko revision in this order of precedence + * cached value + * command line arg --revision (development) + * buildbot properties (try with buildbot forced build) + * buildbot change (try with buildbot scheduler) + * from the en-US build (m-c & m-a) + + This will fail the last case if the build hasn't been pulled yet. + """ + if self.revision: + return self.revision + + self.read_buildbot_config() + config = self.config + revision = None + if config.get("revision"): + revision = config["revision"] + elif 'revision' in self.buildbot_properties: + revision = self.buildbot_properties['revision'] + elif (self.buildbot_config and + self.buildbot_config.get('sourcestamp', {}).get('revision')): + revision = self.buildbot_config['sourcestamp']['revision'] + elif self.buildbot_config and self.buildbot_config.get('revision'): + revision = self.buildbot_config['revision'] + elif config.get("update_gecko_source_to_enUS", True): + revision = self._query_enUS_revision() + + if not revision: + self.fatal("Can't determine revision!") + self.revision = str(revision) + return self.revision + + def _query_enUS_revision(self): """Get revision from the objdir. Only valid after setup is run. """ - if self.revision: - return self.revision + if self.enUS_revision: + return self.enUS_revision r = re.compile(r"^(gecko|fx)_revision ([0-9a-f]+\+?)$") output = self._query_make_ident_output() for line in output.splitlines(): match = r.match(line) if match: - self.revision = match.groups()[1] - return self.revision + self.enUS_revision = match.groups()[1] + return self.enUS_revision def _query_make_variable(self, variable, make_args=None, exclude_lines=PyMakeIgnoreList): @@ -557,12 +633,18 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, replace_dict = {} if config.get("user_repo_override"): replace_dict['user_repo_override'] = config['user_repo_override'] + # this is OK so early because we get it from buildbot, or + # the command line for local dev + replace_dict['revision'] = self._query_revision() for repository in config['repos']: current_repo = {} for key, value in repository.iteritems(): try: current_repo[key] = value % replace_dict + except TypeError: + # pass through non-interpolables, like booleans + current_repo[key] = value except KeyError: self.error('not all the values in "{0}" can be replaced. Check your configuration'.format(value)) raise @@ -581,30 +663,32 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, self._run_make_in_config_dir() self.make_wget_en_US() self.make_unpack_en_US() - revision = self._query_revision() - if not revision: - self.fatal("Can't determine revision!") - # TODO do this through VCSMixin instead of hardcoding hg - # self.update(dest=dirs["abs_mozilla_dir"], revision=revision) - hg = self.query_exe("hg") - self.run_command([hg, "update", "-r", revision], - cwd=dirs["abs_mozilla_dir"], - env=self.query_bootstrap_env(), - error_list=BaseErrorList, - halt_on_failure=True, fatal_exit_code=3) - # if checkout updates CLOBBER file with a newer timestamp, - # next make -f client.mk configure will delete archives - # downloaded with make wget_en_US, so just touch CLOBBER file - _clobber_file = self._clobber_file() - if os.path.exists(_clobber_file): - self._touch_file(_clobber_file) - # and again... - # thanks to the last hg update, we can be on different firefox 'version' - # than the one on default, - self._mach_configure() - self._run_make_in_config_dir() self.download_mar_tools() + # on try we want the source we already have, otherwise update to the + # same as the en-US binary + if self.config.get("update_gecko_source_to_enUS", True): + revision = self._query_enUS_revision() + # TODO do this through VCSMixin instead of hardcoding hg + # self.update(dest=dirs["abs_mozilla_dir"], revision=revision) + hg = self.query_exe("hg") + self.run_command([hg, "update", "-r", revision], + cwd=dirs["abs_mozilla_dir"], + env=self.query_bootstrap_env(), + error_list=BaseErrorList, + halt_on_failure=True, fatal_exit_code=3) + # if checkout updates CLOBBER file with a newer timestamp, + # next make -f client.mk configure will delete archives + # downloaded with make wget_en_US, so just touch CLOBBER file + _clobber_file = self._clobber_file() + if os.path.exists(_clobber_file): + self._touch_file(_clobber_file) + # and again... + # thanks to the last hg update, we can be on different firefox 'version' + # than the one on default, + self._mach_configure() + self._run_make_in_config_dir() + def _run_make_in_config_dir(self): """this step creates nsinstall, needed my make_wget_en_US() """ @@ -706,9 +790,12 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, env = self.query_l10n_env() dirs = self.query_abs_dirs() buildid = self._query_buildid() + replace_dict = { + 'buildid': buildid, + 'branch': config['branch'] + } try: - env['POST_UPLOAD_CMD'] = config['base_post_upload_cmd'] % {'buildid': buildid, - 'branch': config['branch']} + env['POST_UPLOAD_CMD'] = config['base_post_upload_cmd'] % replace_dict except KeyError: # no base_post_upload_cmd in configuration, just skip it pass @@ -765,7 +852,7 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, env=env, halt_on_failure=False) def repack_locale(self, locale): - """wraps the logic for comapare locale, make installers and generate + """wraps the logic for compare locale, make installers and generating complete updates.""" if self.run_compare_locales(locale) != SUCCESS: @@ -783,7 +870,7 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, return FAILURE # set_upload_files() should be called after make upload, to make sure - # we have all files in place (cheksums, etc) + # we have all files in place (checksums, etc) if self.set_upload_files(locale): self.error("failed to get list of files to upload for locale %s" % locale) return FAILURE @@ -814,7 +901,7 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, return self.abs_dirs def submit_to_balrog(self): - """submit to barlog""" + """submit to balrog""" if not self.config.get("balrog_servers"): self.info("balrog_servers not set; skipping balrog submission.") return @@ -954,6 +1041,9 @@ class DesktopSingleLocale(LocalesMixin, ReleaseMixin, MockMixin, BuildbotMixin, auth_file = self._get_tooltool_auth_file() if auth_file and os.path.exists(auth_file): cmd.extend(['--authentication-file', auth_file]) + cache = config['bootstrap_env'].get('TOOLTOOL_CACHE') + if cache: + cmd.extend(['-c', cache]) self.info(str(cmd)) self.run_command(cmd, cwd=dirs['abs_mozilla_dir'], halt_on_failure=True)