From f8ccd3e0e0c9fa8c36041563446abf8fb6e7c487 Mon Sep 17 00:00:00 2001 From: Joel Maher Date: Mon, 11 Jul 2022 14:30:26 +0000 Subject: [PATCH] Bug 1776751 - use version with conditioned profile. r=sparky Differential Revision: https://phabricator.services.mozilla.com/D150666 --- testing/condprofile/condprof/client.py | 25 ++++++++++----- testing/condprofile/condprof/creator.py | 38 +++++++++++++--------- testing/condprofile/condprof/util.py | 3 +- testing/condprofile/mach_commands.py | 13 ++++++-- testing/mochitest/runtests.py | 42 +++++++++++++++++++------ testing/xpcshell/runxpcshelltests.py | 36 ++++++++++++++++----- 6 files changed, 116 insertions(+), 41 deletions(-) diff --git a/testing/condprofile/condprof/client.py b/testing/condprofile/condprof/client.py index 53668e83084a..3eb8327369bf 100644 --- a/testing/condprofile/condprof/client.py +++ b/testing/condprofile/condprof/client.py @@ -31,7 +31,7 @@ ROOT_URL = TC_SERVICE + "/api/index" INDEX_PATH = "gecko.v2.%(repo)s.latest.firefox.condprof-%(platform)s-%(scenario)s" PUBLIC_DIR = "artifacts/public/condprof" TC_LINK = ROOT_URL + "/v1/task/" + INDEX_PATH + "/" + PUBLIC_DIR + "/" -ARTIFACT_NAME = "profile-%(platform)s-%(scenario)s-%(customization)s.tgz" +ARTIFACT_NAME = "profile%(version)s-%(platform)s-%(scenario)s-%(customization)s.tgz" CHANGELOG_LINK = ( ROOT_URL + "/v1/task/" + INDEX_PATH + "/" + PUBLIC_DIR + "/changelog.json" ) @@ -100,23 +100,26 @@ def _check_profile(profile_dir): _clean_pref_file("user.js") -def _retries(callable, onerror=None): - retries = 0 +def _retries(callable, onerror=None, retries=RETRIES): + _retry_count = 0 pause = RETRY_PAUSE - while retries < RETRIES: + while _retry_count < retries: try: return callable() except Exception as e: if onerror is not None: onerror(e) logger.info("Failed, retrying") - retries += 1 + _retry_count += 1 time.sleep(pause) pause *= 1.5 # If we reach that point, it means all attempts failed - logger.error("All attempt failed") + if _retry_count >= RETRIES: + logger.error("All attempt failed") + else: + logger.info("Retried %s attempts and failed" % _retry_count) raise RetriesError() @@ -129,6 +132,8 @@ def get_profile( download_cache=True, repo="mozilla-central", remote_test_root="/sdcard/test_root/", + version=None, + retries=RETRIES, ): """Extract a conditioned profile in the target directory. @@ -137,12 +142,18 @@ def get_profile( """ # XXX assert values + if version: + version = "-v%s" % version + else: + version = "" + params = { "platform": platform, "scenario": scenario, "customization": customization, "task_id": task_id, "repo": repo, + "version": version, } logger.info("Getting conditioned profile with arguments: %s" % params) filename = ARTIFACT_NAME % params @@ -208,7 +219,7 @@ def get_profile( logger.error("Could not remove the file") try: - return _retries(_get_profile, onerror) + return _retries(_get_profile, onerror, retries) except RetriesError: raise ProfileNotFoundError(url) diff --git a/testing/condprofile/condprof/creator.py b/testing/condprofile/condprof/creator.py index bcf901b3ebb1..81a1a340179a 100644 --- a/testing/condprofile/condprof/creator.py +++ b/testing/condprofile/condprof/creator.py @@ -30,7 +30,7 @@ import tempfile from arsenic import get_session from arsenic.browsers import Firefox -from condprof.util import fresh_profile, logger, obfuscate_file, obfuscate +from condprof.util import fresh_profile, logger, obfuscate_file, obfuscate, get_version from condprof.helpers import close_extra_windows from condprof.scenarii import scenarii from condprof.client import get_profile, ProfileNotFoundError @@ -105,20 +105,28 @@ class ProfileCreator: if not self.archive: return - logger.info("Creating archive") - archiver = Archiver(self.scenario, self.env.profile, self.archive) - # the archive name is of the form - # profile---.tgz - name = "profile-%(platform)s-%(name)s-%(customization)s.tgz" - name = name % metadata - archive_name = os.path.join(self.archive, name) - dir = os.path.dirname(archive_name) - if not os.path.exists(dir): - os.makedirs(dir) - archiver.create_archive(archive_name) - logger.info("Archive created at %s" % archive_name) - statinfo = os.stat(archive_name) - logger.info("Current size is %d" % statinfo.st_size) + logger.info("Creating generic archive") + names = ["profile-%(platform)s-%(name)s-%(customization)s.tgz"] + if metadata["name"] == "full" and metadata["customization"] == "default": + names = [ + "profile-%(platform)s-%(name)s-%(customization)s.tgz", + "profile-v%(version)s-%(platform)s-%(name)s-%(customization)s.tgz", + ] + + for name in names: + archiver = Archiver(self.scenario, self.env.profile, self.archive) + # the archive name is of the form + # profile[-vXYZ.x]---.tgz + name = name % metadata + archive_name = os.path.join(self.archive, name) + dir = os.path.dirname(archive_name) + if not os.path.exists(dir): + os.makedirs(dir) + archiver.create_archive(archive_name) + logger.info("Archive created at %s" % archive_name) + statinfo = os.stat(archive_name) + logger.info("Current size is %d" % statinfo.st_size) + logger.info("Extracting logs") if "logs" in metadata: logs = metadata.pop("logs") diff --git a/testing/condprofile/condprof/util.py b/testing/condprofile/condprof/util.py index 3a6f594215c1..48fb2d62d8f0 100644 --- a/testing/condprofile/condprof/util.py +++ b/testing/condprofile/condprof/util.py @@ -327,7 +327,8 @@ def write_yml_file(yml_file, yml_data): def get_version(firefox): p = Popen([firefox, "--version"], stdin=PIPE, stdout=PIPE, stderr=PIPE) output, __ = p.communicate() - res = output.strip().split()[-1] + first_line = output.strip().split(b"\n")[0] + res = first_line.split()[-1] return res.decode("utf-8") diff --git a/testing/condprofile/mach_commands.py b/testing/condprofile/mach_commands.py index 893d413fe36b..0e9d0c00b8a6 100644 --- a/testing/condprofile/mach_commands.py +++ b/testing/condprofile/mach_commands.py @@ -45,7 +45,7 @@ def fetch( ): _init(command_context) from condprof.client import get_profile - from condprof.util import get_current_platform + from condprof.util import get_current_platform, get_version if platform is None: platform = get_current_platform() @@ -53,8 +53,17 @@ def fetch( if target_dir is None: target_dir = tempfile.mkdtemp() + version = get_version(command_context.get_binary_path()) + get_profile( - target_dir, platform, scenario, customization, task_id, download_cache, repo + target_dir, + platform, + scenario, + customization, + task_id, + download_cache, + repo, + version, ) diff --git a/testing/mochitest/runtests.py b/testing/mochitest/runtests.py index 18ee1b5e0559..b7e4114909e0 100644 --- a/testing/mochitest/runtests.py +++ b/testing/mochitest/runtests.py @@ -2151,9 +2151,9 @@ toolbar#nav-bar { self.log.info("Created a conditioned-profile copy: %s" % condprof_copy) return condprof_copy - def downloadConditionedProfile(self, profile_scenario): + def downloadConditionedProfile(self, profile_scenario, app): from condprof.client import get_profile - from condprof.util import get_current_platform + from condprof.util import get_current_platform, get_version if self.conditioned_profile_dir: # We already have a directory, so provide a copy that @@ -2169,12 +2169,36 @@ toolbar#nav-bar { if not profile_scenario: profile_scenario = "settled" - cond_prof_target_dir = get_profile( - temp_download_dir, - platform, - profile_scenario, - repo="mozilla-central", - ) + version = get_version(app) + try: + cond_prof_target_dir = get_profile( + temp_download_dir, + platform, + profile_scenario, + repo="mozilla-central", + version=version, + retries=2, # quicker failure + ) + except Exception: + if version is None: + # any other error is a showstopper + self.log.critical("Could not get the conditioned profile") + traceback.print_exc() + raise + version = None + try: + self.log.info("retrying a profile with no version specified") + cond_prof_target_dir = get_profile( + temp_download_dir, + platform, + profile_scenario, + repo="mozilla-central", + version=version, + ) + except Exception: + self.log.critical("Could not get the conditioned profile") + traceback.print_exc() + raise # Now get the full directory path to our fetched conditioned profile self.conditioned_profile_dir = os.path.join( @@ -2216,7 +2240,7 @@ toolbar#nav-bar { if options.conditionedProfile: if options.profilePath and os.path.exists(options.profilePath): shutil.rmtree(options.profilePath, ignore_errors=True) - options.profilePath = self.downloadConditionedProfile("full") + options.profilePath = self.downloadConditionedProfile("full", options.app) # This is causing `certutil -N -d -f`` to not use -f (pwd file) try: diff --git a/testing/xpcshell/runxpcshelltests.py b/testing/xpcshell/runxpcshelltests.py index 2bec621b815b..3b47bdbc2cf0 100755 --- a/testing/xpcshell/runxpcshelltests.py +++ b/testing/xpcshell/runxpcshelltests.py @@ -1557,9 +1557,9 @@ class XPCShellTests(object): self.log.info("Created a conditioned-profile copy: %s" % condprof_copy) return condprof_copy - def downloadConditionedProfile(self, profile_scenario): + def downloadConditionedProfile(self, profile_scenario, app): from condprof.client import get_profile - from condprof.util import get_current_platform + from condprof.util import get_current_platform, get_version if self.conditioned_profile_dir: # We already have a directory, so provide a copy that @@ -1576,6 +1576,9 @@ class XPCShellTests(object): # call condprof's client API to yield our platform-specific # conditioned-profile binary platform = get_current_platform() + version = None + if isinstance(app, str): + version = get_version(app) if not profile_scenario: profile_scenario = "settled" @@ -1585,12 +1588,29 @@ class XPCShellTests(object): platform, profile_scenario, repo="mozilla-central", + version=version, + retries=2, ) except Exception: - # any other error is a showstopper - self.log.critical("Could not get the conditioned profile") - traceback.print_exc() - raise + if version is None: + # any other error is a showstopper + self.log.critical("Could not get the conditioned profile") + traceback.print_exc() + raise + version = None + try: + self.log.info("Retrying a profile with no version specified") + cond_prof_target_dir = get_profile( + temp_download_dir, + platform, + profile_scenario, + repo="mozilla-central", + version=version, + ) + except Exception: + self.log.critical("Could not get the conditioned profile") + traceback.print_exc() + raise # now get the full directory path to our fetched conditioned profile self.conditioned_profile_dir = os.path.join( @@ -1732,7 +1752,9 @@ class XPCShellTests(object): self.todoCount = 0 if self.conditionedProfile: - self.conditioned_profile_dir = self.downloadConditionedProfile("full") + self.conditioned_profile_dir = self.downloadConditionedProfile( + "full", self.appPath + ) options["self_test"] = False if not options["test_tags"]: options["test_tags"] = []