Bug 1776751 - use version with conditioned profile. r=sparky

Differential Revision: https://phabricator.services.mozilla.com/D150666

testing/condprofile/condprof/client.py

@@ -31,7 +31,7 @@ ROOT_URL = TC_SERVICE + "/api/index"
 INDEX_PATH = "gecko.v2.%(repo)s.latest.firefox.condprof-%(platform)s-%(scenario)s"
 PUBLIC_DIR = "artifacts/public/condprof"
 TC_LINK = ROOT_URL + "/v1/task/" + INDEX_PATH + "/" + PUBLIC_DIR + "/"
-ARTIFACT_NAME = "profile-%(platform)s-%(scenario)s-%(customization)s.tgz"
+ARTIFACT_NAME = "profile%(version)s-%(platform)s-%(scenario)s-%(customization)s.tgz"
 CHANGELOG_LINK = (
     ROOT_URL + "/v1/task/" + INDEX_PATH + "/" + PUBLIC_DIR + "/changelog.json"
 )
@@ -100,23 +100,26 @@ def _check_profile(profile_dir):
     _clean_pref_file("user.js")
 
 
-def _retries(callable, onerror=None):
-    retries = 0
+def _retries(callable, onerror=None, retries=RETRIES):
+    _retry_count = 0
     pause = RETRY_PAUSE
 
-    while retries < RETRIES:
+    while _retry_count < retries:
         try:
             return callable()
         except Exception as e:
             if onerror is not None:
                 onerror(e)
             logger.info("Failed, retrying")
-            retries += 1
+            _retry_count += 1
             time.sleep(pause)
             pause *= 1.5
 
     # If we reach that point, it means all attempts failed
-    logger.error("All attempt failed")
+    if _retry_count >= RETRIES:
+        logger.error("All attempt failed")
+    else:
+        logger.info("Retried %s attempts and failed" % _retry_count)
     raise RetriesError()
 
 
@@ -129,6 +132,8 @@ def get_profile(
     download_cache=True,
     repo="mozilla-central",
     remote_test_root="/sdcard/test_root/",
+    version=None,
+    retries=RETRIES,
 ):
     """Extract a conditioned profile in the target directory.
 
@@ -137,12 +142,18 @@ def get_profile(
     """
 
     # XXX assert values
+    if version:
+        version = "-v%s" % version
+    else:
+        version = ""
+
     params = {
         "platform": platform,
         "scenario": scenario,
         "customization": customization,
         "task_id": task_id,
         "repo": repo,
+        "version": version,
     }
     logger.info("Getting conditioned profile with arguments: %s" % params)
     filename = ARTIFACT_NAME % params
@@ -208,7 +219,7 @@ def get_profile(
                 logger.error("Could not remove the file")
 
     try:
-        return _retries(_get_profile, onerror)
+        return _retries(_get_profile, onerror, retries)
     except RetriesError:
         raise ProfileNotFoundError(url)
 

testing/condprofile/condprof/creator.py

@@ -30,7 +30,7 @@ import tempfile
 from arsenic import get_session
 from arsenic.browsers import Firefox
 
-from condprof.util import fresh_profile, logger, obfuscate_file, obfuscate
+from condprof.util import fresh_profile, logger, obfuscate_file, obfuscate, get_version
 from condprof.helpers import close_extra_windows
 from condprof.scenarii import scenarii
 from condprof.client import get_profile, ProfileNotFoundError
@@ -105,20 +105,28 @@ class ProfileCreator:
         if not self.archive:
             return
 
-        logger.info("Creating archive")
-        archiver = Archiver(self.scenario, self.env.profile, self.archive)
-        # the archive name is of the form
-        # profile-<platform>-<scenario>-<customization>.tgz
-        name = "profile-%(platform)s-%(name)s-%(customization)s.tgz"
-        name = name % metadata
-        archive_name = os.path.join(self.archive, name)
-        dir = os.path.dirname(archive_name)
-        if not os.path.exists(dir):
-            os.makedirs(dir)
-        archiver.create_archive(archive_name)
-        logger.info("Archive created at %s" % archive_name)
-        statinfo = os.stat(archive_name)
-        logger.info("Current size is %d" % statinfo.st_size)
+        logger.info("Creating generic archive")
+        names = ["profile-%(platform)s-%(name)s-%(customization)s.tgz"]
+        if metadata["name"] == "full" and metadata["customization"] == "default":
+            names = [
+                "profile-%(platform)s-%(name)s-%(customization)s.tgz",
+                "profile-v%(version)s-%(platform)s-%(name)s-%(customization)s.tgz",
+            ]
+
+        for name in names:
+            archiver = Archiver(self.scenario, self.env.profile, self.archive)
+            # the archive name is of the form
+            # profile[-vXYZ.x]-<platform>-<scenario>-<customization>.tgz
+            name = name % metadata
+            archive_name = os.path.join(self.archive, name)
+            dir = os.path.dirname(archive_name)
+            if not os.path.exists(dir):
+                os.makedirs(dir)
+            archiver.create_archive(archive_name)
+            logger.info("Archive created at %s" % archive_name)
+            statinfo = os.stat(archive_name)
+            logger.info("Current size is %d" % statinfo.st_size)
+
         logger.info("Extracting logs")
         if "logs" in metadata:
             logs = metadata.pop("logs")

testing/condprofile/condprof/util.py

@@ -327,7 +327,8 @@ def write_yml_file(yml_file, yml_data):
 def get_version(firefox):
     p = Popen([firefox, "--version"], stdin=PIPE, stdout=PIPE, stderr=PIPE)
     output, __ = p.communicate()
-    res = output.strip().split()[-1]
+    first_line = output.strip().split(b"\n")[0]
+    res = first_line.split()[-1]
     return res.decode("utf-8")
 
 

testing/condprofile/mach_commands.py

@@ -45,7 +45,7 @@ def fetch(
 ):
     _init(command_context)
     from condprof.client import get_profile
-    from condprof.util import get_current_platform
+    from condprof.util import get_current_platform, get_version
 
     if platform is None:
         platform = get_current_platform()
@@ -53,8 +53,17 @@ def fetch(
     if target_dir is None:
         target_dir = tempfile.mkdtemp()
 
+    version = get_version(command_context.get_binary_path())
+
     get_profile(
-        target_dir, platform, scenario, customization, task_id, download_cache, repo
+        target_dir,
+        platform,
+        scenario,
+        customization,
+        task_id,
+        download_cache,
+        repo,
+        version,
     )
 
 

testing/mochitest/runtests.py

@@ -2151,9 +2151,9 @@ toolbar#nav-bar {
         self.log.info("Created a conditioned-profile copy: %s" % condprof_copy)
         return condprof_copy
 
-    def downloadConditionedProfile(self, profile_scenario):
+    def downloadConditionedProfile(self, profile_scenario, app):
         from condprof.client import get_profile
-        from condprof.util import get_current_platform
+        from condprof.util import get_current_platform, get_version
 
         if self.conditioned_profile_dir:
             # We already have a directory, so provide a copy that
@@ -2169,12 +2169,36 @@ toolbar#nav-bar {
         if not profile_scenario:
             profile_scenario = "settled"
 
-        cond_prof_target_dir = get_profile(
-            temp_download_dir,
-            platform,
-            profile_scenario,
-            repo="mozilla-central",
-        )
+        version = get_version(app)
+        try:
+            cond_prof_target_dir = get_profile(
+                temp_download_dir,
+                platform,
+                profile_scenario,
+                repo="mozilla-central",
+                version=version,
+                retries=2,  # quicker failure
+            )
+        except Exception:
+            if version is None:
+                # any other error is a showstopper
+                self.log.critical("Could not get the conditioned profile")
+                traceback.print_exc()
+                raise
+            version = None
+            try:
+                self.log.info("retrying a profile with no version specified")
+                cond_prof_target_dir = get_profile(
+                    temp_download_dir,
+                    platform,
+                    profile_scenario,
+                    repo="mozilla-central",
+                    version=version,
+                )
+            except Exception:
+                self.log.critical("Could not get the conditioned profile")
+                traceback.print_exc()
+                raise
 
         # Now get the full directory path to our fetched conditioned profile
         self.conditioned_profile_dir = os.path.join(
@@ -2216,7 +2240,7 @@ toolbar#nav-bar {
         if options.conditionedProfile:
             if options.profilePath and os.path.exists(options.profilePath):
                 shutil.rmtree(options.profilePath, ignore_errors=True)
-            options.profilePath = self.downloadConditionedProfile("full")
+            options.profilePath = self.downloadConditionedProfile("full", options.app)
 
             # This is causing `certutil -N -d -f`` to not use -f (pwd file)
             try:

testing/xpcshell/runxpcshelltests.py

@@ -1557,9 +1557,9 @@ class XPCShellTests(object):
         self.log.info("Created a conditioned-profile copy: %s" % condprof_copy)
         return condprof_copy
 
-    def downloadConditionedProfile(self, profile_scenario):
+    def downloadConditionedProfile(self, profile_scenario, app):
         from condprof.client import get_profile
-        from condprof.util import get_current_platform
+        from condprof.util import get_current_platform, get_version
 
         if self.conditioned_profile_dir:
             # We already have a directory, so provide a copy that
@@ -1576,6 +1576,9 @@ class XPCShellTests(object):
         # call condprof's client API to yield our platform-specific
         # conditioned-profile binary
         platform = get_current_platform()
+        version = None
+        if isinstance(app, str):
+            version = get_version(app)
 
         if not profile_scenario:
             profile_scenario = "settled"
@@ -1585,12 +1588,29 @@ class XPCShellTests(object):
                 platform,
                 profile_scenario,
                 repo="mozilla-central",
+                version=version,
+                retries=2,
             )
         except Exception:
-            # any other error is a showstopper
-            self.log.critical("Could not get the conditioned profile")
-            traceback.print_exc()
-            raise
+            if version is None:
+                # any other error is a showstopper
+                self.log.critical("Could not get the conditioned profile")
+                traceback.print_exc()
+                raise
+            version = None
+            try:
+                self.log.info("Retrying a profile with no version specified")
+                cond_prof_target_dir = get_profile(
+                    temp_download_dir,
+                    platform,
+                    profile_scenario,
+                    repo="mozilla-central",
+                    version=version,
+                )
+            except Exception:
+                self.log.critical("Could not get the conditioned profile")
+                traceback.print_exc()
+                raise
 
         # now get the full directory path to our fetched conditioned profile
         self.conditioned_profile_dir = os.path.join(
@@ -1732,7 +1752,9 @@ class XPCShellTests(object):
         self.todoCount = 0
 
         if self.conditionedProfile:
-            self.conditioned_profile_dir = self.downloadConditionedProfile("full")
+            self.conditioned_profile_dir = self.downloadConditionedProfile(
+                "full", self.appPath
+            )
             options["self_test"] = False
             if not options["test_tags"]:
                 options["test_tags"] = []