Bug 1701764 - Only use a conditioned profile if a scenario is passed on the command line; r=perftest-reviewers,Bebe

Differential Revision: https://phabricator.services.mozilla.com/D110213

taskcluster/ci/test/browsertime-desktop.yml

@@ -35,7 +35,6 @@ job-defaults:
                     - raptor/linux_config.py
         extra-options:
             - --browsertime
-            - --no-conditioned-profile
     variants:
         by-app:
             firefox:
@@ -63,12 +62,10 @@ browsertime-tp6:
                     windows10.*shippable.*-qr.*:
                         - --chimera
                         - --browsertime
-                        - --no-conditioned-profile
                         - --browsertime-no-ffwindowrecorder
                     default:
                         - --chimera
                         - --browsertime
-                        - --no-conditioned-profile
         raptor-test: tp6
         run-on-projects:
             by-variant:

taskcluster/ci/test/browsertime-mobile.yml

@@ -55,7 +55,6 @@ job-defaults:
             - --browsertime
             # Bug 1635749, disable window recorder temporarily
             - --browsertime-no-ffwindowrecorder
-            - --no-conditioned-profile
     optimization:
         skip-unless-backstop: null
     python-3: true

testing/mozharness/mozharness/mozilla/testing/raptor.py

@@ -225,15 +225,6 @@ class Raptor(
                     "help": "Enable the WebRender compositor in Gecko.",
                 },
             ],
-            [
-                ["--no-conditioned-profile"],
-                {
-                    "action": "store_true",
-                    "dest": "no_conditioned_profile",
-                    "default": False,
-                    "help": "Run without the conditioned profile.",
-                },
-            ],
             [
                 ["--device-name"],
                 {
@@ -394,12 +385,12 @@ class Raptor(
                 },
             ],
             [
-                ["--conditioned-profile-scenario"],
+                ["--conditioned-profile"],
                 {
-                    "dest": "conditioned_profile_scenario",
+                    "dest": "conditioned_profile",
                     "type": "str",
-                    "default": "settled",
-                    "help": "Name of profile scenario.",
+                    "default": None,
+                    "help": "Name of conditioned profile to use.",
                 },
             ],
             [
@@ -607,9 +598,7 @@ class Raptor(
         self.live_sites = self.config.get("live_sites")
         self.chimera = self.config.get("chimera")
         self.disable_perf_tuning = self.config.get("disable_perf_tuning")
-        self.conditioned_profile_scenario = self.config.get(
-            "conditioned_profile_scenario", "settled"
-        )
+        self.conditioned_profile = self.config.get("conditioned_profile")
         self.extra_prefs = self.config.get("extra_prefs")
         self.environment = self.config.get("environment")
         self.is_release_build = self.config.get("is_release_build")
@@ -846,10 +835,8 @@ class Raptor(
             kw_options["device-name"] = self.config["device_name"]
         if self.config.get("activity") is not None:
             kw_options["activity"] = self.config["activity"]
-        if self.config.get("conditioned_profile_scenario") is not None:
-            kw_options["conditioned-profile-scenario"] = self.config[
-                "conditioned_profile_scenario"
-            ]
+        if self.config.get("conditioned_profile") is not None:
+            kw_options["conditioned-profile"] = self.config["conditioned_profile"]
 
         kw_options.update(kw)
         if self.host:
@@ -884,8 +871,6 @@ class Raptor(
             options.extend(["--cold"])
         if self.config.get("enable_webrender", False):
             options.extend(["--enable-webrender"])
-        if self.config.get("no_conditioned_profile", False):
-            options.extend(["--no-conditioned-profile"])
         if self.config.get("enable_fission", False):
             options.extend(["--enable-fission"])
         if self.config.get("verbose", False):

testing/raptor/mach_commands.py

@@ -63,7 +63,7 @@ class RaptorRunner(MozbuildObject):
         self.cpu_test = kwargs["cpu_test"]
         self.live_sites = kwargs["live_sites"]
         self.disable_perf_tuning = kwargs["disable_perf_tuning"]
-        self.conditioned_profile_scenario = kwargs["conditioned_profile_scenario"]
+        self.conditioned_profile = kwargs["conditioned_profile"]
         self.device_name = kwargs["device_name"]
         self.enable_marionette_trace = kwargs["enable_marionette_trace"]
         self.browsertime_visualmetrics = kwargs["browsertime_visualmetrics"]
@@ -173,7 +173,7 @@ class RaptorRunner(MozbuildObject):
             "cpu_test": self.cpu_test,
             "live_sites": self.live_sites,
             "disable_perf_tuning": self.disable_perf_tuning,
-            "conditioned_profile_scenario": self.conditioned_profile_scenario,
+            "conditioned_profile": self.conditioned_profile,
             "is_release_build": self.is_release_build,
             "device_name": self.device_name,
             "enable_marionette_trace": self.enable_marionette_trace,

testing/raptor/raptor/browsertime/base.py

@@ -299,7 +299,7 @@ class Browsertime(Perftest):
             browsertime_options.extend(["--video", "false", "--visualMetrics", "false"])
 
         # have browsertime use our newly-created conditioned-profile path
-        if self.using_condprof:
+        if self.config.get("conditioned_profile"):
             self.profile.profile = self.conditioned_profile_dir
 
         if self.config["gecko_profile"]:

testing/raptor/raptor/cmdline.py

@@ -270,13 +270,6 @@ def create_parser(mach_interface=False):
         default=False,
         help="Enable the WebRender compositor in Gecko.",
     )
-    add_arg(
-        "--no-conditioned-profile",
-        dest="no_conditioned_profile",
-        action="store_true",
-        default=False,
-        help="Run Raptor tests without a conditioned profile.",
-    )
     add_arg(
         "--device-name",
         dest="device_name",
@@ -343,11 +336,11 @@ def create_parser(mach_interface=False):
         help="Disable performance tuning on android.",
     )
     add_arg(
-        "--conditioned-profile-scenario",
-        dest="conditioned_profile_scenario",
-        default="settled",
+        "--conditioned-profile",
+        dest="conditioned_profile",
+        default=None,
         type=str,
-        help="Name of profile scenario.",
+        help="Name of conditioned profile to use.",
     )
 
     # for browsertime jobs, cold page load is determined by a '--cold' cmd line argument

testing/raptor/raptor/perftest.py

@@ -91,10 +91,9 @@ class Perftest(object):
         e10s=True,
         enable_webrender=False,
         results_handler_class=RaptorResultsHandler,
-        no_conditioned_profile=False,
         device_name=None,
         disable_perf_tuning=False,
-        conditioned_profile_scenario="settled",
+        conditioned_profile=None,
         chimera=False,
         extra_prefs={},
         environment={},
@@ -130,11 +129,10 @@ class Perftest(object):
             "enable_control_server_wait": memory_test or cpu_test,
             "e10s": e10s,
             "enable_webrender": enable_webrender,
-            "no_conditioned_profile": no_conditioned_profile,
             "device_name": device_name,
             "enable_fission": extra_prefs.get("fission.autostart", False),
             "disable_perf_tuning": disable_perf_tuning,
-            "conditioned_profile_scenario": conditioned_profile_scenario,
+            "conditioned_profile": conditioned_profile,
             "chimera": chimera,
             "extra_prefs": extra_prefs,
             "environment": environment,
@@ -143,19 +141,19 @@ class Perftest(object):
         }
 
         self.firefox_android_apps = FIREFOX_ANDROID_APPS
+
         # We are deactivating the conditioned profiles for:
         # - win10-aarch64 : no support for geckodriver see 1582757
         # - reference browser: no conditioned profiles created see 1606767
-        self.using_condprof = not (
-            (self.config["platform"] == "win" and self.config["processor"] == "aarch64")
-            or self.config["binary"] == "org.mozilla.reference.browser.raptor"
-            or self.config["no_conditioned_profile"]
-        )
-        if self.using_condprof:
+        if (
+            self.config["platform"] == "win" and self.config["processor"] == "aarch64"
+        ) or self.config["binary"] == "org.mozilla.reference.browser.raptor":
+            self.config["conditioned_profile"] = None
+
+        if self.config["conditioned_profile"]:
             LOG.info("Using a conditioned profile.")
         else:
             LOG.info("Using an empty profile.")
-        self.config["using_condprof"] = self.using_condprof
 
         # To differentiate between chrome/firefox failures, we
         # set an app variable in the logger which prefixes messages
@@ -189,7 +187,7 @@ class Perftest(object):
 
         # For the post startup delay, we want to max it to 1s when using the
         # conditioned profiles.
-        if self.using_condprof and not self.run_local:
+        if self.config.get("conditioned_profile") and not self.run_local:
             self.post_startup_delay = min(post_startup_delay, POST_DELAY_CONDPROF)
         else:
             # if running debug-mode reduce the pause after browser startup
@@ -267,7 +265,7 @@ class Perftest(object):
         alternate_repo = "mozilla-central" if repo != "mozilla-central" else "try"
         LOG.info("Getting profile from project %s" % repo)
 
-        profile_scenario = self.config.get("conditioned_profile_scenario", "settled")
+        profile_scenario = self.config.get("conditioned_profile")
         try:
             cond_prof_target_dir = get_profile(
                 temp_download_dir, platform, profile_scenario, repo=repo
@@ -303,11 +301,10 @@ class Perftest(object):
         return self.conditioned_profile_copy
 
     def build_browser_profile(self):
-        if not self.using_condprof or self.config["app"] in [
-            "chrome",
-            "chromium",
-            "chrome-m",
-        ]:
+        if (
+            self.config["app"] in ["chrome", "chromium", "chrome-m"]
+            or self.config.get("conditioned_profile") is None
+        ):
             self.profile = create_profile(self.profile_class)
         else:
             # use mozprofile to create a profile for us, from our conditioned profile's path

testing/raptor/raptor/raptor.py

@@ -141,9 +141,8 @@ def main(args=sys.argv[1:]):
             extra_prefs=args.extra_prefs or {},
             environment=args.environment or {},
             device_name=args.device_name,
-            no_conditioned_profile=args.no_conditioned_profile,
             disable_perf_tuning=args.disable_perf_tuning,
-            conditioned_profile_scenario=args.conditioned_profile_scenario,
+            conditioned_profile=args.conditioned_profile,
             chimera=args.chimera,
             project=args.project,
             verbose=args.verbose,

testing/raptor/raptor/results.py

@@ -18,7 +18,7 @@ from output import RaptorOutput, BrowsertimeOutput
 
 LOG = RaptorLogger(component="perftest-results-handler")
 KNOWN_TEST_MODIFIERS = [
-    "nocondprof",
+    "condprof-settled",
     "fission",
     "live",
     "gecko_profile",
@@ -39,7 +39,7 @@ class PerftestResultsHandler(object):
         memory_test=False,
         live_sites=False,
         app=None,
-        no_conditioned_profile=False,
+        conditioned_profile=None,
         cold=False,
         enable_webrender=False,
         chimera=False,
@@ -51,6 +51,7 @@ class PerftestResultsHandler(object):
         self.memory_test = memory_test
         self.live_sites = live_sites
         self.app = app
+        self.conditioned_profile = conditioned_profile
         self.results = []
         self.page_timeout_list = []
         self.images = []
@@ -61,7 +62,6 @@ class PerftestResultsHandler(object):
         self.webrender_enabled = enable_webrender
         self.browser_version = None
         self.browser_name = None
-        self.no_conditioned_profile = no_conditioned_profile
         self.cold = cold
         self.chimera = chimera
 
@@ -76,8 +76,8 @@ class PerftestResultsHandler(object):
         # checking all known fields. Otherwise, we only check
         # the fields that were given to us.
         if modifiers is None:
-            if self.no_conditioned_profile:
-                extra_options.append("nocondprof")
+            if self.conditioned_profile:
+                extra_options.append("condprof-%s" % self.conditioned_profile)
             if self.fission_enabled:
                 extra_options.append("fission")
             if self.live_sites:
@@ -256,7 +256,10 @@ class RaptorResultsHandler(PerftestResultsHandler):
         new_result_json.setdefault("extra_options", []).extend(
             self.build_extra_options(
                 [
-                    (self.no_conditioned_profile, "nocondprof"),
+                    (
+                        self.conditioned_profile,
+                        "condprof-%s" % self.conditioned_profile,
+                    ),
                     (self.fission_enabled, "fission"),
                     (self.webrender_enabled, "webrender"),
                 ]

testing/raptor/raptor/webextension/base.py

@@ -35,7 +35,6 @@ class WebExtension(Perftest):
         self.cpu_profiler = None
 
         super(WebExtension, self).__init__(*args, **kwargs)
-        self.using_condprof = self.config.get("using_condprof", True)
 
         # set up the results handler
         self.results_handler = RaptorResultsHandler(**self.config)

testing/raptor/test/conftest.py

@@ -23,7 +23,6 @@ def options(request):
     opts = {
         "app": "firefox",
         "binary": "path/to/dummy/browser",
-        "no_conditioned_profile": True,
         "browsertime_visualmetrics": False,
     }
 

testing/raptor/test/test_cpu.py

@@ -23,7 +23,6 @@ def test_no_device():
         "geckoview",
         "org.mozilla.org.mozilla.geckoview_example",
         cpu_test=True,
-        no_conditioned_profile=True,
     )
     raptor.device = None
     resp = cpu.start_android_cpu_profiler(raptor)