Bug 1628982 - Activate the condprofile on desktop + GV r=Bebe,perftest-reviewers,whimboo

This patches fixes several problems found on Raptor and the condprof:

Raptor:

- Make sure the conditioned profile dir is removed after
  it's been used, not before.
- Adds the --project option to raptor so we know if we're on try
  autoland or mozilla-central.
- Both Fennec and Fenix are deactivated for now
- Use the allow-downgrade flag to be flexible on build ids (the next step will be bug 1628666)

Conditioned profiles, curation of the profile prefs:
- Fully deactivates Normandy during Raptor tests (app.normandy.enabled)
- Removes any GFX blacklisting (gfx.blacklist.*)
- Removes any marionette pref
- Enforce extensions sideloading (extensions.startupScanScopes)

Differential Revision: https://phabricator.services.mozilla.com/D70518

.eslintignore

@@ -177,6 +177,7 @@ services/fxaccounts/FxAccountsPairingChannel.js
 servo/
 
 # Test files that we don't want to lint (preprocessed, minified etc)
+testing/condprofile/condprof/tests/profile
 testing/marionette/atom.js
 testing/mozbase/mozprofile/tests/files/prefs_with_comments.js
 testing/talos/talos/scripts/jszip.min.js

taskcluster/taskgraph/transforms/raptor.py

@@ -276,6 +276,8 @@ def add_extra_options(config, tests):
         if test['require-signed-extensions']:
             extra_options.append('--is-release-build')
 
+        extra_options.append("--project={}".format(config.params.get('project')))
+
         # add urlparams based on platform, test names and projects
         testurlparams_by_platform_and_project = {
             "android-hw-g5": [

testing/condprofile/condprof/client.py

@@ -12,6 +12,8 @@ import tempfile
 import shutil
 import time
 
+from mozprofile.prefs import Preferences
+
 from condprof import check_install  # NOQA
 from condprof import progress
 from condprof.util import download_file, TASK_CLUSTER, logger, ArchiveNotFound
@@ -36,6 +38,36 @@ class ProfileNotFoundError(Exception):
     pass
 
 
+def _check_profile(profile_dir):
+    """Checks for prefs we need to remove or set.
+    """
+    to_remove = ("gfx.blacklist.", "marionette.")
+
+    def _keep_pref(name, value):
+        for item in to_remove:
+            if not name.startswith(item):
+                continue
+            logger.info("Removing pref %s: %s" % (name, value))
+            return False
+        return True
+
+    def _clean_pref_file(name):
+        js_file = os.path.join(profile_dir, name)
+        prefs = Preferences.read_prefs(js_file)
+        cleaned_prefs = dict([pref for pref in prefs if _keep_pref(*pref)])
+        if name == "prefs.js":
+            # When we start Firefox, forces startupScanScopes to SCOPE_PROFILE (1)
+            # otherwise, side loading will be deactivated and the
+            # Raptor web extension won't be able to run.
+            cleaned_prefs["extensions.startupScanScopes"] = 1
+
+        with open(js_file, "w") as f:
+            Preferences.write(f, cleaned_prefs)
+
+    _clean_pref_file("prefs.js")
+    _clean_pref_file("user.js")
+
+
 def get_profile(
     target_dir,
     platform,
@@ -109,6 +141,8 @@ def get_profile(
             finally:
                 if not download_cache:
                     shutil.rmtree(download_dir)
+
+            _check_profile(target_dir)
             logger.info("Success, we have a profile to work with")
             return target_dir
         except Exception:

testing/condprofile/condprof/tests/profile/prefs.js

@@ -0,0 +1 @@
+user_pref("gfx.blacklist.direct2d", 1);

testing/condprofile/condprof/tests/profile/user.js

@@ -0,0 +1,11 @@
+
+#Prefs used for the unit test
+user_pref("focusmanager.testmode", true);
+user_pref("marionette.defaultPrefs.port", 2828);
+user_pref("marionette.port", 2828);
+user_pref("marionette.enabled", true);
+user_pref("marionette.log.level", "Trace");
+user_pref("marionette.log.truncate", false);
+user_pref("marionette.contentListener", false);
+user_pref("extensions.autoDisableScopes", 0);
+user_pref("devtools.debugger.remote-enabled", true);

testing/condprofile/condprof/tests/test_client.py

@@ -5,20 +5,33 @@ import shutil
 import responses
 import re
 import json
+import tarfile
+
+from mozprofile.prefs import Preferences
 
 from condprof.client import get_profile, ROOT_URL
 from condprof.util import _DEFAULT_SERVER
 
 
 PROFILE = re.compile(ROOT_URL + "/.*/.*tgz")
-with open(os.path.join(os.path.dirname(__file__), "profile.tgz"), "rb") as f:
-    PROFILE_DATA = f.read()
+PROFILE_FOR_TESTS = os.path.join(os.path.dirname(__file__), "profile")
 SECRETS = re.compile(_DEFAULT_SERVER + "/.*")
 SECRETS_PROXY = re.compile("http://taskcluster/secrets/.*")
 
 
 class TestClient(unittest.TestCase):
     def setUp(self):
+        self.profile_dir = tempfile.mkdtemp()
+
+        # creating profile.tgz on the fly for serving it
+        profile_tgz = os.path.join(self.profile_dir, "profile.tgz")
+        with tarfile.open(profile_tgz, "w:gz") as tar:
+            tar.add(PROFILE_FOR_TESTS, arcname=".")
+
+        # self.profile_data is the tarball we're sending back via HTTP
+        with open(profile_tgz, "rb") as f:
+            self.profile_data = f.read()
+
         self.target = tempfile.mkdtemp()
         self.download_dir = os.path.expanduser("~/.condprof-cache")
         if os.path.exists(self.download_dir):
@@ -27,8 +40,8 @@ class TestClient(unittest.TestCase):
         responses.add(
             responses.GET,
             PROFILE,
-            body=PROFILE_DATA,
-            headers={"content-length": str(len(PROFILE_DATA)), "ETag": "'12345'"},
+            body=self.profile_data,
+            headers={"content-length": str(len(self.profile_data)), "ETag": "'12345'"},
             status=200,
         )
 
@@ -36,7 +49,7 @@ class TestClient(unittest.TestCase):
             responses.HEAD,
             PROFILE,
             body="",
-            headers={"content-length": str(len(PROFILE_DATA)), "ETag": "'12345'"},
+            headers={"content-length": str(len(self.profile_data)), "ETag": "'12345'"},
             status=200,
         )
 
@@ -54,6 +67,7 @@ class TestClient(unittest.TestCase):
     def tearDown(self):
         shutil.rmtree(self.target)
         shutil.rmtree(self.download_dir)
+        shutil.rmtree(self.profile_dir)
 
     @responses.activate
     def test_cache(self):
@@ -85,6 +99,22 @@ class TestClient(unittest.TestCase):
         # even the TC secret
         self.assertEqual(len(responses.calls), 1)
 
+        prefs_js = os.path.join(self.target, "prefs.js")
+        prefs = Preferences.read_prefs(prefs_js)
+
+        # check that the gfx.blacklist prefs where cleaned out
+        for name, value in prefs:
+            self.assertFalse(name.startswith("gfx.blacklist"))
+
+        # check that we have the startupScanScopes option forced
+        prefs = dict(prefs)
+        self.assertEqual(prefs["extensions.startupScanScopes"], 1)
+
+        # make sure we don't have any marionette option set
+        user_js = os.path.join(self.target, "user.js")
+        for name, value in Preferences.read_prefs(user_js):
+            self.assertFalse(name.startswith("marionette."))
+
 
 if __name__ == "__main__":
     try:

testing/mozharness/mozharness/mozilla/testing/raptor.py

@@ -204,6 +204,13 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin):
             "help": "The number of times a cold load test is repeated (for cold load tests only, "
                     "where the browser is shutdown and restarted between test iterations)."
         }],
+        [["--project"], {
+            "action": "store",
+            "dest": "project",
+            "default": "mozilla-central",
+            "type": "str",
+            "help": "Name of the project (try, mozilla-central, etc.)"
+        }],
         [["--test-url-params"], {
             "action": "store",
             "dest": "test_url_params",

testing/profiles/raptor/user.js

@@ -1,5 +1,8 @@
 // Preferences file used by the raptor harness
 /* globals user_pref */
+// prevents normandy from running updates during the tests
+user_pref("app.normandy.enabled", false);
+
 user_pref("dom.performance.time_to_non_blank_paint.enabled", true);
 user_pref("dom.performance.time_to_contentful_paint.enabled", true);
 user_pref("dom.performance.time_to_dom_content_flushed.enabled", true);

testing/raptor/raptor/browsertime/base.py

@@ -223,7 +223,7 @@ class Browsertime(Perftest):
             ])
 
         # have browsertime use our newly-created conditioned-profile path
-        if not self.no_condprof:
+        if self.using_condprof:
             self.profile.profile = self.conditioned_profile_dir
 
         if self.config["gecko_profile"]:

testing/raptor/raptor/cmdline.py

@@ -136,7 +136,9 @@ def create_parser(mach_interface=False):
             help='How long to wait (ms) after browser start-up before starting the tests')
     add_arg('--browser-cycles', dest="browser_cycles", type=int,
             help="The number of times a cold load test is repeated (for cold load tests only, "
-            "where the browser is shutdown and restarted between test iterations)")
+            "where the browser is shutdown and restarted between test iterations)"),
+    add_arg('--project', dest='project', type=str, default='mozilla-central',
+            help="Project name (try, mozilla-central, etc.)"),
     add_arg('--test-url-params', dest='test_url_params',
             help="Parameters to add to the test_url query string")
     add_arg('--print-tests', action=_PrintTests,

testing/raptor/raptor/perftest.py

@@ -96,8 +96,10 @@ either Raptor or browsertime."""
         disable_perf_tuning=False,
         conditioned_profile_scenario='settled',
         extra_prefs={},
+        project="mozilla-central",
         **kwargs
     ):
+        self._dirs_to_remove = []
 
         # Override the magic --host HOST_IP with the value of the environment variable.
         if host == "HOST_IP":
@@ -128,19 +130,28 @@ either Raptor or browsertime."""
             "disable_perf_tuning": disable_perf_tuning,
             "conditioned_profile_scenario": conditioned_profile_scenario,
             "extra_prefs": extra_prefs,
+            "project": project
         }
 
         self.firefox_android_apps = FIREFOX_ANDROID_APPS
-        # See bugs 1582757, 1606199, and 1606767; until we support win10-aarch64,
-        # fennec_aurora, and reference browser conditioned-profile builds,
-        # fall back to mozrunner-created profiles
-        self.no_condprof = (
-            (self.config["platform"] == "win" and self.config["processor"] == "aarch64")
-            or self.config["binary"] == "org.mozilla.fennec_aurora"
-            or self.config["binary"] == "org.mozilla.reference.browser.raptor"
-            or self.config["no_conditioned_profile"]
+        # We are deactivating the conditioned profiles for:
+        # - win10-aarch64 : no support for geckodriver see 1582757
+        # - fennec_aurora: no conditioned profiles created see 1606199
+        # - reference browser: no conditioned profiles created see 1606767
+        # - crashes for fennec68 and fenix, to be fixed, see 1626729
+        self.using_condprof = not (
+             (self.config["platform"] == "win" and self.config["processor"] == "aarch64")
+             or self.config["binary"] == "org.mozilla.fennec_aurora"
+             or self.config["binary"] == "org.mozilla.reference.browser.raptor"
+             or self.config["binary"] == "org.mozilla.firefox"
+             or self.config["binary"].startswith("org.mozilla.fenix")
+             or self.config["no_conditioned_profile"]
         )
-        LOG.info("self.no_condprof is: {}".format(self.no_condprof))
+        if self.using_condprof:
+            LOG.info("Using a conditioned profile.")
+        else:
+            LOG.info("Using an empty profile.")
+        self.config["using_condprof"] = self.using_condprof
 
         # We can never use e10s on fennec
         if self.config["app"] == "fennec":
@@ -172,7 +183,7 @@ either Raptor or browsertime."""
 
         # For the post startup delay, we want to max it to 1s when using the
         # conditioned profiles.
-        if not self.no_condprof and not self.run_local:
+        if self.using_condprof and not self.run_local:
             self.post_startup_delay = min(post_startup_delay, POST_DELAY_CONDPROF)
         else:
             # if running debug-mode reduce the pause after browser startup
@@ -189,6 +200,11 @@ either Raptor or browsertime."""
         # Crashes counter
         self.crashes = 0
 
+    def _get_temp_dir(self):
+        tempdir = tempfile.mkdtemp()
+        self._dirs_to_remove.append(tempdir)
+        return tempdir
+
     @property
     def is_localhost(self):
         return self.config.get("host") in ("localhost", "127.0.0.1")
@@ -198,7 +214,7 @@ either Raptor or browsertime."""
         condprofile client API; returns a self.conditioned_profile_dir"""
 
         # create a temp file to help ensure uniqueness
-        temp_download_dir = tempfile.mkdtemp()
+        temp_download_dir = self._get_temp_dir()
         LOG.info(
             "Making temp_download_dir from inside get_conditioned_profile {}".format(
                 temp_download_dir
@@ -216,20 +232,33 @@ either Raptor or browsertime."""
             platform = get_current_platform()
 
         LOG.info("Platform used: %s" % platform)
+
+        # when running under mozharness, the --project value
+        # is set to match the project (try, mozilla-central, etc.)
+        # By default it's mozilla-central, even on local runs.
+        # We use it to prioritize conditioned profiles indexed
+        # into the same project when it runs on the CI
+        repo = self.config["project"]
+
+        # we fall back to mozilla-central in all cases. If it
+        # was already mozilla-central, we fall back to try
+        alternate_repo = "mozilla-central" if repo != "mozilla-central" else "try"
+        LOG.info("Getting profile from project %s" % repo)
+
         profile_scenario = self.config.get("conditioned_profile_scenario", "settled")
         try:
             cond_prof_target_dir = get_profile(
                 temp_download_dir,
                 platform,
-                profile_scenario
+                profile_scenario,
+                repo=repo
             )
         except ProfileNotFoundError:
-            # If we can't find the profile on mozilla-central, we look on try
             cond_prof_target_dir = get_profile(
                 temp_download_dir,
                 platform,
                 profile_scenario,
-                repo="try"
+                repo=alternate_repo
             )
         except Exception:
             # any other error is a showstopper
@@ -258,12 +287,10 @@ either Raptor or browsertime."""
                 self.conditioned_profile_dir
             )
         )
-        shutil.rmtree(temp_download_dir)
-
         return self.conditioned_profile_dir
 
     def build_browser_profile(self):
-        if self.no_condprof or self.config['app'] in ['chrome', 'chromium', 'chrome-m']:
+        if not self.using_condprof or self.config['app'] in ['chrome', 'chromium', 'chrome-m']:
             self.profile = create_profile(self.profile_class)
         else:
             self.get_conditioned_profile()
@@ -388,9 +415,13 @@ either Raptor or browsertime."""
     def check_for_crashes(self):
         pass
 
-    @abstractmethod
     def clean_up(self):
-        pass
+        for dir_to_rm in self._dirs_to_remove:
+            if not os.path.exists(dir_to_rm):
+                continue
+            LOG.info("Removing temporary directory: {}".format(dir_to_rm))
+            shutil.rmtree(dir_to_rm, ignore_errors=True)
+        self._dirs_to_remove = []
 
     def get_page_timeout_list(self):
         return self.results_handler.page_timeout_list

testing/raptor/raptor/raptor.py

@@ -134,6 +134,7 @@ def main(args=sys.argv[1:]):
             no_conditioned_profile=args.no_conditioned_profile,
             disable_perf_tuning=args.disable_perf_tuning,
             conditioned_profile_scenario=args.conditioned_profile_scenario,
+            project=args.project
         )
     except Exception:
         traceback.print_exc()

testing/raptor/raptor/webextension/android.py

@@ -81,6 +81,7 @@ class WebExtensionAndroid(PerftestAndroid, WebExtension):
             args=[
                 "--profile",
                 self.remote_profile,
+                "--allow-downgrade",
                 "use_multiprocess",
                 self.config["e10s"],
             ],
@@ -127,6 +128,7 @@ class WebExtensionAndroid(PerftestAndroid, WebExtension):
 
         extra_args = [
             "-profile", self.remote_profile,
+            "--allow-downgrade",
             "--es", "env0",
             "LOG_VERBOSE=1",
             "--es", "env1",

testing/raptor/raptor/webextension/base.py

@@ -35,6 +35,7 @@ class WebExtension(Perftest):
         self.cpu_profiler = None
 
         super(WebExtension, self).__init__(*args, **kwargs)
+        self.using_condprof = self.config.get("using_condprof", True)
 
         # set up the results handler
         self.results_handler = RaptorResultsHandler(

testing/raptor/raptor/webextension/desktop.py

@@ -28,10 +28,12 @@ class WebExtensionDesktop(PerftestDesktop, WebExtension):
         LOG.info("creating browser runner using mozrunner")
         self.output_handler = OutputHandler()
         process_args = {"processOutputLine": [self.output_handler]}
+        firefox_args = ["--allow-downgrade"]
         runner_cls = runners[self.config["app"]]
         self.runner = runner_cls(
             self.config["binary"],
             profile=self.profile,
+            cmdargs=firefox_args,
             process_args=process_args,
             symbols_path=self.config["symbols_path"],
         )