Bug 1537944. Add conditioned-profile support for desktop Firefox in Raptor. r=perftest-reviewers,tarek,sparky, ?tarek

Differential Revision: https://phabricator.services.mozilla.com/D49970

--HG--
extra : moz-landing-system : lando

taskcluster/ci/test/raptor.yml

@@ -47,6 +47,7 @@ raptor-tp6-1-firefox:
     variants: ["fission"]
     try-name: raptor-tp6-1-firefox
     treeherder-symbol: Rap(tp6-1)
+    condprof: True
     mozharness:
         extra-options:
             - --test=raptor-tp6-1
@@ -688,6 +689,7 @@ raptor-tp6-1-firefox-cold:
     try-name: raptor-tp6-1-firefox-cold
     treeherder-symbol: Rap(tp6-c-1)
     tier: 2
+    condprof: True
     mozharness:
         extra-options:
             - --test=raptor-tp6-1-cold

taskcluster/taskgraph/transforms/raptor.py

@@ -40,6 +40,10 @@ raptor_description_schema = Schema({
         'test-platform', 'app',
         Any('cold', 'warm', 'both'),
     ),
+    Optional('condprof'): optionally_keyed_by(
+        'app',
+        bool,
+    ),
     # Configs defined in the 'test_description_schema'.
     Optional('max-run-time'): optionally_keyed_by(
         'app',
@@ -121,6 +125,7 @@ def split_apps(config, tests):
 @transforms.add
 def handle_keyed_by_app(config, tests):
     fields = [
+        'condprof',
         'variants',
         'limit-platforms',
         'activity',
@@ -163,6 +168,31 @@ def split_pageload(config, tests):
         yield test
 
 
+@transforms.add
+def build_condprof_tests(config, tests):
+    for test in tests:
+        if not test.pop('condprof', False):
+            yield test
+            continue
+
+        # Make condprof test
+        condprof_test = deepcopy(test)
+        yield test
+
+        extra_options = condprof_test.setdefault('mozharness', {}).setdefault('extra-options', [])
+        extra_options.append('--with-conditioned-profile')
+
+        group, symbol = split_symbol(condprof_test['treeherder-symbol'])
+        symbol += '-condprof'
+
+        condprof_test['description'] += " with condprof"
+        condprof_test['try-name'] += '-condprof'
+        condprof_test['test-name'] += '-condprof'
+        condprof_test['treeherder-symbol'] = join_symbol(group, symbol)
+
+        yield condprof_test
+
+
 @transforms.add
 def add_extra_options(config, tests):
     for test in tests:

testing/condprofile/condprof/util.py

@@ -75,11 +75,13 @@ def get_logger():
 
     if sys.version_info.major == 3:
         # plugging the logger into arsenic
-        from arsenic import connection
-        from structlog import wrap_logger
-
-        logger = wrap_logger(NullLogger(), processors=[])
-        connection.log = logger
+        try:
+            from arsenic import connection
+            from structlog import wrap_logger
+            logger = wrap_logger(NullLogger(), processors=[])
+            connection.log = logger
+        except ImportError:
+            logger = NullLogger()
     else:
         # on python 2, just using the plain logger
         logger = NullLogger()

testing/mozharness/mozharness/mozilla/testing/raptor.py

@@ -133,6 +133,12 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin):
             "default": False,
             "help": "Enable the WebRender compositor in Gecko.",
         }],
+        [["--with-conditioned-profile"], {
+            "action": "store_true",
+            "dest": "with_conditioned_profile",
+            "default": False,
+            "help": "Run using the conditioned profile.",
+        }],
         [["--geckoProfile"], {
             "dest": "gecko_profile",
             "action": "store_true",
@@ -490,6 +496,8 @@ class Raptor(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin):
             options.extend(['--cpu-test'])
         if self.config.get('enable_webrender', False):
             options.extend(['--enable-webrender'])
+        if self.config.get('with_conditioned_profile', False):
+            options.extend(['--with-conditioned-profile'])
 
         for (arg,), details in Raptor.browsertime_options:
             # Allow overriding defaults on the `./mach raptor-test ...` command-line

testing/raptor/raptor/cmdline.py

@@ -140,6 +140,9 @@ def create_parser(mach_interface=False):
             help="Run without multiple processes (e10s).")
     add_arg('--enable-webrender', dest="enable_webrender", action="store_true", default=False,
             help="Enable the WebRender compositor in Gecko.")
+    add_arg('--with-conditioned-profile', dest="with_conditioned_profile", action="store_true",
+            default=False,
+            help="Run Raptor tests with a conditioned profile.")
     if not mach_interface:
         add_arg('--run-local', dest="run_local", default=False, action="store_true",
                 help="Flag which indicates if Raptor is running locally or in production")

testing/raptor/raptor/raptor.py

@@ -26,6 +26,8 @@ import mozinfo
 import mozprocess
 import mozproxy.utils as mpu
 import mozversion
+from condprof.client import get_profile
+from condprof.util import get_current_platform
 from logger.logger import RaptorLogger
 from mozdevice import ADBDevice
 from mozlog import commandline
@@ -103,7 +105,7 @@ either Raptor or browsertime."""
                  symbols_path=None, host=None, power_test=False, cpu_test=False, memory_test=False,
                  is_release_build=False, debug_mode=False, post_startup_delay=None,
                  interrupt_handler=None, e10s=True, enable_webrender=False,
-                 results_handler_class=RaptorResultsHandler,
+                 results_handler_class=RaptorResultsHandler, with_conditioned_profile=False,
                  **kwargs):
 
         # Override the magic --host HOST_IP with the value of the environment variable.
@@ -129,6 +131,7 @@ either Raptor or browsertime."""
             'enable_control_server_wait': memory_test or cpu_test,
             'e10s': e10s,
             'enable_webrender': enable_webrender,
+            'with_conditioned_profile': with_conditioned_profile,
         }
         # We can never use e10s on fennec
         if self.config['app'] == 'fennec':
@@ -146,6 +149,7 @@ either Raptor or browsertime."""
         self.post_startup_delay = post_startup_delay
         self.device = None
         self.profile_class = profile_class or app
+        self.conditioned_profile_dir = None
         self.firefox_android_apps = FIREFOX_ANDROID_APPS
         self.interrupt_handler = interrupt_handler
         self.results_handler = results_handler_class(**self.config)
@@ -168,8 +172,41 @@ either Raptor or browsertime."""
 
         self.build_browser_profile()
 
+    def get_conditioned_profile(self):
+        """Downloads a platform-specific conditioned profile, using the
+        condprofile client API; returns a self.conditioned_profile_dir"""
+
+        # create a temp file to help ensure uniqueness
+        temp_download_dir = tempfile.mkdtemp()
+        LOG.info("Making temp_download_dir from inside get_conditioned_profile {}"
+                 .format(temp_download_dir))
+        platform = get_current_platform()
+        cond_prof_target_dir = get_profile(temp_download_dir, platform, "cold")
+        LOG.info("temp_download_dir is: {}".format(temp_download_dir))
+        LOG.info("cond_prof_target_dir is: {}".format(cond_prof_target_dir))
+
+        self.conditioned_profile_dir = os.path.join(temp_download_dir, cond_prof_target_dir)
+        if not os.path.exists(cond_prof_target_dir):
+            LOG.critical("Can't find target_dir {}, from get_profile()"
+                         "temp_download_dir {}, platform {}, cold"
+                         .format(cond_prof_target_dir, temp_download_dir, platform))
+            raise OSError
+
+        LOG.info("self.conditioned_profile_dir is now set: {}"
+                 .format(self.conditioned_profile_dir))
+        shutil.rmtree(temp_download_dir)
+
+        return self.conditioned_profile_dir
+
     def build_browser_profile(self):
-        self.profile = create_profile(self.profile_class)
+        # if --with-conditioned-profile was passed in via the commandline,
+        # we need to fetch and use a conditioned profile
+        if self.config['with_conditioned_profile']:
+            self.get_conditioned_profile()
+            self.profile = create_profile(self.profile_class, profile=self.conditioned_profile_dir)
+        else:
+            # have mozprofile create a new profile for us
+            self.profile = create_profile(self.profile_class)
 
         # Merge extra profile data from testing/profiles
         with open(os.path.join(self.profile_data_dir, 'profiles.json'), 'r') as fh:
@@ -811,6 +848,7 @@ class Raptor(Perftest):
             power_test=self.config.get('power_test'),
             cpu_test=self.config.get('cpu_test'),
             memory_test=self.config.get('memory_test'),
+            with_conditioned_profile=self.config['with_conditioned_profile']
         )
         browser_name, browser_version = self.get_browser_meta()
         self.results_handler.add_browser_meta(self.config['app'], browser_version)
@@ -1727,6 +1765,7 @@ def main(args=sys.argv[1:]):
                           intent=args.intent,
                           interrupt_handler=SignalHandler(),
                           enable_webrender=args.enable_webrender,
+                          with_conditioned_profile=args.with_conditioned_profile,
                           )
 
     success = raptor.run_tests(raptor_test_list, raptor_test_names)

testing/raptor/raptor/results.py

@@ -23,7 +23,8 @@ class PerftestResultsHandler(object):
     __metaclass__ = ABCMeta
 
     def __init__(self, gecko_profile=False, power_test=False,
-                 cpu_test=False, memory_test=False, app=None, **kwargs):
+                 cpu_test=False, memory_test=False, app=None, with_conditioned_profile=False,
+                 **kwargs):
         self.gecko_profile = gecko_profile
         self.power_test = power_test
         self.cpu_test = cpu_test
@@ -35,6 +36,7 @@ class PerftestResultsHandler(object):
         self.supporting_data = None
         self.browser_version = None
         self.browser_name = None
+        self.with_conditioned_profile = with_conditioned_profile
 
     @abstractmethod
     def add(self, new_result_json):
@@ -173,6 +175,8 @@ class RaptorResultsHandler(PerftestResultsHandler):
         # add to results
         LOG.info("received results in RaptorResultsHandler.add")
         new_result = RaptorTestResult(new_result_json)
+        if self.with_conditioned_profile:
+            new_result.extra_options.append('condprof')
         self.results.append(new_result)
 
     def summarize_and_output(self, test_config, tests, test_names):

testing/raptor/requirements.txt

@@ -8,3 +8,4 @@ mozproxy >= 1.0
 pyyaml ~= 3.1
 mozpower >= 1.0.0
 mozversion >= 1.0
+conditioned-profile >= 0.1