Bug 1628278 - added conditioned profile support r=acreskey

Adds conditioned profiles and make the Profile
layer a system layer instead of a test layer

Differential Revision: https://phabricator.services.mozilla.com/D83612

build/sparse-profiles/perftest

@@ -3,6 +3,5 @@
 [include]
 path:tools/lint/eslint/
 path:testing/performance
+path:testing/condprofile
 glob:**/perftest_*.js
-
-

python/mozbuild/mozbuild/action/test_archive.py

@@ -471,6 +471,10 @@ ARCHIVE_FILES = {
             'source': buildconfig.topsrcdir,
             'pattern': 'testing/mozbase/**',
         },
+        {
+            'source': buildconfig.topsrcdir,
+            'pattern': 'testing/condprofile/**',
+        },
         {
             'source': buildconfig.topsrcdir,
             'pattern': 'third_party/python/**',

python/mozperftest/mozperftest/mach_commands.py

@@ -114,6 +114,9 @@ class PerftestTests(MachCommandBase):
         default=False,
         help="Skip flake8 and black",
     )
+    @CommandArgument(
+        "-v", "--verbose", action="store_true", default=False, help="Verbose mode",
+    )
     def run_tests(self, **kwargs):
         MachCommandBase._activate_virtualenv(self)
 
@@ -180,6 +183,10 @@ class PerftestTests(MachCommandBase):
 
         import pytest
 
+        options = "-xs"
+        if kwargs.get("verbose"):
+            options += "v"
+
         with temporary_env(COVERAGE_RCFILE=str(here / ".coveragerc")):
             if run_coverage_check:
                 assert self._run_python_script(
@@ -188,7 +195,7 @@ class PerftestTests(MachCommandBase):
             args = [
                 "run",
                 pytest.__file__,
-                "-xs",
+                options,
                 tests,
             ]
             assert self._run_python_script("coverage", *args, label="running tests")

python/mozperftest/mozperftest/runner.py

@@ -39,6 +39,7 @@ SEARCH_PATHS = [
     "python/mozperftest",
     "python/mozterm",
     "python/mozversioncontrol",
+    "testing/condprofile",
     "testing/mozbase/mozdevice",
     "testing/mozbase/mozfile",
     "testing/mozbase/mozinfo",

python/mozperftest/mozperftest/system/__init__.py

@@ -4,15 +4,16 @@
 from mozperftest.layers import Layers
 from mozperftest.system.proxy import ProxyRunner
 from mozperftest.system.android import AndroidDevice
+from mozperftest.system.profile import Profile
 
 
 def get_layers():
-    return (ProxyRunner, AndroidDevice)
+    return Profile, ProxyRunner, AndroidDevice
 
 
 def pick_system(env, flavor, mach_cmd):
     if flavor == "desktop-browser":
-        return Layers(env, mach_cmd, (ProxyRunner,))
+        return Layers(env, mach_cmd, (Profile, ProxyRunner,))
     if flavor == "mobile-browser":
-        return Layers(env, mach_cmd, (ProxyRunner, AndroidDevice))
+        return Layers(env, mach_cmd, (Profile, ProxyRunner, AndroidDevice))
     raise NotImplementedError(flavor)

python/mozperftest/mozperftest/system/profile.py

@@ -3,7 +3,11 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import os
 import shutil
+import tempfile
+from pathlib import Path
 
+from condprof.client import get_profile, ProfileNotFoundError
+from condprof.util import get_current_platform
 from mozperftest.layers import Layer
 from mozprofile import create_profile
 from mozprofile.prefs import Preferences
@@ -18,6 +22,27 @@ class Profile(Layer):
     arguments = {
         "directory": {"type": str, "default": None, "help": "Profile to use"},
         "user-js": {"type": str, "default": None, "help": "Custom user.js"},
+        "conditioned": {
+            "action": "store_true",
+            "default": False,
+            "help": "Use a conditioned profile.",
+        },
+        "conditioned-scenario": {
+            "type": str,
+            "default": "settled",
+            "help": "Conditioned scenario to use",
+        },
+        "conditioned-platform": {
+            "type": str,
+            "default": None,
+            "help": "Conditioned platform to use (use local by default)",
+        },
+        "conditioned-project": {
+            "type": str,
+            "default": "mozilla-central",
+            "help": "Conditioned project",
+            "choices": ["try", "mozilla-central"],
+        },
     }
 
     def __init__(self, env, mach_cmd):
@@ -30,12 +55,42 @@ class Profile(Layer):
     def _cleanup(self):
         pass
 
+    def _get_conditioned_profile(self):
+        platform = self.get_arg("conditioned-platform")
+        if platform is None:
+            platform = get_current_platform()
+        scenario = self.get_arg("conditioned-scenario")
+        project = self.get_arg("conditioned-project")
+        alternate_project = "mozilla-central" if project != "mozilla-central" else "try"
+
+        temp_dir = tempfile.mkdtemp()
+        try:
+            condprof = get_profile(temp_dir, platform, scenario, repo=project)
+        except ProfileNotFoundError:
+            condprof = get_profile(temp_dir, platform, scenario, repo=alternate_project)
+        except Exception:
+            raise
+
+        # now get the full directory path to our fetched conditioned profile
+        condprof = Path(temp_dir, condprof)
+        if not condprof.exists():
+            raise OSError(str(condprof))
+
+        return condprof
+
     def run(self, metadata):
+        # using a conditioned profile
+        if self.get_arg("conditioned"):
+            profile_dir = self._get_conditioned_profile()
+            self.set_arg("profile-directory", str(profile_dir))
+            self._created_dirs.append(str(profile_dir))
+            return metadata
+
         if self.get_arg("directory") is not None:
             # no need to create one or load a conditioned one
             return
 
-        # XXX we'll use conditioned profiles later
+        # fresh profile
         profile = create_profile(app="firefox")
 
         # mozprofile.Profile.__del__ silently deletes the profile

python/mozperftest/mozperftest/test/__init__.py

@@ -2,19 +2,18 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 from mozperftest.test.browsertime import BrowsertimeRunner
-from mozperftest.test.profile import Profile
 from mozperftest.test.androidlog import AndroidLog
 from mozperftest.layers import Layers
 
 
 def get_layers():
-    return (Profile, BrowsertimeRunner, AndroidLog)
+    return (BrowsertimeRunner, AndroidLog)
 
 
 def pick_test(env, flavor, mach_cmd):
     if flavor == "desktop-browser":
-        return Layers(env, mach_cmd, (Profile, BrowsertimeRunner))
+        return Layers(env, mach_cmd, (BrowsertimeRunner,))
     if flavor == "mobile-browser":
-        return Layers(env, mach_cmd, (Profile, BrowsertimeRunner, AndroidLog))
+        return Layers(env, mach_cmd, (BrowsertimeRunner, AndroidLog))
 
     raise NotImplementedError(flavor)

python/mozperftest/mozperftest/test/browsertime/runner.py

@@ -29,6 +29,7 @@ def matches(args, *flags):
 
     Maybe with an argument.
     """
+
     for flag in flags:
         if flag in args or any(arg.startswith(flag + "=") for arg in args):
             return True

python/mozperftest/mozperftest/tests/test_android.py

@@ -199,7 +199,7 @@ def test_android_log_cat(device):
 
         mach_cmd, metadata, env = get_running_env(**args)
         system = env.layers[SYSTEM]
-        andro = system.layers[0]
+        andro = system.layers[1]
 
         with system as layer, silence(system):
             andro.device = device

python/mozperftest/mozperftest/tests/test_androidlog.py

@@ -64,7 +64,7 @@ def test_android_log(*mocked):
         env.set_arg("tests", [EXAMPLE_TEST])
 
         with env.layers[SYSTEM] as sys, env.layers[TEST] as andro:
-            metadata = sys(andro(metadata))
+            metadata = andro(sys(metadata))
 
         # we want to drop the first result
         metadata._results = metadata._results[1:]

python/mozperftest/mozperftest/tests/test_browsertime.py

@@ -9,7 +9,7 @@ import random
 import pytest
 
 from mozperftest.tests.support import get_running_env, EXAMPLE_TEST
-from mozperftest.environment import TEST
+from mozperftest.environment import TEST, SYSTEM
 from mozperftest.test.browsertime import add_options
 from mozperftest.test.browsertime.runner import (
     NodeException,
@@ -44,12 +44,13 @@ def test_browser(*mocked):
         browsertime_extra_options="one=1,two=2",
     )
 
+    sys = env.layers[SYSTEM]
     browser = env.layers[TEST]
     env.set_arg("tests", [EXAMPLE_TEST])
 
     try:
-        with browser as b, silence():
-            b(metadata)
+        with sys as s, browser as b, silence():
+            b(s(metadata))
     finally:
         shutil.rmtree(mach_cmd._mach_context.state_dir)
     assert mach_cmd.run_process.call_count == 1
@@ -91,9 +92,10 @@ def test_browser_failed(*mocked):
     mach_cmd.run_process.return_value = 1
     browser = env.layers[TEST]
     env.set_arg("tests", [EXAMPLE_TEST])
+    sys = env.layers[SYSTEM]
 
-    with browser as b, silence(), pytest.raises(NodeException):
-        b(metadata)
+    with sys as s, browser as b, silence(), pytest.raises(NodeException):
+        b(s(metadata))
 
 
 @mock.patch("mozperftest.test.browsertime.runner.install_package")
@@ -111,9 +113,10 @@ def test_browser_desktop(*mocked):
     )
     browser = env.layers[TEST]
     env.set_arg("tests", [EXAMPLE_TEST])
+    sys = env.layers[SYSTEM]
 
     try:
-        with browser as b, silence():
+        with sys as s, browser as b, silence():
             # just checking that the setup_helper property gets
             # correctly initialized
             browsertime = browser.layers[-1]
@@ -121,7 +124,7 @@ def test_browser_desktop(*mocked):
             helper = browsertime.setup_helper
             assert browsertime.setup_helper is helper
 
-            b(metadata)
+            b(s(metadata))
     finally:
         shutil.rmtree(mach_cmd._mach_context.state_dir)
 
@@ -153,10 +156,11 @@ def test_install_url(*mocked):
     mach, metadata, env = get_running_env(browsertime_install_url=url)
     browser = env.layers[TEST]
     env.set_arg("tests", [EXAMPLE_TEST])
+    sys = env.layers[SYSTEM]
 
     try:
-        with temporary_env(MOZ_AUTOMATION="1"), browser as b, silence():
-            b(metadata)
+        with sys as s, temporary_env(MOZ_AUTOMATION="1"), browser as b, silence():
+            b(s(metadata))
     finally:
         shutil.rmtree(mach._mach_context.state_dir)
 
@@ -176,11 +180,12 @@ def test_install_url_bad(*mocked):
     mach, metadata, env = get_running_env(browsertime_install_url="meh")
     browser = env.layers[TEST]
     env.set_arg("tests", [EXAMPLE_TEST])
+    sys = env.layers[SYSTEM]
 
     with pytest.raises(ValueError):
         try:
-            with browser as b, silence():
-                b(metadata)
+            with sys as s, browser as b, silence():
+                b(s(metadata))
         finally:
             shutil.rmtree(mach._mach_context.state_dir)
 

python/mozperftest/mozperftest/tests/test_profile.py

@@ -4,8 +4,11 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 import os
 import mozunit
+from unittest import mock
+import tempfile
+
 from mozperftest.tests.support import get_running_env
-from mozperftest.test.profile import Profile
+from mozperftest.system.profile import Profile, ProfileNotFoundError
 
 
 def test_profile():
@@ -19,5 +22,30 @@ def test_profile():
     assert not os.path.exists(profile_dir)
 
 
+CALLS = [0]
+
+
+def _return_profile(*args, **kw):
+    if CALLS[0] == 0:
+        CALLS[0] = 1
+        raise ProfileNotFoundError()
+
+    tempdir = tempfile.mkdtemp()
+
+    return tempdir
+
+
+@mock.patch("mozperftest.system.profile.get_profile", new=_return_profile)
+def test_conditionedprofile():
+    mach_cmd, metadata, env = get_running_env(profile_conditioned=True)
+
+    with Profile(env, mach_cmd) as profile:
+        profile(metadata)
+        profile_dir = env.get_arg("profile-directory")
+        assert os.path.exists(profile_dir)
+
+    assert not os.path.exists(profile_dir)
+
+
 if __name__ == "__main__":
     mozunit.main()