Bug 1743578: Activate Mach site in mozperftest/runner.py r=sparky

When activating a command virtualenv, it rightfully complains if it
isn't already running within an active Mach site - the abstraction has
restrictions to act as guardrails around the risky business that is
`sys.path` patching.

Replace `mozperftest/runner.py`'s ad-hoc `sys.path` initializing with
`MachSiteManager`'s `activate()` instead.

Removes the calls to `_setup_path()` where the `sys.path` should
already be set-up.

Differential Revision: https://phabricator.services.mozilla.com/D132503

python/mach/mach/site.py

@@ -188,7 +188,7 @@ class MachSiteManager:
     def __init__(
         self,
         topsrcdir: str,
-        state_dir: str,
+        state_dir: Optional[str],
         requirements: MachEnvRequirements,
         external_python: "ExternalPythonSite",
         site_packages_source: SitePackagesSource,
@@ -226,7 +226,7 @@ class MachSiteManager:
             state_dir: The path to the state_dir, generally ~/.mozbuild
         """
 
-        requirements = _requirements(topsrcdir, "mach")
+        requirements = resolve_requirements(topsrcdir, "mach")
         # Mach needs to operate in environments in which no pip packages are installed
         # yet, and the system isn't guaranteed to have the packages we need. For example,
         # "./mach bootstrap" can't have any dependencies.
@@ -449,7 +449,7 @@ class CommandSiteManager:
             active_metadata
         ), "A Mach-managed site must be active before doing work with command sites"
 
-        requirements = _requirements(topsrcdir, site_name)
+        requirements = resolve_requirements(topsrcdir, site_name)
         if not _system_python_env_variable_present() or site_name != "build":
             source = SitePackagesSource.VENV
         elif not active_metadata.external_python.has_pip():
@@ -580,7 +580,9 @@ class CommandSiteManager:
         """
 
         # Prioritize Mach's vendored and first-party modules first.
-        lines = _requirements(self._topsrcdir, "mach").pths_as_absolute(self._topsrcdir)
+        lines = resolve_requirements(self._topsrcdir, "mach").pths_as_absolute(
+            self._topsrcdir
+        )
         mach_site_packages_source = self._mach_site_packages_source
         if mach_site_packages_source == SitePackagesSource.SYSTEM:
             # When Mach is using the system environment, add it next.
@@ -810,7 +812,7 @@ class ExternalPythonSite:
 
 
 @functools.lru_cache(maxsize=None)
-def _requirements(topsrcdir, virtualenv_name):
+def resolve_requirements(topsrcdir, virtualenv_name):
     manifest_path = os.path.join(
         topsrcdir, "build", f"{virtualenv_name}_virtualenv_packages.txt"
     )

python/mozperftest/mozperftest/mach_commands.py

@@ -191,7 +191,6 @@ def run_tests(command_context, **kwargs):
 
 def _run_tests(command_context, **kwargs):
     from pathlib import Path
-    from mozperftest.runner import _setup_path
     from mozperftest.utils import (
         install_package,
         ON_TRY,
@@ -203,8 +202,6 @@ def _run_tests(command_context, **kwargs):
     skip_linters = kwargs.get("skip_linters", False)
     verbose = kwargs.get("verbose", False)
 
-    # include in sys.path all deps
-    _setup_path()
     try:
         import coverage  # noqa
     except ImportError:

python/mozperftest/mozperftest/runner.py

@@ -35,47 +35,6 @@ TASKCLUSTER = "TASK_ID" in os.environ.keys()
 RUNNING_TESTS = "RUNNING_TESTS" in os.environ.keys()
 HERE = Path(__file__).parent
 SRC_ROOT = Path(HERE, "..", "..", "..").resolve()
-SEARCH_PATHS = [
-    "python/mach",
-    "python/mozboot",
-    "python/mozbuild",
-    "python/mozperftest",
-    "python/mozterm",
-    "python/mozversioncontrol",
-    "testing/condprofile",
-    "testing/mozbase/mozdevice",
-    "testing/mozbase/mozfile",
-    "testing/mozbase/mozinfo",
-    "testing/mozbase/mozlog",
-    "testing/mozbase/mozprocess",
-    "testing/mozbase/mozprofile",
-    "testing/mozbase/mozproxy",
-    "third_party/python/attrs",
-    "third_party/python/blessings",
-    "third_party/python/certifi",
-    "third_party/python/chardet",
-    "third_party/python/distro",
-    "third_party/python/dlmanager",
-    "third_party/python/esprima",
-    "third_party/python/idna",
-    "third_party/python/importlib_metadata",
-    "third_party/python/jsmin",
-    "third_party/python/jsonschema",
-    "third_party/python/packaging",
-    "third_party/python/pyparsing",
-    "third_party/python/pyrsistent",
-    "third_party/python/PyYAML/lib3",
-    "third_party/python/redo",
-    "third_party/python/requests",
-    "third_party/python/six",
-    "third_party/python/typing_extensions",
-    "third_party/python/urllib3",
-    "third_party/python/zipp",
-]
-
-
-if TASKCLUSTER:
-    SEARCH_PATHS.append("xpcshell")
 
 
 # XXX need to make that for all systems flavors
@@ -83,16 +42,40 @@ if "SHELL" not in os.environ:
     os.environ["SHELL"] = "/bin/bash"
 
 
-def _setup_path():
+def _activate_mach_virtualenv():
     """Adds all available dependencies in the path.
 
     This is done so the runner can be used with no prior
     install in all execution environments.
     """
-    for path in SEARCH_PATHS:
-        path = Path(SRC_ROOT, path).resolve()
-        if path.exists():
-            sys.path.insert(0, str(path))
+
+    # We need the "mach" module to access the logic to parse virtualenv
+    # requirements. Since that depends on "packaging" (and, transitively,
+    # "pyparsing"), we add those to the path too.
+    sys.path[0:0] = [
+        os.path.join(SRC_ROOT, module)
+        for module in (
+            os.path.join("python", "mach"),
+            os.path.join("third_party", "python", "packaging"),
+            os.path.join("third_party", "python", "pyparsing"),
+        )
+    ]
+
+    from mach.site import (
+        resolve_requirements,
+        MachSiteManager,
+        ExternalPythonSite,
+        SitePackagesSource,
+    )
+
+    mach_site = MachSiteManager(
+        str(SRC_ROOT),
+        None,
+        resolve_requirements(str(SRC_ROOT), "mach"),
+        ExternalPythonSite(sys.executable),
+        SitePackagesSource.NONE,
+    )
+    mach_site.activate()
 
 
 def run_tests(mach_cmd, kwargs, client_args):
@@ -102,7 +85,6 @@ def run_tests(mach_cmd, kwargs, client_args):
     `PERFTEST_OPTIONS` environment variable that contains all options passed by
     the user via a ./mach perftest --push-to-try call.
     """
-    _setup_path()
     on_try = kwargs.pop("on_try", False)
 
     # trying to get the arguments from the task params
@@ -179,7 +161,7 @@ def run_tests(mach_cmd, kwargs, client_args):
 
 def main(argv=sys.argv[1:]):
     """Used when the runner is directly called from the shell"""
-    _setup_path()
+    _activate_mach_virtualenv()
 
     from mozbuild.mozconfig import MozconfigLoader
     from mozbuild.base import MachCommandBase, MozbuildObject