Bug 1659539 - [python-test] Remove '--python' and stop using pipenv to manage virtualenvs r=ahal

Pipenv is heavy weight and overkill for the purposes it is being used. We'd like to remove it from the tree and |mach python-test| was one of the last remanining use cases.

Remove the `--python` command-line argument as a result. Users who wish to run unit tests with Python 2 can do `MACH_PY2=1 ./mach python-test ...` or `python2 ./mach python-test ...`.

Also update a few unit tests that would break otherwise in the presence of this change.

There were a couple lines in the `setup.py` for `mozlog` that were problematic for tests and was resulting in errors due to the `mozlog` plugin being loaded by `pytest` more than once. We just delete those lines and bump up the major version number of the package to fix it.

Differential Revision: https://phabricator.services.mozilla.com/D88296

python/mach_commands.py

@@ -27,7 +27,6 @@ from manifestparser import filters as mpf
 from mozbuild.base import (
     MachCommandBase,
 )
-from mozbuild.virtualenv import VirtualenvManager
 
 from mach.decorators import (
     CommandArgument,
@@ -92,17 +91,12 @@ class MachCommands(MachCommandBase):
                                 python_unbuffered=False,  # Leave input buffered.
                                 append_env=append_env)
 
-    @Command('python-test', category='testing',
-             description='Run Python unit tests with an appropriate test runner.')
+    @Command('python-test', category='testing', virtualenv_name='python-test',
+             description='Run Python unit tests with pytest.')
     @CommandArgument('-v', '--verbose',
                      default=False,
                      action='store_true',
                      help='Verbose output.')
-    @CommandArgument('--python',
-                     default='3',
-                     help='Version of Python for Pipenv to use. When given a '
-                          'Python version, Pipenv will automatically scan your '
-                          'system for a Python that matches that given version.')
     @CommandArgument('-j', '--jobs',
                      default=None,
                      type=int,
@@ -142,12 +136,11 @@ class MachCommands(MachCommandBase):
                          subsuite=None,
                          verbose=False,
                          jobs=None,
-                         python=None,
                          exitfirst=False,
                          extra=None,
                          **kwargs):
-        self._activate_test_virtualenvs(python)
 
+        self.activate_virtualenv()
         if test_objects is None:
             from moztest.resolve import TestResolver
             resolver = self._spawn(TestResolver)
@@ -238,38 +231,6 @@ class MachCommands(MachCommandBase):
                  'Return code from mach python-test: {return_code}')
         return return_code
 
-    def _activate_test_virtualenvs(self, python):
-        """Make sure the test suite virtualenvs are set up and activated.
-
-        Args:
-            python: Optional python version string we want to run the suite with.
-                See the `--python` argument to the `mach python-test` command.
-        """
-        from mozbuild.pythonutil import find_python3_executable
-
-        default_manager = self.virtualenv_manager
-
-        # Grab the default virtualenv properties before we activate other virtualenvs.
-        python = python or default_manager.python_path
-        py3_root = default_manager.virtualenv_root + '_py3'
-
-        self.activate_pipenv(os.path.dirname(default_manager.virtualenv_root),
-                             pipfile=None, populate=True, python=python)
-
-        # If the current process is running under Python 2, the Python 3
-        # virtualenv may not be set up yet. To avoid problems in tests
-        # that implicitly depend on the Python 3 virtualenv we ensure the Python 3
-        # virtualenv is up to date before the tests start.
-        if not six.PY3:
-            python3, version = find_python3_executable(min_version='3.5.0')
-            py3_manager = VirtualenvManager(
-                default_manager.topsrcdir,
-                py3_root,
-                default_manager.log_handle,
-                default_manager.manifest_path,
-            )
-            py3_manager.ensure(python3)
-
     def _run_python_test(self, test):
         from mozprocess import ProcessHandler
 

python/mozbuild/mozbuild/test/configure/common.py

@@ -284,7 +284,8 @@ class BaseConfigureTest(unittest.TestCase):
             environ = dict(
                 environ,
                 OLD_CONFIGURE=os.path.join(topsrcdir, 'old-configure'),
-                MOZCONFIG=mozconfig_path)
+                MOZCONFIG=mozconfig_path,
+                VIRTUALENV_NAME='python-test')
 
             paths = dict(paths)
             autoconf_dir = mozpath.join(topsrcdir, 'build', 'autoconf')

python/mozbuild/mozbuild/test/configure/test_toolkit_moz_configure.py

@@ -30,7 +30,9 @@ class TestToolkitMozConfigure(BaseConfigureTest):
             del sandbox._implied_options[:]
             result = sandbox._value_for(sandbox['all_configure_options'])
             shell = mozpath.abspath('/bin/sh')
-            return result.replace('CONFIG_SHELL=%s ' % shell, '')
+            return (result
+                    .replace('CONFIG_SHELL=%s ' % shell, '')
+                    .replace('VIRTUALENV_NAME=python-test ', ''))
 
         self.assertEquals('--enable-application=browser',
                           get_value_for(['--enable-application=browser']))

python/mozbuild/mozbuild/test/python.ini

@@ -24,13 +24,13 @@ skip-if = python == 2
 [configure/test_lint.py]
 skip-if = python == 2
 [configure/test_moz_configure.py]
-skip-if = python == 2 && os == 'mac'
+skip-if = python == 2
 [configure/test_options.py]
 [configure/test_toolchain_configure.py]
-skip-if = python == 2 && os == 'mac'
+skip-if = python == 2
 [configure/test_toolchain_helpers.py]
 [configure/test_toolkit_moz_configure.py]
-skip-if = python == 2 && os == 'mac'
+skip-if = python == 2
 [configure/test_util.py]
 [controller/test_ccachestats.py]
 [controller/test_clobber.py]

taskcluster/ci/source-test/python.yml

@@ -112,7 +112,7 @@ mochitest-harness:
             start_xvfb '1600x1200x24' 0 &&
             export GECKO_BINARY_PATH=$MOZ_FETCHES_DIR/firefox/firefox &&
             export TEST_HARNESS_ROOT=$MOZ_FETCHES_DIR/tests &&
-            ./mach python-test --python 2 --subsuite mochitest
+            python2 ./mach python-test --subsuite mochitest
     fetches:
         build:
             - target.tar.bz2
@@ -293,7 +293,7 @@ reftest-harness:
             start_xvfb '1600x1200x24' 0 &&
             export GECKO_BINARY_PATH=$MOZ_FETCHES_DIR/firefox/firefox &&
             export TEST_HARNESS_ROOT=$MOZ_FETCHES_DIR/tests &&
-            ./mach python-test --python 2 --subsuite reftest
+            python2 ./mach python-test --subsuite reftest
     fetches:
         build:
             - target.tar.bz2
@@ -481,7 +481,7 @@ telemetry-integration-tests:
             source /builds/worker/scripts/xvfb.sh &&
             start_xvfb '1600x1200x24' 0 &&
             export GECKO_BINARY_PATH=$MOZ_FETCHES_DIR/firefox/firefox &&
-            ./mach python-test --python 2 --subsuite telemetry-integration-tests
+            python2 ./mach python-test --subsuite telemetry-integration-tests
     fetches:
         build:
             - target.tar.bz2

taskcluster/taskgraph/transforms/job/mach.py

@@ -21,6 +21,11 @@ mach_schema = Schema({
     # The mach command (omitting `./mach`) to run
     Required('mach'): taskref_or_string,
 
+    # The version of Python to run with. Either an absolute path to the binary
+    # on the worker, a version identifier (e.g python2.7 or 3.8). There is no
+    # validation performed to ensure the specified binaries actually exist.
+    Optional('python-version'): Any(text_type, int, float),
+
     # The sparse checkout profile to use. Value is the filename relative to the
     # directory where sparse profiles are defined (build/sparse-profiles/).
     Optional('sparse-profile'): Any(text_type, None),
@@ -43,6 +48,7 @@ defaults = {
 @run_job_using("generic-worker", "mach", schema=mach_schema, defaults=defaults)
 def configure_mach(config, job, taskdesc):
     run = job['run']
+    worker = job['worker']
 
     additional_prefix = []
     if job['worker-type'].endswith('1014'):
@@ -51,6 +57,25 @@ def configure_mach(config, job, taskdesc):
             'LANG=en_US.UTF-8'
         ]
 
+    python = run.get('python-version')
+    if python:
+        del run['python-version']
+
+        if worker['os'] == 'macosx' and python == 3:
+            # OSX hosts can't seem to find python 3 on their own
+            python = '/tools/python37/bin/python3.7'
+            if job['worker-type'].endswith('1014'):
+                python = '/usr/local/bin/python3'
+
+        python = str(python)
+        try:
+            float(python)
+            python = "python" + python
+        except ValueError:
+            pass
+
+        additional_prefix.append(python)
+
     command_prefix = ' '.join(additional_prefix + ['./mach '])
 
     mach = run['mach']

taskcluster/taskgraph/transforms/job/python_test.py

@@ -38,16 +38,8 @@ def configure_python_test(config, job, taskdesc):
     run = job['run']
     worker = job['worker']
 
-    if worker['os'] == 'macosx' and run['python-version'] == 3:
-        # OSX hosts can't seem to find python 3 on their own
-        run['python-version'] = '/tools/python37/bin/python3.7'
-        if job['worker-type'].endswith('1014'):
-            run['python-version'] = '/usr/local/bin/python3'
-
     # defer to the mach implementation
-    run['mach'] = ("python-test --python {python-version} --subsuite {subsuite} "
-                   "--run-slow").format(**run)
+    run['mach'] = ("python-test --subsuite {subsuite} --run-slow").format(**run)
     run['using'] = 'mach'
-    del run['python-version']
     del run['subsuite']
     configure_taskdesc_for_run(config, job, taskdesc, worker['implementation'])

testing/mozbase/mozlog/setup.py

@@ -7,7 +7,7 @@ from __future__ import absolute_import
 from setuptools import setup, find_packages
 
 PACKAGE_NAME = 'mozlog'
-PACKAGE_VERSION = '6.1'
+PACKAGE_VERSION = '7.0'
 DEPS = [
     'blessings>=1.3',
     'mozterm',
@@ -41,8 +41,5 @@ setup(name=PACKAGE_NAME,
       entry_points={
           "console_scripts": [
               "structlog = mozlog.scripts:main"
-          ],
-          'pytest11': [
-              'mozlog = mozlog.pytest_mozlog.plugin',
           ]}
       )

testing/mozbase/mozproxy/tests/test_command_line.py

@@ -72,7 +72,8 @@ class OutputHandler(object):
 
 @pytest.fixture(scope="module")
 def install_mozproxy():
-    build = MozbuildObject.from_environment(cwd=here)
+    build = MozbuildObject.from_environment(
+        cwd=here, virtualenv_name='python-test')
     build.virtualenv_manager.activate()
 
     mozbase = os.path.join(build.topsrcdir, "testing", "mozbase")