Bug 1672181 - [taskgraph] Make test tasks py3 by default and add a py2 override r=bc,bhearsum

Differential Revision: https://phabricator.services.mozilla.com/D97669

taskcluster/ci/test/awsy.yml

@@ -4,6 +4,7 @@
 ---
 job-defaults:
     suite: awsy
+    force-py2: true
     max-run-time: 7200
     instance-size: xlarge
     virtualization:

taskcluster/ci/test/browsertime-desktop.yml

@@ -3,6 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
 job-defaults:
+    force-py2: true
     max-run-time:
         by-test-platform:
             .*-qr/.*: 2400

taskcluster/ci/test/browsertime-mobile.yml

@@ -5,6 +5,7 @@
 job-defaults:
     max-run-time: 3600
     suite: raptor
+    force-py2: true
     run-on-projects:
         by-app:
             refbrow: []

taskcluster/ci/test/compiled.yml

@@ -3,6 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
 job-defaults:
+    force-py2: true
     e10s: false
     test-manifest-loader: null  # don't load tests in the taskgraph
     virtualization:

taskcluster/ci/test/firefox-ui.yml

@@ -5,6 +5,7 @@
 job-defaults:
     suite:
         category: firefox-ui
+    force-py2: true
     max-run-time: 5400
     test-manifest-loader: null  # don't load tests in the taskgraph
     mozharness:

taskcluster/ci/test/marionette.yml

@@ -4,6 +4,7 @@
 ---
 job-defaults:
     suite: marionette
+    force-py2: true
     mozharness:
         script: marionette.py
         config:

taskcluster/ci/test/misc.yml

@@ -3,6 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
 job-defaults:
+    force-py2: true
     test-manifest-loader: null  # don't load tests in the taskgraph
 
 geckoview-junit:

taskcluster/ci/test/mochitest.yml

@@ -5,6 +5,7 @@
 job-defaults:
     suite:
         category: mochitest
+    force-py2: true
     target:
         by-test-platform:
             android-em-7.*: geckoview-androidTest.apk

taskcluster/ci/test/raptor-chrome.yml

@@ -9,6 +9,7 @@ job-defaults:
             .*-ux/.*: 2400
             default: 1800
     suite: raptor
+    force-py2: true
     run-on-projects: []
     test-manifest-loader: null  # don't load tests in the taskgraph
     tier: 2

taskcluster/ci/test/raptor-chromium.yml

@@ -9,6 +9,7 @@ job-defaults:
             .*-ux/.*: 2400
             default: 1800
     suite: raptor
+    force-py2: true
     run-on-projects: []
     tier: 2
     virtualization:

taskcluster/ci/test/raptor-gve.yml

@@ -5,6 +5,7 @@
 job-defaults:
     max-run-time: 1800
     suite: raptor
+    force-py2: true
     webrender-run-on-projects: []
     variants: ["webrender"]
     run-on-projects:

taskcluster/ci/test/raptor-refbrow.yml

@@ -4,6 +4,7 @@
 job-defaults:
     max-run-time: 1800
     suite: raptor
+    force-py2: true
     webrender-run-on-projects: []
     variants: ["webrender"]
     run-on-projects: []

taskcluster/ci/test/raptor.yml

@@ -16,6 +16,7 @@ job-defaults:
                     .*-ux/.*: 2400
                     default: 1800
     suite: raptor
+    force-py2: true
     test-manifest-loader: null  # don't load tests in the taskgraph
     run-on-projects:
         by-app:

taskcluster/ci/test/reftest.yml

@@ -5,6 +5,7 @@
 job-defaults:
     suite:
         category: reftest
+    force-py2: true
     target:
         by-test-platform:
             android-em-7.*: geckoview-androidTest.apk

taskcluster/ci/test/talos.yml

@@ -5,6 +5,7 @@
 job-defaults:
     max-run-time: 3600
     suite: talos
+    force-py2: true
     tier:
         by-test-platform:
             windows10-64-ccov.*/.*: 3

taskcluster/ci/test/web-platform.yml

@@ -5,6 +5,7 @@
 job-defaults:
     suite:
         category: web-platform-tests
+    force-py2: true
     instance-size: xlarge
     max-run-time:
         by-test-platform:

taskcluster/ci/test/xpcshell.yml

@@ -4,6 +4,7 @@
 ---
 job-defaults:
     suite: xpcshell
+    force-py2: true
     variants:
         by-test-platform:
             linux1804-64/debug: ['fission', 'socketprocess_networking']

taskcluster/scripts/tester/test-linux.sh

@@ -239,7 +239,7 @@ fi
 
 # Use |mach python| if a source checkout exists so in-tree packages are
 # available.
-[[ -x "${GECKO_PATH}/mach" ]] && python="python2.7 ${GECKO_PATH}/mach python" || python="python2.7"
+[[ -x "${GECKO_PATH}/mach" ]] && python="${PYTHON} ${GECKO_PATH}/mach python" || python="${PYTHON}"
 
 # Save the computed mozharness command to a binary which is useful for
 # interactive mode.

taskcluster/taskgraph/transforms/job/mozharness_test.py

@@ -45,6 +45,7 @@ mozharness_test_run_schema = Schema(
     {
         Required("using"): "mozharness-test",
         Required("test"): test_description_schema,
+        Optional("force-py2"): bool,
         # Base work directory used to set up the task.
         Optional("workdir"): text_type,
     }
@@ -91,6 +92,7 @@ def mozharness_test_on_docker(config, job, taskdesc):
     worker = taskdesc["worker"] = job["worker"]
 
     # apply some defaults
+    run.setdefault("force-py2", False)
     worker["docker-image"] = test["docker-image"]
     worker["allow-ptrace"] = True  # required for all tests, for crashreporter
     worker["loopback-video"] = test["loopback-video"]
@@ -131,12 +133,13 @@ def mozharness_test_on_docker(config, job, taskdesc):
     env = worker.setdefault("env", {})
     env.update(
         {
+            "ENABLE_E10S": text_type(bool(test.get("e10s"))).lower(),
             "MOZHARNESS_CONFIG": " ".join(mozharness["config"]),
             "MOZHARNESS_SCRIPT": mozharness["script"],
             "MOZILLA_BUILD_URL": {"task-reference": installer},
             "NEED_PULSEAUDIO": "true",
             "NEED_WINDOW_MANAGER": "true",
-            "ENABLE_E10S": text_type(bool(test.get("e10s"))).lower(),
+            "PYTHON": "python2.7" if run["force-py2"] else "python3",
             "WORKING_DIR": "/builds/worker",
         }
     )
@@ -236,10 +239,12 @@ def mozharness_test_on_docker(config, job, taskdesc):
 
 @run_job_using("generic-worker", "mozharness-test", schema=mozharness_test_run_schema)
 def mozharness_test_on_generic_worker(config, job, taskdesc):
+    run = job["run"]
     test = taskdesc["run"]["test"]
     mozharness = test["mozharness"]
     worker = taskdesc["worker"] = job["worker"]
 
+    run.setdefault("force-py2", False)
     bitbar_script = "test-linux.sh"
 
     is_macosx = worker["os"] == "macosx"
@@ -304,6 +309,7 @@ def mozharness_test_on_generic_worker(config, job, taskdesc):
     env = worker.setdefault("env", {})
     env["GECKO_HEAD_REPOSITORY"] = config.params["head_repository"]
     env["GECKO_HEAD_REV"] = config.params["head_rev"]
+    env["PYTHON"] = "python2.7" if run["force-py2"] else "python3"
 
     # this list will get cleaned up / reduced / removed in bug 1354088
     if is_macosx:
@@ -354,7 +360,9 @@ def mozharness_test_on_generic_worker(config, job, taskdesc):
 
     if is_windows:
         mh_command = [
-            "c:\\mozilla-build\\python\\python.exe",
+            "c:\\mozilla-build\\{python}\\{python}.exe".format(
+                python="python" if run["force-py2"] else "python3"
+            ),
             "-u",
             "mozharness\\scripts\\" + normpath(mozharness["script"]),
         ]
@@ -362,7 +370,7 @@ def mozharness_test_on_generic_worker(config, job, taskdesc):
         mh_command = ["bash", "./{}".format(bitbar_script)]
     elif is_macosx and "macosx1014-64" in test["test-platform"]:
         mh_command = [
-            "/usr/local/bin/python2",
+            "/usr/local/bin/{}".format("python2" if run["force-py2"] else "python3"),
             "-u",
             "mozharness/scripts/" + mozharness["script"],
         ]
@@ -372,7 +380,7 @@ def mozharness_test_on_generic_worker(config, job, taskdesc):
             # Using /usr/bin/python2.7 rather than python2.7 because
             # /usr/local/bin/python2.7 is broken on the mac workers.
             # See bug #1547903.
-            "/usr/bin/python2.7",
+            "/usr/bin/{}".format("python2.7" if run["force-py2"] else "python3"),
             "-u",
             "mozharness/scripts/" + mozharness["script"],
         ]

taskcluster/taskgraph/transforms/tests.py

@@ -571,6 +571,8 @@ test_description_schema = Schema(
         Optional("fetches"): {
             text_type: optionally_keyed_by("test-platform", [text_type])
         },
+        # Use Python 2 rather than Python 3.
+        Optional("force-py2"): bool,
     }
 )
 
@@ -640,6 +642,7 @@ def set_defaults(config, tasks):
         task.setdefault("limit-platforms", [])
         task.setdefault("docker-image", {"in-tree": "ubuntu1804-test"})
         task.setdefault("checkout", False)
+        task.setdefault("force-py2", False)
         task.setdefault("require-signed-extensions", False)
         task.setdefault("variants", [])
 
@@ -1900,6 +1903,7 @@ def make_job_description(config, tasks):
         run = jobdesc["run"] = {}
         run["using"] = "mozharness-test"
         run["test"] = task
+        run["force-py2"] = task.pop("force-py2", False)
 
         if "workdir" in task:
             run["workdir"] = task.pop("workdir")