Bug 1672181 - [taskgraph] Make test tasks py3 by default and add a py2 override r=bc,bhearsum

Differential Revision: https://phabricator.services.mozilla.com/D97669
2020-11-20 21:45:17 +00:00 · 2020-11-20 21:45:17 +00:00 · 5e91379180
--- a/taskcluster/ci/test/awsy.yml
+++ b/taskcluster/ci/test/awsy.yml
@ -4,6 +4,7 @@
 ---
 job-defaults:
    suite: awsy
+    force-py2: true
    max-run-time: 7200
    instance-size: xlarge
    virtualization:
--- a/taskcluster/ci/test/browsertime-desktop.yml
+++ b/taskcluster/ci/test/browsertime-desktop.yml
@ -3,6 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
 job-defaults:
+    force-py2: true
    max-run-time:
        by-test-platform:
            .*-qr/.*: 2400
--- a/taskcluster/ci/test/browsertime-mobile.yml
+++ b/taskcluster/ci/test/browsertime-mobile.yml
@ -5,6 +5,7 @@
 job-defaults:
    max-run-time: 3600
    suite: raptor
+    force-py2: true
    run-on-projects:
        by-app:
            refbrow: []
--- a/taskcluster/ci/test/compiled.yml
+++ b/taskcluster/ci/test/compiled.yml
@ -3,6 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
 job-defaults:
+    force-py2: true
    e10s: false
    test-manifest-loader: null  # don't load tests in the taskgraph
    virtualization:
--- a/taskcluster/ci/test/firefox-ui.yml
+++ b/taskcluster/ci/test/firefox-ui.yml
@ -5,6 +5,7 @@
 job-defaults:
    suite:
        category: firefox-ui
+    force-py2: true
    max-run-time: 5400
    test-manifest-loader: null  # don't load tests in the taskgraph
    mozharness:
--- a/taskcluster/ci/test/marionette.yml
+++ b/taskcluster/ci/test/marionette.yml
@ -4,6 +4,7 @@
 ---
 job-defaults:
    suite: marionette
+    force-py2: true
    mozharness:
        script: marionette.py
        config:
--- a/taskcluster/ci/test/misc.yml
+++ b/taskcluster/ci/test/misc.yml
@ -3,6 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
 job-defaults:
+    force-py2: true
    test-manifest-loader: null  # don't load tests in the taskgraph

 geckoview-junit:
--- a/taskcluster/ci/test/mochitest.yml
+++ b/taskcluster/ci/test/mochitest.yml
@ -5,6 +5,7 @@
 job-defaults:
    suite:
        category: mochitest
+    force-py2: true
    target:
        by-test-platform:
            android-em-7.*: geckoview-androidTest.apk
--- a/taskcluster/ci/test/raptor-chrome.yml
+++ b/taskcluster/ci/test/raptor-chrome.yml
@ -9,6 +9,7 @@ job-defaults:
            .*-ux/.*: 2400
            default: 1800
    suite: raptor
+    force-py2: true
    run-on-projects: []
    test-manifest-loader: null  # don't load tests in the taskgraph
    tier: 2
--- a/taskcluster/ci/test/raptor-chromium.yml
+++ b/taskcluster/ci/test/raptor-chromium.yml
@ -9,6 +9,7 @@ job-defaults:
            .*-ux/.*: 2400
            default: 1800
    suite: raptor
+    force-py2: true
    run-on-projects: []
    tier: 2
    virtualization:
--- a/taskcluster/ci/test/raptor-gve.yml
+++ b/taskcluster/ci/test/raptor-gve.yml
@ -5,6 +5,7 @@
 job-defaults:
    max-run-time: 1800
    suite: raptor
+    force-py2: true
    webrender-run-on-projects: []
    variants: ["webrender"]
    run-on-projects:
--- a/taskcluster/ci/test/raptor-refbrow.yml
+++ b/taskcluster/ci/test/raptor-refbrow.yml
@ -4,6 +4,7 @@
 job-defaults:
    max-run-time: 1800
    suite: raptor
+    force-py2: true
    webrender-run-on-projects: []
    variants: ["webrender"]
    run-on-projects: []
--- a/taskcluster/ci/test/raptor.yml
+++ b/taskcluster/ci/test/raptor.yml
@ -16,6 +16,7 @@ job-defaults:
                    .*-ux/.*: 2400
                    default: 1800
    suite: raptor
+    force-py2: true
    test-manifest-loader: null  # don't load tests in the taskgraph
    run-on-projects:
        by-app:
--- a/taskcluster/ci/test/reftest.yml
+++ b/taskcluster/ci/test/reftest.yml
@ -5,6 +5,7 @@
 job-defaults:
    suite:
        category: reftest
+    force-py2: true
    target:
        by-test-platform:
            android-em-7.*: geckoview-androidTest.apk
--- a/taskcluster/ci/test/talos.yml
+++ b/taskcluster/ci/test/talos.yml
@ -5,6 +5,7 @@
 job-defaults:
    max-run-time: 3600
    suite: talos
+    force-py2: true
    tier:
        by-test-platform:
            windows10-64-ccov.*/.*: 3
--- a/taskcluster/ci/test/web-platform.yml
+++ b/taskcluster/ci/test/web-platform.yml
@ -5,6 +5,7 @@
 job-defaults:
    suite:
        category: web-platform-tests
+    force-py2: true
    instance-size: xlarge
    max-run-time:
        by-test-platform:
--- a/taskcluster/ci/test/xpcshell.yml
+++ b/taskcluster/ci/test/xpcshell.yml
@ -4,6 +4,7 @@
 ---
 job-defaults:
    suite: xpcshell
+    force-py2: true
    variants:
        by-test-platform:
            linux1804-64/debug: ['fission', 'socketprocess_networking']
--- a/taskcluster/scripts/tester/test-linux.sh
+++ b/taskcluster/scripts/tester/test-linux.sh
@ -239,7 +239,7 @@ fi

 # Use |mach python| if a source checkout exists so in-tree packages are
 # available.
-[[ -x "${GECKO_PATH}/mach" ]] && python="python2.7 ${GECKO_PATH}/mach python" || python="python2.7"
+[[ -x "${GECKO_PATH}/mach" ]] && python="${PYTHON} ${GECKO_PATH}/mach python" || python="${PYTHON}"

 # Save the computed mozharness command to a binary which is useful for
 # interactive mode.
--- a/taskcluster/taskgraph/transforms/job/mozharness_test.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness_test.py
@ -45,6 +45,7 @@ mozharness_test_run_schema = Schema(
    {
        Required("using"): "mozharness-test",
        Required("test"): test_description_schema,
+        Optional("force-py2"): bool,
        # Base work directory used to set up the task.
        Optional("workdir"): text_type,
    }
@ -91,6 +92,7 @@ def mozharness_test_on_docker(config, job, taskdesc):
    worker = taskdesc["worker"] = job["worker"]

    # apply some defaults
+    run.setdefault("force-py2", False)
    worker["docker-image"] = test["docker-image"]
    worker["allow-ptrace"] = True  # required for all tests, for crashreporter
    worker["loopback-video"] = test["loopback-video"]
@ -131,12 +133,13 @@ def mozharness_test_on_docker(config, job, taskdesc):
    env = worker.setdefault("env", {})
    env.update(
        {
+            "ENABLE_E10S": text_type(bool(test.get("e10s"))).lower(),
            "MOZHARNESS_CONFIG": " ".join(mozharness["config"]),
            "MOZHARNESS_SCRIPT": mozharness["script"],
            "MOZILLA_BUILD_URL": {"task-reference": installer},
            "NEED_PULSEAUDIO": "true",
            "NEED_WINDOW_MANAGER": "true",
-            "ENABLE_E10S": text_type(bool(test.get("e10s"))).lower(),
+            "PYTHON": "python2.7" if run["force-py2"] else "python3",
            "WORKING_DIR": "/builds/worker",
        }
    )
@ -236,10 +239,12 @@ def mozharness_test_on_docker(config, job, taskdesc):

@run_job_using("generic-worker", "mozharness-test", schema=mozharness_test_run_schema)
 def mozharness_test_on_generic_worker(config, job, taskdesc):
+    run = job["run"]
    test = taskdesc["run"]["test"]
    mozharness = test["mozharness"]
    worker = taskdesc["worker"] = job["worker"]

+    run.setdefault("force-py2", False)
    bitbar_script = "test-linux.sh"

    is_macosx = worker["os"] == "macosx"
@ -304,6 +309,7 @@ def mozharness_test_on_generic_worker(config, job, taskdesc):
    env = worker.setdefault("env", {})
    env["GECKO_HEAD_REPOSITORY"] = config.params["head_repository"]
    env["GECKO_HEAD_REV"] = config.params["head_rev"]
+    env["PYTHON"] = "python2.7" if run["force-py2"] else "python3"

    # this list will get cleaned up / reduced / removed in bug 1354088
    if is_macosx:
@ -354,7 +360,9 @@ def mozharness_test_on_generic_worker(config, job, taskdesc):

    if is_windows:
        mh_command = [
-            "c:\\mozilla-build\\python\\python.exe",
+            "c:\\mozilla-build\\{python}\\{python}.exe".format(
+                python="python" if run["force-py2"] else "python3"
+            ),
            "-u",
            "mozharness\\scripts\\" + normpath(mozharness["script"]),
        ]
@ -362,7 +370,7 @@ def mozharness_test_on_generic_worker(config, job, taskdesc):
        mh_command = ["bash", "./{}".format(bitbar_script)]
    elif is_macosx and "macosx1014-64" in test["test-platform"]:
        mh_command = [
-            "/usr/local/bin/python2",
+            "/usr/local/bin/{}".format("python2" if run["force-py2"] else "python3"),
            "-u",
            "mozharness/scripts/" + mozharness["script"],
        ]
@ -372,7 +380,7 @@ def mozharness_test_on_generic_worker(config, job, taskdesc):
            # Using /usr/bin/python2.7 rather than python2.7 because
            # /usr/local/bin/python2.7 is broken on the mac workers.
            # See bug #1547903.
-            "/usr/bin/python2.7",
+            "/usr/bin/{}".format("python2.7" if run["force-py2"] else "python3"),
            "-u",
            "mozharness/scripts/" + mozharness["script"],
        ]
--- a/taskcluster/taskgraph/transforms/tests.py
+++ b/taskcluster/taskgraph/transforms/tests.py
@ -571,6 +571,8 @@ test_description_schema = Schema(
        Optional("fetches"): {
            text_type: optionally_keyed_by("test-platform", [text_type])
        },
+        # Use Python 2 rather than Python 3.
+        Optional("force-py2"): bool,
    }
 )

@ -640,6 +642,7 @@ def set_defaults(config, tasks):
        task.setdefault("limit-platforms", [])
        task.setdefault("docker-image", {"in-tree": "ubuntu1804-test"})
        task.setdefault("checkout", False)
+        task.setdefault("force-py2", False)
        task.setdefault("require-signed-extensions", False)
        task.setdefault("variants", [])

@ -1900,6 +1903,7 @@ def make_job_description(config, tasks):
        run = jobdesc["run"] = {}
        run["using"] = "mozharness-test"
        run["test"] = task
+        run["force-py2"] = task.pop("force-py2", False)

        if "workdir" in task:
            run["workdir"] = task.pop("workdir")