diff --git a/taskcluster/ci/build/kind.yml b/taskcluster/ci/build/kind.yml index 53cc565d056e..c87d17428d0e 100644 --- a/taskcluster/ci/build/kind.yml +++ b/taskcluster/ci/build/kind.yml @@ -33,6 +33,7 @@ job-defaults: worker: env: MACH_BUILD_PYTHON_NATIVE_PACKAGE_SOURCE: system + use-system-python: false run: mozconfig-variant: by-release-type: diff --git a/taskcluster/gecko_taskgraph/transforms/job/__init__.py b/taskcluster/gecko_taskgraph/transforms/job/__init__.py index 626fe1abc916..9b6924b6053b 100644 --- a/taskcluster/gecko_taskgraph/transforms/job/__init__.py +++ b/taskcluster/gecko_taskgraph/transforms/job/__init__.py @@ -62,6 +62,7 @@ job_description_schema = Schema( "optimization" ], Optional("use-sccache"): task_description_schema["use-sccache"], + Optional("use-system-python"): bool, Optional("priority"): task_description_schema["priority"], # The "when" section contains descriptions of the circumstances under which # this task should be included in the task graph. This will be converted @@ -241,6 +242,34 @@ def get_attribute(dict, key, attributes, attribute_name): dict[key] = value +@transforms.add +def use_system_python(config, jobs): + for job in jobs: + if job.pop("use-system-python", True): + yield job + else: + fetches = job.setdefault("fetches", {}) + toolchain = fetches.setdefault("toolchain", []) + + moz_python_home = mozpath.join("fetches", "python") + if "win" in job["worker"]["os"]: + platform = "win64" + elif "linux" in job["worker"]["os"]: + platform = "linux64" + elif "macosx" in job["worker"]["os"]: + platform = "macosx64" + else: + raise ValueError("unexpected worker.os value {}".format(platform)) + + toolchain.append("{}-python".format(platform)) + + worker = job.setdefault("worker", {}) + env = worker.setdefault("env", {}) + env["MOZ_PYTHON_HOME"] = moz_python_home + + yield job + + @transforms.add def use_fetches(config, jobs): artifact_names = {} diff --git a/taskcluster/gecko_taskgraph/transforms/job/mozharness.py b/taskcluster/gecko_taskgraph/transforms/job/mozharness.py index bbcb18d55c73..c023562437a8 100644 --- a/taskcluster/gecko_taskgraph/transforms/job/mozharness.py +++ b/taskcluster/gecko_taskgraph/transforms/job/mozharness.py @@ -283,11 +283,25 @@ def mozharness_on_generic_worker(config, job, taskdesc): mh_command = [] if job["worker"]["os"] == "windows": - mh_command.append("c:/mozilla-build/python3/python3.exe") + system_python_dir = "c:/mozilla-build/python3/" gecko_path = "%GECKO_PATH%" else: + system_python_dir = "" gecko_path = "$GECKO_PATH" + if run.get("use-system-python", False): + python_bindir = system_python_dir + else: + # $MOZ_PYTHON_HOME is going to be substituted in run-task, when we + # know the actual MOZ_PYTHON_HOME value. + is_windows = job["worker"]["os"] == "windows" + if is_windows: + python_bindir = "%MOZ_PYTHON_HOME%/" + else: + python_bindir = "${MOZ_PYTHON_HOME}/bin/" + + mh_command = ["{}python3".format(python_bindir)] + mh_command += [ f"{gecko_path}/mach", "python", diff --git a/taskcluster/scripts/run-task b/taskcluster/scripts/run-task index 9548a81bab80..fb0d32f8f52d 100755 --- a/taskcluster/scripts/run-task +++ b/taskcluster/scripts/run-task @@ -27,7 +27,6 @@ import errno import io import json import os -import platform import random import re import shutil @@ -936,7 +935,7 @@ def main(args): resource_process = None try: - for k in ('GECKO_PATH', 'MOZ_FETCHES_DIR', 'UPLOAD_DIR'): + for k in ('GECKO_PATH', 'MOZ_FETCHES_DIR', 'UPLOAD_DIR', 'MOZ_PYTHON_HOME'): if k in os.environ: # Normalize paths to use forward slashes. Some shell scripts # tolerate that better on Windows. @@ -948,6 +947,43 @@ def main(args): if 'MOZ_FETCHES' in os.environ: fetch_artifacts() + # If Python is a fetch dependency, add it to the PATH and setting + # the mozilla-specific MOZ_PYTHON_HOME to relocate binaries. + if 'MOZ_PYTHON_HOME' in os.environ: + + print_line(b'setup', + b'Setting up local python environment\n') + prev = [os.environ['PATH']] if 'PATH' in os.environ else [] + + moz_python_home = os.environ['MOZ_PYTHON_HOME'] + if IS_WINDOWS: + ext = '.exe' + moz_python_bindir = moz_python_home + else: + ext = '' + moz_python_bindir = moz_python_home + '/bin' + + + # just a sanity check + candidate = os.path.join(moz_python_bindir, f'python3{ext}') + if not os.path.exists(candidate): + raise RuntimeError("Inconsistent Python installation: " + "archive found, but no python3 binary " + "detected") + + new = os.environ['PATH'] = os.pathsep.join([moz_python_bindir] + + prev) + + # Relocate the python binary. Standard way uses PYTHONHOME, but + # this conflicts with system python (e.g. used by hg) so we + # maintain a small patch to use MOZPYTHONHOME instead. + os.environ['MOZPYTHONHOME'] = moz_python_home + + print_line(b'setup', + b'updated PATH with python artifact: ' + + new.encode() + b'\n') + + resource_process = maybe_run_resource_monitoring() return run_and_prefix_output(b'task', task_args, cwd=args.task_cwd)