From d8119e6288bcc273ada4e77cb5c8ae896fb429db Mon Sep 17 00:00:00 2001 From: Ricky Stewart Date: Fri, 13 Nov 2020 20:21:45 +0000 Subject: [PATCH] Bug 1661624 - Include `psutil` in the `mach` `virtualenv`s r=firefox-build-system-reviewers,mhentges Install `psutil` when setting up the `mach` `virtualenv`s and stop importing the in-tree version in the build. Nothing in-tree currently assumes or mandates the installation of `psutil` (all uses of `psutil` are guarded with imports of the form `try : import psutil; except ImportError: psutil = None`), so there's no back-incompatibility concerns here. There will be an awkward period where telemetry will be lacking CPU/disk data for everyone until they re-run `mach bootstrap` or `mach create-mach-environment`, but that will come back as people gradually update their `virtualenv`s. An alternative to circumvent that issue is REQUIRING that `psutil` be installed by adding an assertion in `mach` that `psutil` can be found (allowing us to remove all the conditional logic in-tree around whether `psutil` is installed), but I wouldn't claim that we're ready to do that and deal with whatever fallout might occur. Differential Revision: https://phabricator.services.mozilla.com/D90914 --- build/common_virtualenv_packages.txt | 3 --- python/mozbuild/mozbuild/mach_commands.py | 10 ++++++++++ taskcluster/ci/docker-image/kind.yml | 1 + taskcluster/ci/packages/kind.yml | 14 ++++++++++++++ taskcluster/docker/debian-base/Dockerfile | 2 ++ taskcluster/docker/decision/VERSION | 2 +- taskcluster/docker/decision/system-setup.sh | 4 +++- taskcluster/docker/mingw32-build/Dockerfile | 1 + .../docker/recipes/ubuntu1804-test-system-setup.sh | 3 +++ taskcluster/scripts/builder/build-l10n.sh | 5 ++++- taskcluster/scripts/builder/build-linux.sh | 5 ++++- taskcluster/scripts/builder/requirements.txt | 1 + 12 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 taskcluster/scripts/builder/requirements.txt diff --git a/build/common_virtualenv_packages.txt b/build/common_virtualenv_packages.txt index 37cd97964e8c..35d7a6a2e666 100644 --- a/build/common_virtualenv_packages.txt +++ b/build/common_virtualenv_packages.txt @@ -44,9 +44,6 @@ mozilla.pth:third_party/python/pyrsistent mozilla.pth:third_party/python/python-hglib mozilla.pth:third_party/python/pluggy mozilla.pth:third_party/python/jsmin -!windows:optional:setup.py:third_party/python/psutil:build_ext:--inplace -!windows:mozilla.pth:third_party/python/psutil -windows:mozilla.pth:third_party/python/psutil-cp27-none-win_amd64 mozilla.pth:third_party/python/pylru mozilla.pth:third_party/python/pystache python2:mozilla.pth:third_party/python/PyYAML/lib diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py index 43dcafccae1b..25b05fa4754d 100644 --- a/python/mozbuild/mozbuild/mach_commands.py +++ b/python/mozbuild/mozbuild/mach_commands.py @@ -2097,6 +2097,16 @@ class CreateMachEnvironment(MachCommandBase): manager.install_pip_package("zstandard>=0.9.0,<=0.13.0") + try: + # `mach` can handle it perfectly fine if `psutil` is missing, so + # there's no reason to freak out in this case. + manager.install_pip_package("psutil==5.7.0") + except subprocess.CalledProcessError: + print( + "Could not install psutil, so telemetry will be missing some " + "data. Continuing." + ) + if not PY2: # This can fail on some platforms. See # https://bugzilla.mozilla.org/show_bug.cgi?id=1660120 diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml index b841bc84f23c..b6729caf19f7 100644 --- a/taskcluster/ci/docker-image/kind.yml +++ b/taskcluster/ci/docker-image/kind.yml @@ -58,6 +58,7 @@ jobs: - deb8-mercurial - deb8-python3.6 - deb8-python3-defaults + - deb8-python-psutil - deb8-python-zstandard - deb8-xz-utils deb8-toolchain-build: diff --git a/taskcluster/ci/packages/kind.yml b/taskcluster/ci/packages/kind.yml index c09ab5323f29..790e80c6aced 100644 --- a/taskcluster/ci/packages/kind.yml +++ b/taskcluster/ci/packages/kind.yml @@ -207,6 +207,20 @@ jobs: sha256: 9727dcb3d6b655e4f2a92110f5db076a490aa50b739804be239905ecff3aacc8 patch: gdb-jessie.diff + deb8-python-psutil: + description: "python-psutil rebuild for python 3.6 in Debian jessie" + treeherder: + symbol: Deb8(python-psutil) + run: + using: debian-package + dsc: + url: http://snapshot.debian.org/archive/debian/20150815T034233Z/pool/main/p/python-psutil/python-psutil_2.2.1-3.dsc + sha256: b47d1fc92094dfd5525cff7d0af5855f7c5335ade9de4c0e738ed490aa5bee7c + packages: + - deb8-dh-python + - deb8-python3.6 + - deb8-python3-defaults + deb8-python-zstandard: description: "python-zstandard for Debian jessie" treeherder: diff --git a/taskcluster/docker/debian-base/Dockerfile b/taskcluster/docker/debian-base/Dockerfile index fb8b503d5a48..9c4d59190e1e 100644 --- a/taskcluster/docker/debian-base/Dockerfile +++ b/taskcluster/docker/debian-base/Dockerfile @@ -43,6 +43,8 @@ RUN /usr/local/sbin/setup_packages.sh $TASKCLUSTER_ROOT_URL $DOCKER_IMAGE_PACKAG python3-minimal \ python-zstandard \ python3-zstandard \ + python-psutil \ + python3-psutil \ vim-tiny \ xz-utils diff --git a/taskcluster/docker/decision/VERSION b/taskcluster/docker/decision/VERSION index c043eea7767e..b1b25a5ffae4 100644 --- a/taskcluster/docker/decision/VERSION +++ b/taskcluster/docker/decision/VERSION @@ -1 +1 @@ -2.2.1 +2.2.2 diff --git a/taskcluster/docker/decision/system-setup.sh b/taskcluster/docker/decision/system-setup.sh index f33ce6b4f447..f90052565ff6 100644 --- a/taskcluster/docker/decision/system-setup.sh +++ b/taskcluster/docker/decision/system-setup.sh @@ -10,7 +10,9 @@ apt-get install -y --force-yes --no-install-recommends \ python \ sudo \ python3 \ - python3-yaml + python3-yaml \ + python-psutil \ + python3-psutil BUILD=/root/build mkdir "$BUILD" diff --git a/taskcluster/docker/mingw32-build/Dockerfile b/taskcluster/docker/mingw32-build/Dockerfile index cc71c03d0ab2..b6a615de78d5 100644 --- a/taskcluster/docker/mingw32-build/Dockerfile +++ b/taskcluster/docker/mingw32-build/Dockerfile @@ -27,6 +27,7 @@ RUN apt-get update && \ libucl1 \ libxml2 \ patch \ + python3-dev \ p7zip-full \ scons \ tar \ diff --git a/taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh b/taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh index 11f96e68240b..e3c5f86c71b1 100644 --- a/taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh +++ b/taskcluster/docker/recipes/ubuntu1804-test-system-setup.sh @@ -144,6 +144,9 @@ pip install virtualenv==15.2.0 pip install zstandard==0.13.0 pip3 install zstandard==0.13.0 +pip install psutil==5.7.0 +pip3 install psutil==5.7.0 + # Build a list of packages to purge from the image. apt_packages=() apt_packages+=('*cheese*') diff --git a/taskcluster/scripts/builder/build-l10n.sh b/taskcluster/scripts/builder/build-l10n.sh index 18b238b16e22..e9e9db81e8c2 100755 --- a/taskcluster/scripts/builder/build-l10n.sh +++ b/taskcluster/scripts/builder/build-l10n.sh @@ -99,7 +99,10 @@ fi cd /builds/worker -$GECKO_PATH/mach python $GECKO_PATH/testing/${MOZHARNESS_SCRIPT} \ +$GECKO_PATH/mach python \ + --requirements $GECKO_PATH/taskcluster/scripts/builder/requirements.txt \ + -- \ + $GECKO_PATH/testing/${MOZHARNESS_SCRIPT} \ ${config_path_cmds} \ ${config_cmds} \ $actions \ diff --git a/taskcluster/scripts/builder/build-linux.sh b/taskcluster/scripts/builder/build-linux.sh index 037915687e5f..bcc5b34606cf 100755 --- a/taskcluster/scripts/builder/build-linux.sh +++ b/taskcluster/scripts/builder/build-linux.sh @@ -114,7 +114,10 @@ fi cd /builds/worker -$GECKO_PATH/mach python $GECKO_PATH/testing/${MOZHARNESS_SCRIPT} \ +$GECKO_PATH/mach python \ + --requirements $GECKO_PATH/taskcluster/scripts/builder/requirements.txt \ + -- \ + $GECKO_PATH/testing/${MOZHARNESS_SCRIPT} \ ${config_path_cmds} \ ${config_cmds} \ $debug_flag \ diff --git a/taskcluster/scripts/builder/requirements.txt b/taskcluster/scripts/builder/requirements.txt new file mode 100644 index 000000000000..24156d6d30e2 --- /dev/null +++ b/taskcluster/scripts/builder/requirements.txt @@ -0,0 +1 @@ +psutil==5.7.0