From 1e84a04fb4fd57f8dbd30760fca0907d9b97c3eb Mon Sep 17 00:00:00 2001 From: Edwin Takahashi Date: Thu, 16 Jul 2020 23:36:01 +0000 Subject: [PATCH] Bug 1648851 - Part 0: create android-test docker image based on Ubuntu 20.04 r=jmaher,gbrown Changes: - set up a new minimal docker image for `android-em` tests. - reorganize commonly referenced files into a specific folder `taskcluster/docker/files` to reduce instances of same file being copy-pasted into directories. - add android-test docker build task to taskcluster. Differential Revision: https://phabricator.services.mozilla.com/D82204 --- taskcluster/ci/docker-image/kind.yml | 2 + taskcluster/docker/android-test/Dockerfile | 87 +++++++++++++++++++ .../autostart/gnome-software-service.desktop | 9 ++ taskcluster/docker/files/dbus.sh | 7 ++ taskcluster/docker/files/motd | 6 ++ .../files/taskcluster-interactive-shell | 22 +++++ .../recipes/android-test-system-setup.sh | 76 ++++++++++++++++ .../docker/recipes/install-mercurial.sh | 2 +- 8 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 taskcluster/docker/android-test/Dockerfile create mode 100644 taskcluster/docker/files/autostart/gnome-software-service.desktop create mode 100644 taskcluster/docker/files/dbus.sh create mode 100644 taskcluster/docker/files/motd create mode 100644 taskcluster/docker/files/taskcluster-interactive-shell create mode 100644 taskcluster/docker/recipes/android-test-system-setup.sh diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml index 4e3709966bdf..195e42b7ce71 100644 --- a/taskcluster/ci/docker-image/kind.yml +++ b/taskcluster/ci/docker-image/kind.yml @@ -155,6 +155,8 @@ jobs: parent: debian10-base packages: - deb8-valgrind + android-test: + symbol: I(android-test) fetch: symbol: I(fetch) parent: debian10-raw diff --git a/taskcluster/docker/android-test/Dockerfile b/taskcluster/docker/android-test/Dockerfile new file mode 100644 index 000000000000..8f3eea36d3f9 --- /dev/null +++ b/taskcluster/docker/android-test/Dockerfile @@ -0,0 +1,87 @@ +FROM ubuntu:focal-20200606 +MAINTAINER Edwin Takahashi + +# Create necessary directories and worker user account +RUN mkdir -p /builds && mkdir -p artifacts +RUN id worker || useradd -d /builds/worker -s /bin/bash -m worker +WORKDIR /builds/worker + +# We need to declare all potentially cache volumes as caches. Also, +# making high I/O paths volumes increase I/O throughput because of +# AUFS slowness. +VOLUME /builds/worker/.cache +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/tooltool-cache +VOLUME /builds/worker/workspace + +# In test.sh we accept START_VNC to start a vnc daemon. +# Exposing this port allows it to work. +EXPOSE 5900 + +# %include python/mozbuild/mozbuild/action/tooltool.py +ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /setup/tooltool.py + +# %include testing/mozharness/external_tools/robustcheckout.py +ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py + +# %include taskcluster/docker/recipes/hgrc +COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc + +# %include taskcluster/docker/recipes/common.sh +ADD topsrcdir/taskcluster/docker/recipes/common.sh /setup/common.sh + +# %include taskcluster/docker/recipes/install-mercurial.sh +ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh + +# Add wrapper scripts for xvfb allowing tasks to easily retry starting up xvfb +# %include taskcluster/docker/recipes/xvfb.sh +ADD topsrcdir/taskcluster/docker/recipes/xvfb.sh /builds/worker/scripts/xvfb.sh + +# %include taskcluster/scripts/run-task +ADD topsrcdir/taskcluster/scripts/run-task /builds/worker/bin/run-task + +# %include taskcluster/scripts/misc/fetch-content +ADD topsrcdir/taskcluster/scripts/misc/fetch-content /builds/worker/bin/fetch-content + +ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /builds/worker/scripts/tooltool.py + +# %include taskcluster/docker/recipes/android-test-system-setup.sh +ADD topsrcdir/taskcluster/docker/recipes/android-test-system-setup.sh /setup/system-setup.sh +RUN bash /setup/system-setup.sh + +# %include taskcluster/scripts/tester/test-linux.sh +ADD topsrcdir/taskcluster/scripts/tester/test-linux.sh /builds/worker/bin/test-linux.sh + +# Set variable normally configured at login, by the shells parent process, these +# are taken from GNU su manual +ENV HOME /builds/worker +ENV SHELL /bin/bash +ENV USER worker +ENV LOGNAME worker +ENV HOSTNAME taskcluster-worker +ENV LANG en_US.UTF-8 +ENV PATH $PATH:/builds/worker/bin + +# Bug 1345105 - Do not run periodical update checks and downloads +# %include taskcluster/docker/files/autostart/gnome-software-service.desktop +ADD topsrcdir/taskcluster/docker/files/autostart/gnome-software-service.desktop /etc/xdg/autostart/gnome-software-service.desktop + +# Set execution and ownership privileges +RUN chmod +x bin/*; chown -R worker:worker /builds/worker + +# Set up first-run experience for interactive mode +# %include taskcluster/docker/files/motd +ADD topsrcdir/taskcluster/docker/files/motd /etc/taskcluster-motd +# %include taskcluster/docker/files/taskcluster-interactive-shell +ADD topsrcdir/taskcluster/docker/files/taskcluster-interactive-shell /bin/taskcluster-interactive-shell +RUN chmod +x /bin/taskcluster-interactive-shell + +# Ubuntu 18.04 (or any other GNOME3 based systems) needs dbus to have various +# test function as expected. Use entrypoint to initialize dbus as root. +# %include taskcluster/docker/files/dbus.sh +ADD topsrcdir/taskcluster/docker/files/dbus.sh /usr/local/bin/dbus.sh +RUN chmod +x /usr/local/bin/dbus.sh +ENTRYPOINT ["/usr/local/bin/dbus.sh"] + +# Set a default command useful for debugging +CMD ["/bin/bash", "--login"] diff --git a/taskcluster/docker/files/autostart/gnome-software-service.desktop b/taskcluster/docker/files/autostart/gnome-software-service.desktop new file mode 100644 index 000000000000..b563cc306b06 --- /dev/null +++ b/taskcluster/docker/files/autostart/gnome-software-service.desktop @@ -0,0 +1,9 @@ +[Desktop Entry] +Type=Application +Name=GNOME Software +Exec=/usr/bin/gnome-software --gapplication-service +OnlyShowIn=GNOME;Unity; +X-Ubuntu-Gettext-Domain=gnome-software + +# Bug 1345105 - Do not run periodical update checks and downloads +X-GNOME-Autostart-enabled=false diff --git a/taskcluster/docker/files/dbus.sh b/taskcluster/docker/files/dbus.sh new file mode 100644 index 000000000000..f27ec664f14a --- /dev/null +++ b/taskcluster/docker/files/dbus.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +/etc/init.d/dbus start 2>&1 + +exec "${@}" diff --git a/taskcluster/docker/files/motd b/taskcluster/docker/files/motd new file mode 100644 index 000000000000..f958393cd8b3 --- /dev/null +++ b/taskcluster/docker/files/motd @@ -0,0 +1,6 @@ +Welcome to your taskcluster interactive shell! The regularly scheduled task +has been paused to give you a chance to set up your debugging environment. + +For your convenience, the exact mozharness command needed for this task can +be invoked using the 'run-mozharness' command. + diff --git a/taskcluster/docker/files/taskcluster-interactive-shell b/taskcluster/docker/files/taskcluster-interactive-shell new file mode 100644 index 000000000000..c782c0ea9600 --- /dev/null +++ b/taskcluster/docker/files/taskcluster-interactive-shell @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +download() { + name=`basename $1` + url=${GECKO_HEAD_REPOSITORY}/raw-file/${GECKO_HEAD_REV}/$1 + if ! curl --fail --silent -o ./$name --retry 10 $url; then + fail "failed downloading $1 from ${GECKO_HEAD_REPOSITORY}" + fi +} + +cd $HOME/bin; +download taskcluster/scripts/tester/run-wizard; +chmod +x run-wizard; +./run-wizard; + +SPAWN="$SHELL"; +if [ "$SHELL" = "bash" ]; then + SPAWN="bash -li"; +fi; + +cd $HOME; +exec $SPAWN; diff --git a/taskcluster/docker/recipes/android-test-system-setup.sh b/taskcluster/docker/recipes/android-test-system-setup.sh new file mode 100644 index 000000000000..62bdf3aa124c --- /dev/null +++ b/taskcluster/docker/recipes/android-test-system-setup.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +set -ve + +test "$(whoami)" == 'root' + +mkdir -p /setup +cd /setup + +# To speed up docker image build times as well as number of network/disk I/O +# build a list of packages to be installed nad call it in one go. +apt_packages=() + +# Core utilities +apt_packages+=('curl') +apt_packages+=('git') +apt_packages+=('libdbus-glib-1-2') +apt_packages+=('libgtk-3-0') +apt_packages+=('libx11-6') +apt_packages+=('libxcb1') +apt_packages+=('libxcursor1') +apt_packages+=('python-dev-is-python2') +apt_packages+=('python2') +apt_packages+=('python3-pip') +apt_packages+=('qemu-kvm') +apt_packages+=('screen') +apt_packages+=('sudo') +apt_packages+=('unzip') +apt_packages+=('uuid') +apt_packages+=('wget') +apt_packages+=('x11-utils') +apt_packages+=('xvfb') +apt_packages+=('xwit') +apt_packages+=('yasm') +apt_packages+=('zip') + + +# APT update takes very long on Ubuntu. Run it at the last possible minute. +apt-get update + +# This allows ubuntu-desktop to be installed without human interaction. +# Also force the cleanup after installation of packages to reduce image size. +export DEBIAN_FRONTEND=noninteractive +apt-get install -y -f "${apt_packages[@]}" + +# Clean up installation files. +rm -rf /var/lib/apt/lists/* + +# Ubuntu 20.04 officially deprecates Python 2, so this is the workaround. +curl https://bootstrap.pypa.io/get-pip.py --output get-pip.py +python get-pip.py + +# Install tooltool and mercurial now that dependencies are in place. +. /setup/common.sh +. /setup/install-mercurial.sh + +# Install specified versions of virtualenv and zstandard. +pip3 install virtualenv==15.2.0 +pip3 install zstandard==0.13.0 + +# Clear apt cache one last time +rm -rf /var/cache/apt/archives + +# We don't need no docs! +rm -rf /usr/share/help /usr/share/doc /usr/share/man + +# Remove all locale files other than en_US.UTF-8 +rm -rf /usr/share/locale/ /usr/share/locale-langpack/ /usr/share/locales/ + +# Further cleanup +cd / +rm -rf /setup ~/.ccache ~/.cache ~/.npm +apt-get -y autoremove +apt-get clean +apt-get autoclean +rm -f "$0" \ No newline at end of file diff --git a/taskcluster/docker/recipes/install-mercurial.sh b/taskcluster/docker/recipes/install-mercurial.sh index afb55a46f00e..3db896f1ab95 100755 --- a/taskcluster/docker/recipes/install-mercurial.sh +++ b/taskcluster/docker/recipes/install-mercurial.sh @@ -13,7 +13,7 @@ if [ -f /etc/lsb-release ]; then # shellcheck disable=SC1091 . /etc/lsb-release - if [ "${DISTRIB_ID}" = "Ubuntu" ] && [[ "${DISTRIB_RELEASE}" = "16.04" || "${DISTRIB_RELEASE}" = "17.10" || "${DISTRIB_RELEASE}" = "18.04" ]] + if [ "${DISTRIB_ID}" = "Ubuntu" ] && [[ "${DISTRIB_RELEASE}" = "16.04" || "${DISTRIB_RELEASE}" = "17.10" || "${DISTRIB_RELEASE}" = "18.04" || "${DISTRIB_RELEASE}" = "20.04" ]] then HG_DEB=1 HG_DIGEST=54a215232a340139707a968b58943c2903a8297f0da32f96622d1acab47de6013a5b96d2ca4ba241b1fee142b4098a6cdd236b308a1657c31f42807d7385d327