Bug 1648851 - Part 0: create android-test docker image based on Ubuntu 20.04 r=jmaher,gbrown

Changes:
  - set up a new minimal docker image for `android-em` tests.
  - reorganize commonly referenced files into a specific folder `taskcluster/docker/files` to reduce instances of same file being copy-pasted into directories.
  - add android-test docker build task to taskcluster.

Differential Revision: https://phabricator.services.mozilla.com/D82204

taskcluster/ci/docker-image/kind.yml

@@ -155,6 +155,8 @@ jobs:
         parent: debian10-base
         packages:
             - deb8-valgrind
+    android-test:
+        symbol: I(android-test)
     fetch:
         symbol: I(fetch)
         parent: debian10-raw

taskcluster/docker/android-test/Dockerfile

@@ -0,0 +1,87 @@
+FROM          ubuntu:focal-20200606
+MAINTAINER    Edwin Takahashi <egao@mozilla.com>
+
+# Create necessary directories and worker user account
+RUN     mkdir -p /builds && mkdir -p artifacts
+RUN     id worker || useradd -d /builds/worker -s /bin/bash -m worker
+WORKDIR /builds/worker
+
+# We need to declare all potentially cache volumes as caches. Also,
+# making high I/O paths volumes increase I/O throughput because of
+# AUFS slowness.
+VOLUME  /builds/worker/.cache
+VOLUME  /builds/worker/checkouts
+VOLUME  /builds/worker/tooltool-cache
+VOLUME  /builds/worker/workspace
+
+# In test.sh we accept START_VNC to start a vnc daemon.
+# Exposing this port allows it to work.
+EXPOSE  5900
+
+# %include python/mozbuild/mozbuild/action/tooltool.py
+ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /setup/tooltool.py
+
+# %include testing/mozharness/external_tools/robustcheckout.py
+ADD topsrcdir/testing/mozharness/external_tools/robustcheckout.py /usr/local/mercurial/robustcheckout.py
+
+# %include taskcluster/docker/recipes/hgrc
+COPY topsrcdir/taskcluster/docker/recipes/hgrc /etc/mercurial/hgrc.d/mozilla.rc
+
+# %include taskcluster/docker/recipes/common.sh
+ADD topsrcdir/taskcluster/docker/recipes/common.sh /setup/common.sh
+
+# %include taskcluster/docker/recipes/install-mercurial.sh
+ADD topsrcdir/taskcluster/docker/recipes/install-mercurial.sh /setup/install-mercurial.sh
+
+# Add wrapper scripts for xvfb allowing tasks to easily retry starting up xvfb
+# %include taskcluster/docker/recipes/xvfb.sh
+ADD topsrcdir/taskcluster/docker/recipes/xvfb.sh /builds/worker/scripts/xvfb.sh
+
+# %include taskcluster/scripts/run-task
+ADD topsrcdir/taskcluster/scripts/run-task /builds/worker/bin/run-task
+
+# %include taskcluster/scripts/misc/fetch-content
+ADD topsrcdir/taskcluster/scripts/misc/fetch-content /builds/worker/bin/fetch-content
+
+ADD topsrcdir/python/mozbuild/mozbuild/action/tooltool.py /builds/worker/scripts/tooltool.py
+
+# %include taskcluster/docker/recipes/android-test-system-setup.sh
+ADD topsrcdir/taskcluster/docker/recipes/android-test-system-setup.sh /setup/system-setup.sh
+RUN     bash /setup/system-setup.sh
+
+# %include taskcluster/scripts/tester/test-linux.sh
+ADD topsrcdir/taskcluster/scripts/tester/test-linux.sh /builds/worker/bin/test-linux.sh
+
+# Set variable normally configured at login, by the shells parent process, these
+# are taken from GNU su manual
+ENV     HOME            /builds/worker
+ENV     SHELL           /bin/bash
+ENV     USER            worker
+ENV     LOGNAME         worker
+ENV     HOSTNAME        taskcluster-worker
+ENV     LANG            en_US.UTF-8
+ENV     PATH            $PATH:/builds/worker/bin
+
+# Bug 1345105 - Do not run periodical update checks and downloads
+# %include taskcluster/docker/files/autostart/gnome-software-service.desktop
+ADD topsrcdir/taskcluster/docker/files/autostart/gnome-software-service.desktop /etc/xdg/autostart/gnome-software-service.desktop
+
+# Set execution and ownership privileges
+RUN     chmod +x bin/*; chown -R worker:worker /builds/worker
+
+# Set up first-run experience for interactive mode
+# %include taskcluster/docker/files/motd
+ADD topsrcdir/taskcluster/docker/files/motd /etc/taskcluster-motd
+# %include taskcluster/docker/files/taskcluster-interactive-shell
+ADD topsrcdir/taskcluster/docker/files/taskcluster-interactive-shell /bin/taskcluster-interactive-shell
+RUN     chmod +x /bin/taskcluster-interactive-shell
+
+# Ubuntu 18.04 (or any other GNOME3 based systems) needs dbus to have various
+# test function as expected. Use entrypoint to initialize dbus as root.
+# %include taskcluster/docker/files/dbus.sh
+ADD topsrcdir/taskcluster/docker/files/dbus.sh /usr/local/bin/dbus.sh
+RUN     chmod +x /usr/local/bin/dbus.sh
+ENTRYPOINT ["/usr/local/bin/dbus.sh"]
+
+# Set a default command useful for debugging
+CMD ["/bin/bash", "--login"]

taskcluster/docker/files/autostart/gnome-software-service.desktop

@@ -0,0 +1,9 @@
+[Desktop Entry]
+Type=Application
+Name=GNOME Software
+Exec=/usr/bin/gnome-software --gapplication-service
+OnlyShowIn=GNOME;Unity;
+X-Ubuntu-Gettext-Domain=gnome-software
+
+# Bug 1345105 - Do not run periodical update checks and downloads
+X-GNOME-Autostart-enabled=false

taskcluster/docker/files/dbus.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e
+
+/etc/init.d/dbus start 2>&1
+
+exec "${@}"

taskcluster/docker/files/motd

@@ -0,0 +1,6 @@
+Welcome to your taskcluster interactive shell! The regularly scheduled task
+has been paused to give you a chance to set up your debugging environment.
+
+For your convenience, the exact mozharness command needed for this task can
+be invoked using the 'run-mozharness' command.
+

taskcluster/docker/files/taskcluster-interactive-shell

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+download() {
+    name=`basename $1`
+    url=${GECKO_HEAD_REPOSITORY}/raw-file/${GECKO_HEAD_REV}/$1
+    if ! curl --fail --silent -o ./$name --retry 10 $url; then
+        fail "failed downloading $1 from ${GECKO_HEAD_REPOSITORY}"
+    fi
+}
+
+cd $HOME/bin;
+download taskcluster/scripts/tester/run-wizard;
+chmod +x run-wizard;
+./run-wizard;
+
+SPAWN="$SHELL";
+if [ "$SHELL" = "bash" ]; then
+  SPAWN="bash -li";
+fi;
+
+cd $HOME;
+exec $SPAWN;

taskcluster/docker/recipes/android-test-system-setup.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+set -ve
+
+test "$(whoami)" == 'root'
+
+mkdir -p /setup
+cd /setup
+
+# To speed up docker image build times as well as number of network/disk I/O
+# build a list of packages to be installed nad call it in one go.
+apt_packages=()
+
+# Core utilities
+apt_packages+=('curl')
+apt_packages+=('git')
+apt_packages+=('libdbus-glib-1-2')
+apt_packages+=('libgtk-3-0')
+apt_packages+=('libx11-6')
+apt_packages+=('libxcb1')
+apt_packages+=('libxcursor1')
+apt_packages+=('python-dev-is-python2')
+apt_packages+=('python2')
+apt_packages+=('python3-pip')
+apt_packages+=('qemu-kvm')
+apt_packages+=('screen')
+apt_packages+=('sudo')
+apt_packages+=('unzip')
+apt_packages+=('uuid')
+apt_packages+=('wget')
+apt_packages+=('x11-utils')
+apt_packages+=('xvfb')
+apt_packages+=('xwit')
+apt_packages+=('yasm')
+apt_packages+=('zip')
+
+
+# APT update takes very long on Ubuntu. Run it at the last possible minute.
+apt-get update
+
+# This allows ubuntu-desktop to be installed without human interaction.
+# Also force the cleanup after installation of packages to reduce image size.
+export DEBIAN_FRONTEND=noninteractive
+apt-get install -y -f "${apt_packages[@]}"
+
+# Clean up installation files.
+rm -rf /var/lib/apt/lists/*
+
+# Ubuntu 20.04 officially deprecates Python 2, so this is the workaround.
+curl https://bootstrap.pypa.io/get-pip.py --output get-pip.py
+python get-pip.py
+
+# Install tooltool and mercurial now that dependencies are in place.
+. /setup/common.sh
+. /setup/install-mercurial.sh
+
+# Install specified versions of virtualenv and zstandard.
+pip3 install virtualenv==15.2.0
+pip3 install zstandard==0.13.0
+
+# Clear apt cache one last time
+rm -rf /var/cache/apt/archives
+
+# We don't need no docs!
+rm -rf /usr/share/help /usr/share/doc /usr/share/man
+
+# Remove all locale files other than en_US.UTF-8
+rm -rf /usr/share/locale/   /usr/share/locale-langpack/     /usr/share/locales/
+
+# Further cleanup
+cd /
+rm -rf /setup ~/.ccache ~/.cache ~/.npm
+apt-get -y autoremove
+apt-get clean
+apt-get autoclean
+rm -f "$0"

taskcluster/docker/recipes/install-mercurial.sh

@@ -13,7 +13,7 @@ if [ -f /etc/lsb-release ]; then
     # shellcheck disable=SC1091
     . /etc/lsb-release
 
-    if [ "${DISTRIB_ID}" = "Ubuntu" ] && [[ "${DISTRIB_RELEASE}" = "16.04" || "${DISTRIB_RELEASE}" = "17.10" || "${DISTRIB_RELEASE}" = "18.04" ]]
+    if [ "${DISTRIB_ID}" = "Ubuntu" ] && [[ "${DISTRIB_RELEASE}" = "16.04" || "${DISTRIB_RELEASE}" = "17.10" || "${DISTRIB_RELEASE}" = "18.04" || "${DISTRIB_RELEASE}" = "20.04" ]]
     then
         HG_DEB=1
         HG_DIGEST=54a215232a340139707a968b58943c2903a8297f0da32f96622d1acab47de6013a5b96d2ca4ba241b1fee142b4098a6cdd236b308a1657c31f42807d7385d327