Run dockerfile commands with correct user. (#12579)

This updates various things…

* make use o `--user` argument of pip and `$PYTHONUSERBASE` to install python libraries in the correct places where the `olympia` user has permissions
* Add a new `rootshell` command

Fixes #12366

* Correct if statement

* Use separate tag for easier testing

* Try running directly with user olympia

* Small cleanup

* Add docs

* Fix comment

* Add 'rootshell' command

* Build circleci for local dev

* Fix pip execution by using 'python -m' to fetch the correct pip

* Pick up upstream user/group for builds and implement PYTHONUSERBASE and PIP_USER

* Correctly pass USER_ID and GROUP_ID to Dockerfile when building

* Small cleanups

* Build our test branch

* Fall back to manual uploads for testing

* Cleanup

* Revert back to :latest tag

Dockerfile

@@ -2,8 +2,14 @@ FROM python:3.6-slim-stretch
 
 ENV PYTHONDONTWRITEBYTECODE=1
 
-# Run everything as olympia user, by default.
-USER olympia
+ARG GROUP_ID=1000
+ARG USER_ID=1000
+
+# Run all initial setup with root user. This is the default but mentioned here
+# for documentation.
+# We won't switch to the `olympia` user inside the dockerfile
+# but rather use the `user` option in docker-compose.yml instead
+USER root
 
 # Allow scripts to detect we're running in our own container
 RUN touch /addons-server-docker-container
@@ -56,9 +62,9 @@ RUN mkdir -p /usr/local/share/GeoIP \
  && gunzip -c /tmp/GeoLite2-Country.mmdb.gz > /usr/local/share/GeoIP/GeoLite2-Country.mmdb \
  && rm -f /tmp/GeoLite2-Country.mmdb.gz
 
+# Install `file` and `libmagic` from the `buster` repositories for an up-to-date
+# file-detection.
 RUN apt-get update && apt-get -t buster install -y \
-       # For an up-to-date `file` and `libmagic-dev` library for better file
-       # detection.
        file \
        libmagic-dev \
     && rm -rf /var/lib/apt/lists/*
@@ -74,15 +80,34 @@ ENV LC_ALL en_US.UTF-8
 COPY . /code
 WORKDIR /code
 
+RUN groupadd -g ${GROUP_ID} olympia
+RUN useradd -g ${GROUP_ID} -u ${USER_ID} -Md /deps/ olympia
+
+# Create /deps/ and move ownership over to `olympia` user so that
+# we can install things there
+# Also run `chown` on `/code/` which technically doesn't change permissions
+# on the host but ensures that the image knows about correct permissions.
+RUN mkdir /deps/ && chown -R olympia:olympia /deps/ /code/
+
 ENV PIP_BUILD=/deps/build/
 ENV PIP_CACHE_DIR=/deps/cache/
 ENV PIP_SRC=/deps/src/
+
+# Allow us to install all dependencies to the `olympia` users
+# home directory (which is `/deps/`)
+ENV PIP_USER=true
+ENV PYTHONUSERBASE=/deps
+
+# Make sure that installed binaries are accessible
+ENV PATH $PYTHONUSERBASE/bin:$PATH
+
 ENV NPM_CONFIG_PREFIX=/deps/
 ENV SWIG_FEATURES="-D__x86_64__"
 
-# Install all python requires
-RUN mkdir -p /deps/{build,cache,src}/ && \
-    ln -s /code/package.json /deps/package.json && \
+# From now on run everything with the `olympia` user by default.
+USER olympia
+
+RUN ln -s /code/package.json /deps/package.json && \
     make update_deps && \
     rm -rf /deps/build/ /deps/cache/
 

Makefile-docker

@@ -1,6 +1,11 @@
-export PIP_COMMAND=pip3
 export PYTHON_COMMAND=python3
 
+# As we're using user-local installs inside the docker-container we need
+# to be cautious about uprading pip and not confusing it with the
+# globally installed version. This will take `$PYTHONUSERBASE` and `$PIP_USER`
+# into account.
+# See https://github.com/pypa/pip/issues/7205
+export PIP_COMMAND=$(PYTHON_COMMAND) -m pip
 APP=src/olympia/
 
 NUM_ADDONS=10

Makefile-os

@@ -20,6 +20,10 @@ update_docker: ## update all the docker images
 shell: ## connect to a running addons-server docker shell
 	docker-compose exec web bash
 
+.PHONY: shell
+rootshell: ## connect to a running addons-server docker shell with root user
+	docker-compose exec --user root web bash
+
 %: ## This directs any other recipe (command) to the web container's make.
 	docker-compose exec web make $(MAKECMDGOALS) ARGS=$(ARGS)
 

docker-compose.override.yml

@@ -1,4 +1,4 @@
-version: "2.3"
+version: "2.4"
 
 services:
   nginx:

docker-compose.yml

@@ -1,4 +1,4 @@
-version: "2.3"
+version: "2.4"
 
 # If you're changing the &env mapping
 # please make sure to update tests/ui/docker-compose.selenium.yml
@@ -25,7 +25,7 @@ services:
     <<: *env
     image: addons/addons-server:latest
     command: supervisord -n -c /code/docker/supervisor-celery.conf
-    entrypoint: ./scripts/start-docker.sh
+    user: olympia
     volumes:
       - .:/code
     extra_hosts:

scripts/start-docker.sh

@@ -1,28 +0,0 @@
-#!/bin/bash
-# The current directory is a mounted volume, and is owned by the
-# user running Docker on the host machine under linux (inc docker-machine).
-#
-# We don't want to trample all over the contents of this directory
-# with files owned by root. So create a new user with the same UID,
-# and drop privileges before running any commands.
-
-# Get the numeric user ID of the current directory.
-uid=$(ls -nd . | awk '{ print $3 }')
-
-# Create an `olympia` user with that ID, and the current directory
-# as its home directory.
-if [[ $uid -eq 0 ]]; then
-    # Don't try and create a user with uid 0 since it will fail.
-    # Works around issue with docker for mac running containers
-    # as root and not the user. Instead we just create the olympia
-    # user since files will still be owned by the host's user
-    # due to the way osxfs works.
-    useradd -Md $(pwd) olympia
-else
-    useradd -Md $(pwd) -u $uid olympia
-fi
-
-echo "Starting with user: 'olympia' uid: $(id -u olympia)"
-
-# Switch to that user and execute our actual command.
-exec su olympia -c 'exec "$@"' sh -- "$@"