Fixes bug 1376813 - THIS... IS... DOCKER! (#623)

* Fixes bug 1376813 - THIS... IS... DOCKER! [WIP]

* Review comments, and fixed some issues.

* Added a first documentation.

* Fixed requirements.

* Added instruction to load a dump file into postgres.

* Added commands to run tests, and related documentation.

* Updated documentation for documentation and dependencies.

* Remove and create pontoon database before loading a dump.

* review comments.

* Fixed documentation.

.dockerignore

@@ -0,0 +1,7 @@
+.git
+**/*.swp
+**/*.pyc
+__pycache__
+.cache
+venv
+static

.gitignore

@@ -28,3 +28,6 @@ tmp/*
 docs/_build
 venv
 /media/
+
+# For docker
+.docker-build

Makefile

@@ -0,0 +1,34 @@
+DC := $(shell which docker-compose)
+
+.PHONY: dockerbuild dockersetup dockerclean dockertest dockertestshell dockerrun
+
+all: dockerrun
+
+.docker-build:
+	make dockerbuild
+
+dockerbuild:
+	${DC} build base
+	${DC} build webapp
+	touch .docker-build
+
+dockersetup: .docker-build
+	${DC} run webapp /app/docker/set_up_webapp.sh
+
+dockerclean:
+	rm .docker-build
+
+dockertest:
+	./docker/run_tests_in_docker.sh ${ARGS}
+
+dockershell:
+	./docker/run_tests_in_docker.sh --shell
+
+dockerloaddb:
+	-${DC} stop webapp
+	-docker exec -i `${DC} ps -q postgresql` dropdb -U pontoon pontoon
+	docker exec -i `${DC} ps -q postgresql` createdb -U pontoon pontoon
+	docker exec -i `${DC} ps -q postgresql` pg_restore -U pontoon -d pontoon -O < ${DB_DUMP_FILE}
+
+dockerrun:
+	${DC} up webapp

docker-compose.yml

@@ -0,0 +1,44 @@
+# docker-compose for Pontoon development.
+#
+# Note: Requires docker-compose 1.10+.
+version: "2"
+services:
+  # Pontoon base image
+  # This image handles user creation and permissions, so that code can be
+  # shared from your local folder into the docker container.
+  base:
+    build:
+      context: .
+      dockerfile: ./docker/Dockerfile.base
+    image: local/pontoon_base
+
+  # Webapp app
+  webapp:
+    build:
+      context: .
+      dockerfile: ./docker/Dockerfile
+    image: local/pontoon
+    env_file:
+      - docker/config/webapp.env
+    environment:
+      - LOCAL_USER_ID=1000
+    depends_on:
+      - postgresql
+    command: ["/app/docker/run_webapp.sh"]
+    ports:
+      - "8000:8000"
+    volumes:
+      - .:/app
+
+  # -----------------------------
+  # External services
+  # -----------------------------
+
+  # https://hub.docker.com/_/postgres/
+  postgresql:
+    image: postgres:9.4
+    environment:
+      # Create the superuser account
+      - POSTGRES_USER=pontoon
+      - POSTGRES_PASSWORD=asdf
+      - POSTGRES_DB=pontoon

docker/Dockerfile

@@ -0,0 +1,22 @@
+FROM local/pontoon_base
+
+WORKDIR /app
+
+# Install OS-level things
+COPY docker/set_up_ubuntu.sh /app/
+RUN DEBIAN_FRONTEND=noninteractive /app/set_up_ubuntu.sh
+
+# Install Socorro Python requirements
+COPY requirements.txt /app/requirements.txt
+COPY requirements-dev.txt /app/requirements-dev.txt
+RUN pip install -U 'pip>=8'
+RUN pip install --no-cache-dir --require-hashes -r requirements-dev.txt
+
+COPY . /app/
+COPY ./docker/config/webapp.env /app/.env
+
+ENV PYTHONUNBUFFERED 1
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONPATH /app
+
+CMD ["/app/docker/run_webapp.sh"]

docker/Dockerfile.base

@@ -0,0 +1,16 @@
+FROM python:2.7.13-slim
+
+RUN apt-get update && apt-get -y --no-install-recommends install \
+    ca-certificates \
+    curl
+
+RUN gpg --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4
+RUN curl -o /usr/local/bin/gosu -SL "https://github.com/tianon/gosu/releases/download/1.10/gosu-$(dpkg --print-architecture | awk -F- '{ print $NF }')" \
+    && curl -o /usr/local/bin/gosu.asc -SL "https://github.com/tianon/gosu/releases/download/1.10/gosu-$(dpkg --print-architecture | awk -F- '{ print $NF }').asc" \
+    && gpg --verify /usr/local/bin/gosu.asc \
+    && rm /usr/local/bin/gosu.asc \
+    && chmod +x /usr/local/bin/gosu
+
+COPY docker/entrypoint.sh /usr/local/bin/entrypoint.sh
+
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

docker/config/webapp.env

@@ -0,0 +1,10 @@
+SECRET_KEY=insert_random_key
+DJANGO_DEV=True
+DJANGO_DEBUG=True
+DATABASE_URL=postgres://pontoon:asdf@postgresql/pontoon
+SESSION_COOKIE_SECURE=False
+SITE_URL=http://localhost:8000
+FXA_CLIENT_ID=2651b9211a44b7b2
+FXA_SECRET_KEY=a3cafccbafe39db54f2723f8a6f804c337e362950f197b5b33050d784129d570
+FXA_OAUTH_ENDPOINT=https://oauth-stable.dev.lcip.org/v1
+FXA_PROFILE_ENDPOINT=https://stable.dev.lcip.org/profile/v1

docker/entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Add local user
+# Either use the LOCAL_USER_ID if passed in at runtime or
+# fallback
+
+USER_ID=${LOCAL_USER_ID:-10001}
+
+echo "Starting with UID : $USER_ID"
+useradd --shell /bin/bash -u $USER_ID -o -c "" -m app
+export HOME=/home/app
+
+exec /usr/local/bin/gosu app "$@"

docker/run_tests_in_docker.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+# Script that sets up the docker environment to run the tests in and runs the
+# tests.
+
+# Pass --shell to run a shell in the test container.
+
+# Failures should cause setup to fail.
+set -v -e -x
+
+DC="$(which docker-compose)"
+
+# Use the same image we use for building docker images because it'll be cached
+# already.
+BASEIMAGENAME="local/pontoon_base"
+
+# Start services in background (this is idempotent).
+echo "Starting services in the background..."
+${DC} up -d postgresql
+
+# If we're running a shell, then we start up a test container with . mounted
+# to /app.
+if [ "$1" == "--shell" ]; then
+    echo "Running shell..."
+
+    docker run \
+           --rm \
+           --volume "$(pwd)":/app \
+           --workdir /app \
+           --network pontoon_default \
+           --link "${DC} ps -q postgresql" \
+           --env-file ./docker/config/webapp.env \
+           -e LOCAL_USER_ID=$UID \
+           --tty \
+           --interactive \
+           local/pontoon /bin/bash
+
+else
+    # Create a data container to hold the repo directory contents and copy the
+    # contents into it.
+    if [ "$(docker container ls --all | grep pontoon-tests)" != "" ]; then
+        echo "Removing previously existing container"
+        docker rm pontoon-tests
+    fi
+
+    echo "Creating pontoon-tests container..."
+    docker create \
+           --volume "$(pwd)":/app \
+           -e LOCAL_USER_ID=$UID \
+           --name pontoon-tests \
+           ${BASEIMAGENAME} /bin/true
+
+    # Run cmd in that environment and then remove the container.
+    echo "Running tests..."
+    docker run \
+           --rm \
+           --volumes-from pontoon-tests \
+           --workdir /app \
+           --network pontoon_default \
+           --link "${DC} ps -q postgresql" \
+           --env-file ./docker/config/webapp.env \
+           -e LOCAL_USER_ID=$UID \
+           local/pontoon \
+           python manage.py test
+
+    echo "Done!"
+fi

docker/run_webapp.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Prepares then runs the webapp.
+
+echo "Install latest npm modules"
+cd /app
+npm install
+
+echo "Setting up the db for Django"
+python manage.py migrate
+
+echo "Running webapp. Connect with browser using http://localhost:8000/ ."
+python manage.py runserver 0.0.0.0:8000

docker/set_up_ubuntu.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# Installs packages and other things in an Ubuntu Docker image.
+
+# Failures should cause setup to fail
+set -v -e -x
+
+# Update the operating system and install OS-level dependencies
+apt-get update
+
+# Install packages for building python packages, postgres, lxml, sasl, and cffi
+apt-get install -y git nodejs-legacy npm libxml2-dev libxslt1-dev libmemcached-dev \
+    postgresql-server-dev-9.4 postgresql-client-9.4

docker/set_up_webapp.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+# Runs steps needed only once for the webapp.
+
+cd /app
+
+echo "Creating a new django superuser"
+python manage.py createsuperuser
+
+echo "Updating Firefox Account provider settings"
+python manage.py updatefxaprovider
+
+echo "Running migration of the pontoon-intro module"
+python manage.py sync_projects --projects=pontoon-intro --no-commit

docs/dev/install-docker.rst

@@ -0,0 +1,92 @@
+Developer Setup with Docker
+===========================
+
+The following describes how to set up an instance of the site on your
+computer for development with Docker and docker-compose.
+
+   .. Warning::
+
+    These installation steps are made for development only. It is not
+    recommended to run Pontoon via Docker in production.
+
+   .. Note::
+
+    Installation via Docker is still a fresh feature. Problems might happen,
+    bugs might be met. If you have any troubles with it, please
+    `file a bug <https://bugzilla.mozilla.org/enter_bug.cgi?product=Webtools&component=Pontoon>`_
+    or `propose a patch <https://github.com/mozilla/pontoon>`_!
+
+Prerequisites
+-------------
+
+1. Install `Docker <https://docs.docker.com/engine/installation/>`_.
+
+2. Install `docker-compose <https://docs.docker.com/compose/install/>`_. You need
+   1.10 or higher.
+
+3. Install `make <https://www.gnu.org/software/make/>`_ using either your
+   system's package manager (Linux) or homebrew (OSX). On Windows, you can use
+   `MozillaBuild <https://wiki.mozilla.org/MozillaBuild>`_.
+
+Quickstart
+----------
+
+1. From the root of this repository, run::
+
+     $ make dockerbuild
+
+   That will build the containers required for development: base and
+   webapp.
+
+2. Then run the webapp::
+
+      $ make dockerrun
+
+   .. Note::
+
+        The first time you run this, the PostgreSQL container needs to do
+        some work before it becomes available to the webapp container. Hence,
+        the webapp might not be able to perform things like migrations.
+        You can simply wait for the postgresql container to report that it's
+        ready, then abort the process, then restart it. That should let the
+        webapp do all its setup as expected.
+
+        Alternatively, you can run ``docker-compose up postgresql`` and wait
+        until it reports that the database is ready, then stop that and run
+        ``make dockerrun``.
+
+3. Finally you need to run some setup steps, while the webapp is running::
+
+      $ make dockersetup
+
+   This will ask you to create a superuser, and then will update your Firefox
+   account settings.
+
+At that point, you're good to go. Access the webapp via this URL: http://localhost:8000/
+
+Database
+--------
+
+By default, you will have default data loaded for only the Pontoon Intro project.
+If you have a database dump, you can load it into your PostgreSQL database by running::
+
+    $ make dockerloaddb DB_DUMP_FILE=path/to/my/dump
+
+Note that your database container needs to be running while you do that. You
+can start just the postgresql container by runnin::
+
+    $ docker-compose run postgresql
+
+Running tests
+-------------
+
+To run the entire test suite, simply run::
+
+    $ make dockertest
+
+If you want to run only some unit tests, or want to avoid rebuilding the
+docker container every time you run tests, you can start a shell that will
+allow to run your own commands. To do that, run::
+
+    $ make dockershell
+    app@675af05d66ae:/app$ python manage.py test

docs/dev/workflow.rst

@@ -35,12 +35,22 @@ steps, as they don't affect your setup if nothing has changed:
 
 Building the Documentation
 --------------------------
+
+Before you can build the documentation, you will need to install a few
+dependencies. It is recommended to do that in a Python virtual environment:
+
+.. code-block:: bash
+
+    cd docs
+    virtualenv venv
+    source venv/bin/activate
+    pip install --require-hashes -r requirements.txt
+
 You can build the documentation with the following command:
 
 .. code-block:: bash
 
-   # Enter the docs/ subdirectory
-   cd docs
+   # From the docs/ subdirectory
    make html
 
 After running this command, the documentation should be available at
@@ -55,13 +65,14 @@ After running this command, the documentation should be available at
 
 Adding New Dependencies
 -----------------------
-Pontoon uses peep_ to install dependencies. Peep is a wrapper around pip that
+
+Pontoon uses pip_ to install dependencies, and since version 8, that tool
 checks downloaded packages to ensure they haven't been tampered with.
 
 Because of this, adding a new library to ``requirements.txt`` is a bit more work
 as you need to add hashes for each library you want to install. To help make
-this easier, you can use the peepin_ tool to add new dependencies to the
+this easier, you can use the hashin_ tool to add new dependencies to the
 requirements file.
 
-.. _peep: https://github.com/erikrose/peep/
-.. _peepin: https://github.com/peterbe/peepin/
+.. _pip: https://pip.pypa.io/
+.. _hashin: https://github.com/peterbe/hashin/

docs/index.rst

@@ -29,6 +29,7 @@ Contents
    :maxdepth: 2
 
    dev/install
+   dev/install-docker
    dev/fluent
    dev/workflow
    dev/sync

docs/requirements.txt

@@ -1,7 +1,3 @@
-# Temporarily we had to remove support for the secure dependencies, because
-# readthedocs.org don't support pip8.
-# More info at https://github.com/rtfd/readthedocs.org/issues/1843
-
 sphinx-rtd-theme==0.1.8 \
     --hash=sha256:ff9b29268824096fc9113904605d83f6aa817102c0f170115b4b39f0ae0651f3 \
     --hash=sha256:74f633ed3a61da1d1d59c3185483c81a9d7346bf0e7b5f29ad0764a6f159b68a

requirements.txt

@@ -2,8 +2,6 @@
 # readthedocs.org don't support pip8.
 # More info at https://github.com/rtfd/readthedocs.org/issues/1843
 
--r docs/requirements.txt
-
 # Dependencies required by the pontoon to work in the production environment.
 argparse==1.3.0 \
     --hash=sha256:d01978e9a8c51ea7b34ec71f3fced1326437cd366b9c2e19265ace050924e0e2 \
@@ -151,3 +149,21 @@ django-notifications-hq==1.2 \
     --hash=sha256:4a7afdc506a15aad89434fb9fd41b611ba2a75bc0f5ba15179b5998c5181a703
 django-model-utils==2.0.3 \
     --hash=sha256:fae8d517590d99730ed3cfc3fdf9378e0b3815325a8911b963241ee8e9761fd7
+Jinja2==2.8 \
+    --hash=sha256:1cc03ef32b64be19e0a5b54578dd790906a34943fe9102cfdae0d4495bd536b4 \
+    --hash=sha256:bc1ff2ff88dbfacefde4ddde471d1417d3b304e8df103a7a9437d47269201bf4
+MarkupSafe==0.23 \
+    --hash=sha256:a4ec1aff59b95a14b45eb2e23761a0179e98319da5a7eb76b56ea8cdc7b871c3
+pytz==2017.2 \
+    --hash=sha256:c883c2d6670042c7bc1688645cac73dd2b03193d1f7a6847b6154e96890be06d \
+    --hash=sha256:03c9962afe00e503e2d96abab4e8998a0f84d4230fa57afe1e0528473698cdd9 \
+    --hash=sha256:487e7d50710661116325747a9cd1744d3323f8e49748e287bc9e659060ec6bf9 \
+    --hash=sha256:43f52d4c6a0be301d53ebd867de05e2926c35728b3260157d274635a0a947f1c \
+    --hash=sha256:d1d6729c85acea5423671382868627129432fba9a89ecbb248d8d1c7a9f01c67 \
+    --hash=sha256:54a935085f7bf101f86b2aff75bd9672b435f51c3339db2ff616e66845f2b8f9 \
+    --hash=sha256:39504670abb5dae77f56f8eb63823937ce727d7cdd0088e6909e6dcac0f89043 \
+    --hash=sha256:ddc93b6d41cfb81266a27d23a79e13805d4a5521032b512643af8729041a81b4 \
+    --hash=sha256:f5c056e8f62d45ba8215e5cb8f50dfccb198b4b9fbea8500674f3443e4689589
+six==1.10.0 \
+    --hash=sha256:0ff78c403d9bccf5a425a6d31a12aa6b47f1c21ca4dc2573a7e2f32a97335eb1 \
+    --hash=sha256:105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a