Bug 1376808 - Package python_mozetl test infrastructure in docker

.travis.yml

@@ -1,22 +1,21 @@
-language: python
-python:
-    - "2.7"
+env:
+  global:
+    - CACHE_DIR=$HOME/.cache/docker
+    - MOZETL_CACHE=${CACHE_DIR}/mozetl.tar.gz
+
 cache:
-    directories:
-        - $HOME/spark
-addons:
-    apt:
-        packages:
-            - libsnappy-dev
+  directories:
+    - $CACHE_DIR
+
+services:
+  - docker
 before_install:
-    - export PATH=$HOME/.local/bin:$PATH
+  - if [ -f ${MOZETL_CACHE} ]; then gunzip -c ${MOZETL_CACHE}  | docker load; fi
 install:
-    - pip install tox-travis codecov
-    - "[ -f spark ] || mkdir spark && cd spark && wget https://d3kbcqa49mib13.cloudfront.net/spark-2.0.2-bin-hadoop2.7.tgz && cd .."
-    - tar -xf ./spark/spark-2.0.2-bin-hadoop2.7.tgz
-    - export SPARK_HOME=`pwd`/spark-2.0.2-bin-hadoop2.7
-    - export PYTHONPATH=${SPARK_HOME}/python/:$(echo ${SPARK_HOME}/python/lib/py4j-*-src.zip):${PYTHONPATH}
+  - docker build -t mozetl .
+  - mkdir -p $CACHE_DIR
+  - if [ ! -f ${MOZETL_CACHE}  ]; then docker save mozetl | gzip > ${MOZETL_CACHE}; fi
 script:
-    - tox
+  - docker run mozetl ./runtests.sh
 after_success:
-    - codecov
+  - codecov

Dockerfile

@@ -0,0 +1,21 @@
+FROM korekontrol/ubuntu-java-python2
+
+ENV SPARK_VERSION=2.0.2
+
+# install gcc
+RUN apt-get update && apt-get install -y g++ libpython-dev libsnappy-dev wget git
+
+# install spark
+RUN wget -nv https://d3kbcqa49mib13.cloudfront.net/spark-${SPARK_VERSION}-bin-hadoop2.7.tgz
+RUN tar -zxf spark-${SPARK_VERSION}-bin-hadoop2.7.tgz
+
+ENV SPARK_HOME="/spark-${SPARK_VERSION}-bin-hadoop2.7"
+ENV PYTHONPATH="${SPARK_HOME}/python/:${SPARK_HOME}/python/lib/py4j-*-src.zip):${PYTHONPATH}"
+
+RUN pip install tox python_moztelemetry
+
+WORKDIR /python_mozetl
+COPY . /python_mozetl
+
+# installing dependencies takes the majority of the time
+RUN pip install -e .[testing]

README.md

@@ -22,8 +22,17 @@ For more on this see the writeup at
 
 # Tests
 
-We use `tox` for testing.
-To run the tests, clone this repo, enter the project directory and run `tox`.
+To test this package locally, it is recommended to set up the environment and execute tests within a docker container.
+```
+docker build -t mozetl .
+./runtests.sh   # runs tests within docker container
+```
+
+A subset of tests can be specified by adding arguments to runtests.sh:
+```
+./runtests.sh -k tests/test_main.py # runs tests only in the test_main module
+```
+
 Tests are configured in [tox.ini](tox.ini)
 
 # Scheduling

runtests.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# abort immediately on any failure
+set -e
+
+# if we are not inside the docker container, run this command *inside* the
+# docker container
+if [ ! -f /.dockerenv ]; then
+    docker run -t -i -v $PWD:/python_mozetl mozetl ./runtests.sh "$@"
+    exit $?
+fi
+
+# Run tests
+if [ $# -gt 0 ]; then
+    ARGS="$@"
+    tox -- "${ARGS}"
+else
+    tox
+fi

setup.py

@@ -1,6 +1,18 @@
 #!/usr/bin/env python
 from setuptools import setup, find_packages
 
+test_deps = [
+    'coverage',
+    'pytest-cov',
+    'pytest-timeout',
+    'moto',
+    'mock',
+    'pytest',
+]
+
+extras = {
+    'testing': test_deps,
+}
 
 setup(
     name='mozetl',
@@ -17,4 +29,6 @@ setup(
         'requests',
         'arrow',
     ],
+    tests_require=test_deps,
+    extras_require=extras,
 )

tox.ini

@@ -10,17 +10,12 @@ envlist = py27, flake8
 addopts = --cov=mozetl tests/
 
 [testenv]
-sitepackages = True
-commands = pytest
+usedevelop = True
+extras = testing
+commands = pytest {posargs}
 passenv = PYTHONPATH
 setenv =
     SPARK_HOME = {env:SPARK_HOME}
-deps =
-    pytest
-    pytest-cov
-    coverage
-    moto
-    mock
 
 [testenv:flake8]
 basepython = python2.7