Update development Docker config

2018-03-09 14:52:22 -08:00 · 2018-03-09 14:52:22 -08:00 · cebfa8fefe
--- a/4
+++ b/4
@ -46,6 +46,6 @@ RUN chown -R 10001:10001 /app

 USER 10001

-ENTRYPOINT ["/usr/local/bin/gunicorn"]
+ENTRYPOINT ["/bin/bash", "/app/bin/run"]

-CMD ["mozaggregator.service:app", "-k", "gevent", "--bind", "0.0.0.0:5000"]
+CMD ["web"]
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@ -2,6 +2,7 @@ FROM python:2-slim

 ENV PYTHONUNBUFFERED=1 \
    PYTHONDONTWRITEBYTECODE=1 \
+    DEVELOPMENT=1 \
    POSTGRES_HOST=db \
    POSTGRES_USER=postgres \
    PORT=5000 \
@ -26,6 +27,13 @@ RUN apt-get update && \
    apt-get clean && \
    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

+# Install spark
+RUN wget https://d3kbcqa49mib13.cloudfront.net/spark-2.0.2-bin-hadoop2.7.tgz -O /opt/spark.tgz && \
+    tar -xzf /opt/spark.tgz -C /opt
+
+ENV SPARK_HOME=/opt/spark-2.0.2-bin-hadoop2.7 \
+    PYTHONPATH=/opt/spark-2.0.2-bin-hadoop2.7/python:/opt/spark-2.0.2-bin-hadoop2.7/python/lib/py4j-0.10.3-src.zip
+
 # add a non-privileged user for installing and running the application
 RUN mkdir /app && \
    chown 10001:10001 /app && \
@ -42,19 +50,11 @@ RUN pip install --upgrade pip && \

 # Switch back to home directory
 WORKDIR /app
-
 COPY . /app
-
-# Install spark
-RUN wget https://d3kbcqa49mib13.cloudfront.net/spark-2.0.2-bin-hadoop2.7.tgz -O /opt/spark.tgz && \
-    tar -xzf /opt/spark.tgz -C /opt
-
-ENV SPARK_HOME=/opt/spark-2.0.2-bin-hadoop2.7 \
-    PYTHONPATH=/opt/spark-2.0.2-bin-hadoop2.7/python:/opt/spark-2.0.2-bin-hadoop2.7/python/lib/py4j-0.10.3-src.zip
-
 RUN chown -R 10001:10001 /app

 USER 10001

-# docker-compose will invoke this container to run tests
-CMD ["/bin/bash", "run-tests.sh"]
+ENTRYPOINT ["/bin/bash", "/app/bin/run"]
+
+CMD ["web-dev"]
--- a/8
+++ b/8
@ -5,7 +5,7 @@ help:
 	@echo "The list of commands for local development:\n"
 	@echo "  build      Builds the docker images for the docker-compose setup"
 	@echo "  clean      Stops and removes all docker containers"
-	@echo "  test       Cleans, builds, and runs tests"
+	@echo "  test       Runs the Python test suite"
 	@echo "  shell      Opens a Bash shell"
 	@echo "  up         Builds and runs the whole stack"
 	@echo "  stop       Stops the docker containers"
@ -17,10 +17,10 @@ clean: stop
 	docker-compose rm -f

 shell: build
-	docker-compose run web bash
+	docker-compose run --service-ports web bash

 test:
-	docker-compose up --build --abort-on-container-exit test
+	docker-compose run web test

 stop:
 	docker-compose down
@ -28,4 +28,4 @@ stop:

 up: clean
 	docker-compose pull
-	docker-compose up --build -d web
+	docker-compose up --build web
--- a/README.md
+++ b/README.md
@ -28,7 +28,8 @@ make test

 To manually run tests on running web container:
 ```
-ssh into container and ./run-tests.sh
+make shell
+./bin/run test
 ```

 ## Deployment
@ -40,19 +41,19 @@ Aggregates are made available through a HTTP API. There are two kinds of aggrega

 To access the aggregates use the ```aggregates_by/build_id/``` and ```aggregates_by/submission_date/``` prefix respectively.

-In the URLs below, replace `SERVICE` with the origin of this service's instance. The official service is `http://aggregates.telemetry.mozilla.org`.
+In the URLs below, replace `SERVICE` with the origin of this service's instance. The official service is `https://aggregates.telemetry.mozilla.org`.

 The following examples are based on build-id aggregates. Replace `build_id` with `submission_date` to use aggregates per submission date instead.

 ##### Get available channels:
 ```bash
-curl -X GET http://SERVICE/aggregates_by/build_id/channels/
+curl -X GET https://SERVICE/aggregates_by/build_id/channels/
 ["nightly","beta","aurora"]
 ```

 ##### Get a list of options for the available dimensions on a given channel and version:
 ```bash
-curl -X GET "http://SERVICE/filters/?channel=nightly&version=42"
+curl -X GET "https://SERVICE/filters/?channel=nightly&version=42"
 {"metric":["A11Y_CONSUMERS","A11Y_IATABLE_USAGE_FLAG",...], 
 "application":["Fennec","Firefox"],
 ...}
@ -60,13 +61,13 @@ curl -X GET "http://SERVICE/filters/?channel=nightly&version=42"

 ##### Get a list of available build-ids for a given channel:
 ```bash
-curl -X GET "http://SERVICE/aggregates_by/build_id/channels/nightly/dates/"
+curl -X GET "https://SERVICE/aggregates_by/build_id/channels/nightly/dates/"
 [{"date":"20150630","version":"42"}, {"date":"20150629","version":"42"}]
 ```

 ##### Given a set of build-ids, retrieve for each of build-id the aggregated histogram that complies with the requested filters:
 ```bash
-curl -X GET "http://SERVICE/aggregates_by/build_id/channels/nightly/?version=41&dates=20150615,20150616&metric=GC_MS&os=Windows_NT"
+curl -X GET "https://SERVICE/aggregates_by/build_id/channels/nightly/?version=41&dates=20150615,20150616&metric=GC_MS&os=Windows_NT"
 {"buckets":[0, ..., 10000],
 "data":[{"date":"20150615",
          "count":239459,
--- a/bin/run
+++ b/bin/run
@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+# default variables
+: "${PORT:=5000}"
+
+usage() {
+  echo "usage: bin/run web|web-dev|test"
+  exit 1
+}
+
+[ $# -lt 1 ] && usage
+
+# Only wait for backend services in development.
+# http://stackoverflow.com/a/13864829
+[ ! -z ${DEVELOPMENT+check} ] && ./bin/wait-for-it.sh db:5432 --timeout=0 --strict
+
+case $1 in
+  web)
+    # TODO: Use newrelic:
+    # exec newrelic-admin run-program gunicorn mozaggregator.service:app -b 0.0.0.0:${PORT} -k gevent --workers 4 --access-logfile -
+    exec gunicorn mozaggregator.service:app -b 0.0.0.0:${PORT} -k gevent --workers 4 --access-logfile -
+    ;;
+  web-dev)
+    exec python mozaggregator/service.py 0.0.0.0:${PORT}
+    ;;
+  test)
+    nosetests --with-coverage --cover-package=mozaggregator ./tests/*.py || exit 1
+
+    if [[ ! -z ${CI+check} ]]; then
+      echo "TODO: Set up codecov."
+      # bash <(curl -s https://codecov.io/bash) -s /tmp
+    else
+      coverage report -m
+    fi
+    ;;
+  *)
+    exec "$@"
+    ;;
+esac
--- a/bin/wait-for-it.sh
+++ b/bin/wait-for-it.sh
@ -0,0 +1,161 @@
+#!/usr/bin/env bash
+#   Use this script to test if a given TCP host/port are available
+
+cmdname=$(basename $0)
+
+echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $cmdname host:port [-s] [-t timeout] [-- command args]
+    -h HOST | --host=HOST       Host or IP under test
+    -p PORT | --port=PORT       TCP port under test
+                                Alternatively, you specify the host and port as host:port
+    -s | --strict               Only execute subcommand if the test succeeds
+    -q | --quiet                Don't output any status messages
+    -t TIMEOUT | --timeout=TIMEOUT
+                                Timeout in seconds, zero for no timeout
+    -- COMMAND ARGS             Execute command with args after the test finishes
+USAGE
+    exit 1
+}
+
+wait_for()
+{
+    if [[ $TIMEOUT -gt 0 ]]; then
+        echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT"
+    else
+        echoerr "$cmdname: waiting for $HOST:$PORT without a timeout"
+    fi
+    start_ts=$(date +%s)
+    while :
+    do
+        (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1
+        result=$?
+        if [[ $result -eq 0 ]]; then
+            end_ts=$(date +%s)
+            echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds"
+            break
+        fi
+        sleep 1
+    done
+    return $result
+}
+
+wait_for_wrapper()
+{
+    # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+    if [[ $QUIET -eq 1 ]]; then
+        timeout $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT &
+    else
+        timeout $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT &
+    fi
+    PID=$!
+    trap "kill -INT -$PID" INT
+    wait $PID
+    RESULT=$?
+    if [[ $RESULT -ne 0 ]]; then
+        echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT"
+    fi
+    return $RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        *:* )
+        hostport=(${1//:/ })
+        HOST=${hostport[0]}
+        PORT=${hostport[1]}
+        shift 1
+        ;;
+        --child)
+        CHILD=1
+        shift 1
+        ;;
+        -q | --quiet)
+        QUIET=1
+        shift 1
+        ;;
+        -s | --strict)
+        STRICT=1
+        shift 1
+        ;;
+        -h)
+        HOST="$2"
+        if [[ $HOST == "" ]]; then break; fi
+        shift 2
+        ;;
+        --host=*)
+        HOST="${1#*=}"
+        shift 1
+        ;;
+        -p)
+        PORT="$2"
+        if [[ $PORT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --port=*)
+        PORT="${1#*=}"
+        shift 1
+        ;;
+        -t)
+        TIMEOUT="$2"
+        if [[ $TIMEOUT == "" ]]; then break; fi
+        shift 2
+        ;;
+        --timeout=*)
+        TIMEOUT="${1#*=}"
+        shift 1
+        ;;
+        --)
+        shift
+        CLI="$@"
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$HOST" == "" || "$PORT" == "" ]]; then
+    echoerr "Error: you need to provide a host and port to test."
+    usage
+fi
+
+TIMEOUT=${TIMEOUT:-15}
+STRICT=${STRICT:-0}
+CHILD=${CHILD:-0}
+QUIET=${QUIET:-0}
+
+if [[ $CHILD -gt 0 ]]; then
+    wait_for
+    RESULT=$?
+    exit $RESULT
+else
+    if [[ $TIMEOUT -gt 0 ]]; then
+        wait_for_wrapper
+        RESULT=$?
+    else
+        wait_for
+        RESULT=$?
+    fi
+fi
+
+if [[ $CLI != "" ]]; then
+    if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then
+        echoerr "$cmdname: strict mode, refusing to execute subprocess"
+        exit $RESULT
+    fi
+    exec $CLI
+else
+    exit $RESULT
+fi
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -5,25 +5,15 @@ services:
    image: postgres:9.4

  web:
-    build:
-      context: .
-      dockerfile: Dockerfile
-    ports:
-      - "8000:5000"
-    depends_on:
-      - db
-    links:
-      - db
-
-  test:
    build:
      context: .
      dockerfile: Dockerfile.dev
    ports:
-      - "8000:5000"
+      - "5000:5000"
    volumes:
-      - .:/app
+      - $PWD:/app
    depends_on:
      - db
    links:
      - db
+    command: web-dev
--- a/run-tests.sh
+++ b/run-tests.sh
@ -1,12 +0,0 @@
-#!/bin/bash
-set -eo pipefail
-
-if [ -z "$SPARK_HOME" ]; then
-    echo 'You need to set $SPARK_HOME to run these tests.' >&2
-    exit 1
-fi
-
-python "$(which nosetests)" --with-coverage --cover-package=mozaggregator ./tests/*.py || exit 1
-
-# Show a text coverage report after a test run.
-coverage report -m