Add initial skeleton for job

Dockerfile

@@ -0,0 +1,43 @@
+FROM python:3
+MAINTAINER Frank Bertsch <frank@mozilla.com>
+
+ARG APP_NAME=leanplum_data_export
+ENV APP_NAME=${APP_NAME}
+
+# Guidelines here: https://github.com/mozilla-services/Dockerflow/blob/master/docs/building-container.md
+ARG USER_ID="10001"
+ARG GROUP_ID="app"
+ARG HOME="/app"
+
+ENV HOME=${HOME}
+RUN groupadd --gid ${USER_ID} ${GROUP_ID} && \
+    useradd --create-home --uid ${USER_ID} --gid ${GROUP_ID} --home-dir /app ${GROUP_ID}
+
+# List packages here
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        file        \
+        gcc         \
+        libwww-perl && \
+    apt-get autoremove -y && \
+    apt-get clean
+
+# Upgrade pip
+RUN pip install --upgrade pip
+
+WORKDIR ${HOME}
+
+ADD requirements requirements/
+RUN pip install -r requirements/requirements.txt
+RUN pip install -r requirements/test_requirements.txt
+
+ADD . ${HOME}/${APP_NAME}
+ENV PATH $PATH:${HOME}/${APP_NAME}/bin
+
+RUN pip install -e ${HOME}/${APP_NAME}
+
+# Drop root and change ownership of the application folder to the user
+RUN chown -R ${USER_ID}:${GROUP_ID} ${HOME}
+USER ${USER_ID}
+
+ENTRYPOINT ["entrypoint"]

Makefile

@@ -0,0 +1,50 @@
+.PHONY: help clean clean-pyc clean-build list test coverage release
+
+help:
+	@echo "  clean-build -          Remove build artifacts"
+	@echo "  clean-pyc -            Remove Python file artifacts"
+	@echo "  lint -                 Check style with flake8"
+	@echo "  test -                 Run tests quickly with the default Python"
+	@echo "  install-requirements - install the requirements for development"
+	@echo "  build                  Builds the docker images for the docker-compose setup"
+	@echo "  docker-rm              Stops and removes all docker containers"
+	@echo "  run                    Run a command. Can run scripts, e.g. make run COMMAND=\"./scripts/schema_generator.sh\""
+	@echo "  shell                  Opens a Bash shell"
+
+clean: clean-build clean-pyc docker-rm
+
+clean-build:
+	rm -fr build/
+	rm -fr dist/
+	rm -fr *.egg-info
+
+clean-pyc:
+	find . -name '*.pyc' -exec rm -f {} +
+	find . -name '*.pyo' -exec rm -f {} +
+	find . -name '*~' -exec rm -f {} +
+
+lint:
+	flake8 --max-line-length 100
+
+test:
+	docker-compose run app test
+
+install-requirements:
+	pip install -r requirements/requirements.txt
+	pip install -r requirements/test_requirements.txt
+
+build:
+	docker-compose build
+
+docker-rm: stop
+	docker-compose rm -f
+
+shell:
+	docker-compose run app bash
+
+run:
+	docker-compose run app $(COMMAND)
+
+stop:
+	docker-compose down
+	docker-compose stop

README.md

@@ -1 +1,37 @@
-# leanplum_data_export
+[![CircleCI](https://circleci.com/gh/mozilla/leanplum_data_export.svg?style=svg)](https://circleci.com/gh/mozilla/leanplum_data_export)
+
+# Leanplum Data Export
+This repository is the job to export Mozilla data from Leanplum and into BQ.
+
+## Development and Testing
+
+While iterating on development, we recommend using virtualenv
+to run the tests locally.
+
+### Run tests locally
+
+Install requirements locally:
+```
+python3 -m virtualenv venv
+source venv/bin/activate
+make install-requirements
+```
+
+Run tests locally:
+```
+pytest tests/
+```
+
+### Run tests in docker
+
+You can run the tests just as CI does by building the container
+and running the tests.
+
+```
+make clean && make build
+make test
+```
+
+### Deployment
+
+This project deploys automatically to Dockerhub. The latest release is used to run the job.

bin/entrypoint

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+# Use credentials from environment
+if [ -n "$GCLOUD_SERVICE_KEY" ]; then
+    # Google's client libraries will check for GOOGLE_APPLICATION_CREDENTIALS
+    # and use a file in that location for credentials if present;
+    # See https://cloud.google.com/docs/authentication/production
+    export GOOGLE_APPLICATION_CREDENTIALS="${GOOGLE_APPLICATION_CREDENTIALS:-/tmp/gcp.json}"
+    echo "$GCLOUD_SERVICE_KEY" > "$GOOGLE_APPLICATION_CREDENTIALS"
+fi
+
+if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ] && which gcloud > /dev/null; then
+    gcloud --quiet auth activate-service-account --key-file "$GOOGLE_APPLICATION_CREDENTIALS"
+    python -W ignore -c 'import google.auth; print("project = ", google.auth.default()[1])' >> ~/.config/gcloud/configurations/config_default
+    sed "1i--project_id=$(gcloud config get-value project)" -i ~/.bigqueryrc
+fi
+
+if [ "$1" = test ]; then
+    exec pytest "${@:2}"
+else
+    exec "$@"
+fi

docker-compose.yml

@@ -0,0 +1,12 @@
+version: '2'
+
+services:
+  app:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    restart: "no"
+    command: "true"
+    environment:
+      - GCLOUD_SERVICE_KEY
+      - GOOGLE_APPLICATION_CREDENTIALS

leanplum_data_export/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+__author__ = 'Frank Bertsch'
+__email__ = 'frank@mozilla.com'
+__version__ = '0.1.0'

leanplum_data_export/__main__.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import click
+import logging
+import sys
+
+from .application import App
+
+
+@click.command()
+def export_leanplum():
+    app = App()
+    app.export_leanplum()
+
+
+@click.group()
+def main(args=None):
+    """Command line utility"""
+    logging.basicConfig(stream=sys.stderr, level=logging.INFO)
+
+
+main.add_command(export_leanplum)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

leanplum_data_export/application.py

@@ -0,0 +1,8 @@
+
+class App(object):
+
+    def __init__(self):
+        pass
+
+    def get_hello_world(self):
+        return "Hello, World"

requirements/test_requirements.txt

@@ -0,0 +1 @@
+pytest==5.2.0

setup.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from setuptools import setup, find_packages
+
+readme = open('README.md').read()
+
+setup(
+    # Change these when cloning this repo!
+    name='leanplum_data_export',
+    description='A job to export leanplum data to BigQuery',
+    author='Frank Bertsch',
+    author_email='frank@mozilla.com',
+    url='https://github.com/mozilla/leanplum_data_export',
+    packages=find_packages(include=['leanplum_data_export']),
+    package_dir={'leanplum-data-export': 'leanplum_data_export'},
+    entry_points={
+        'console_scripts': [
+            'leanplum-data-export=leanplum_data_export.__main__:main',
+        ],
+    },
+
+    # These don't necessarily need changed
+    python_requires='>=3.6.0',
+    version='0.0.0',
+    long_description=readme,
+    include_package_data=True,
+    install_requires=[
+        'click',
+    ],
+    license='Mozilla',
+)

tests/test_application.py

@@ -0,0 +1,14 @@
+import pytest
+
+from leanplum_data_export.application import App
+
+
+@pytest.fixture
+def app():
+    return App()
+
+
+class TestApplication(object):
+
+    def test_return_value(self, app):
+        pass