Initial Commit

Add a runnable generic python-docker image that uploads to Dockerhub. Tests can run in the container or not. This repository should be used as an example for future python applications that will run in GKE.
2019-10-07 11:32:32 -04:00 · 2019-10-07 11:32:32 -04:00 · 53ae175b39
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -0,0 +1,113 @@
+####################
+# CircleCI configuration reference:
+#   https://circleci.com/docs/2.0/configuration-reference
+####################
+
+version: 2
+
+#####################################################
+# Jobs: see https://circleci.com/docs/2.0/jobs-steps/
+#####################################################
+
+jobs:
+  build:
+    docker:
+      - image: docker:stable-git
+    working_directory: /dockerflow
+    steps:
+      - checkout
+      - setup_remote_docker
+
+      - run:
+          name: Create a version.json
+          command: |
+            # create a version.json per https://github.com/mozilla-services/Dockerflow/blob/master/docs/version_object.md
+            printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' \
+            "$CIRCLE_SHA1" \
+            "$CIRCLE_TAG" \
+            "$CIRCLE_PROJECT_USERNAME" \
+            "$CIRCLE_PROJECT_REPONAME" \
+            "$CIRCLE_BUILD_URL" > version.json
+      - run:
+          name: Build Docker image
+          command: docker build -t app:build .
+
+      # save the built docker container into CircleCI's cache. This is
+      # required since Workflows do not have the same remote docker instance.
+      - run:
+          name: docker save app:build
+          command: mkdir -p /cache; docker save -o /cache/docker.tar "app:build"
+      - save_cache:
+          key: v1-{{ .Branch }}-{{epoch}}
+          paths:
+            - /cache/docker.tar
+
+  test:
+    docker:
+      - image: docker:18.02.0-ce
+    steps:
+      - setup_remote_docker
+      - restore_cache:
+          key: v1-{{.Branch}}
+      - run:
+          name: Restore Docker image cache
+          command: docker load -i /cache/docker.tar
+      - run:
+          name: Test Code
+          command: make test
+
+  deploy:
+    docker:
+      - image: docker:18.02.0-ce
+    steps:
+      - setup_remote_docker
+      - restore_cache:
+          key: v1-{{.Branch}}
+      - run:
+          name: Restore Docker image cache
+          command: docker load -i /cache/docker.tar
+      - run:
+          name: Deploy to Dockerhub
+          command: |
+            echo $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
+            # deploy master
+            if [ "${CIRCLE_BRANCH}" == "master" ]; then
+              docker tag app:build ${DOCKERHUB_REPO}:latest
+              docker push ${DOCKERHUB_REPO}:latest
+            elif  [ ! -z "${CIRCLE_TAG}" ]; then
+            # deploy a release tag...
+              echo "${DOCKERHUB_REPO}:${CIRCLE_TAG}"
+              docker tag app:build "${DOCKERHUB_REPO}:${CIRCLE_TAG}"
+              docker images
+              docker push "${DOCKERHUB_REPO}:${CIRCLE_TAG}"
+            fi
+
+#########################################################
+# Workflows: see https://circleci.com/docs/2.0/workflows/
+#########################################################
+
+workflows:
+  version: 2
+  commit:
+    jobs:
+      - build:
+          filters:
+            tags:
+              only: /.*/
+
+      - test:
+          requires:
+            - build
+          filters:
+            tags:
+              only: /.*/
+
+      - deploy:
+          requires:
+            - build
+            - test
+          filters:
+            tags:
+              only: /.*/
+            branches:
+              only: master
--- a/39
+++ b/39
@ -0,0 +1,39 @@
+FROM python:3
+MAINTAINER Frank Bertsch <frank@mozilla.com>
+
+# Guidelines here: https://github.com/mozilla-services/Dockerflow/blob/master/docs/building-container.md
+ARG USER_ID="10001"
+ARG GROUP_ID="app"
+ARG HOME="/app"
+
+ENV HOME=${HOME}
+RUN groupadd --gid ${USER_ID} ${GROUP_ID} && \
+    useradd --create-home --uid ${USER_ID} --gid ${GROUP_ID} --home-dir /app ${GROUP_ID}
+
+# List packages here
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        file        \
+        gcc         \
+        libwww-perl && \
+    apt-get autoremove -y && \
+    apt-get clean
+
+# Upgrade pip
+RUN pip install --upgrade pip
+
+WORKDIR ${HOME}
+
+ADD requirements requirements/
+RUN pip install -r requirements/requirements.txt
+RUN pip install -r requirements/test_requirements.txt
+
+ADD . ${HOME}/python-application
+
+RUN pip install -e ${HOME}/python-application
+
+# Drop root and change ownership of the application folder to the user
+RUN chown -R ${USER_ID}:${GROUP_ID} ${HOME}
+USER ${USER_ID}
+
+ENTRYPOINT ["/app/python-application/bin/entrypoint"]
--- a/53
+++ b/53
@ -0,0 +1,53 @@
+.PHONY: help clean clean-pyc clean-build list test coverage release
+
+help:
+	@echo "  clean-build -          Remove build artifacts"
+	@echo "  clean-pyc -            Remove Python file artifacts"
+	@echo "  lint -                 Check style with flake8"
+	@echo "  test -                 Run tests quickly with the default Python"
+	@echo "  install-requirements - install the requirements for development"
+	@echo "  build                  Builds the docker images for the docker-compose setup"
+	@echo "  docker-rm              Stops and removes all docker containers"
+	@echo "  run                    Run a command. Can run scripts, e.g. make run COMMAND=\"./scripts/schema_generator.sh\""
+	@echo "  shell                  Opens a Bash shell"
+
+clean: clean-build clean-pyc docker-rm
+
+clean-build:
+	rm -fr build/
+	rm -fr dist/
+	rm -fr *.egg-info
+
+clean-pyc:
+	find . -name '*.pyc' -exec rm -f {} +
+	find . -name '*.pyo' -exec rm -f {} +
+	find . -name '*~' -exec rm -f {} +
+
+lint:
+	flake8 --max-line-length 100
+
+test:
+	docker-compose run app test
+
+install-requirements:
+	pip install -r requirements/requirements.txt
+	pip install -r requirements/test_requirements.txt
+
+build:
+	docker-compose build
+
+docker-rm: stop
+	docker-compose rm -f
+
+shell:
+	docker-compose run app bash
+
+run:
+	docker-compose run app $(COMMAND)
+
+stop:
+	docker-compose down
+	docker-compose stop
+
+hello-world:
+	docker-compose run app python-application hello-world
--- a/README.md
+++ b/README.md
@ -0,0 +1,151 @@
+[![CircleCI](https://circleci.com/gh/mozilla/mozilla-schema-generator/tree/master.svg?style=svg)](https://circleci.com/gh/mozilla/mozilla-schema-generator/tree/master)
+
+# Mozilla Schema Generator
+
+A library for generating full representations of Mozilla telemetry pings.
+
+See [Mozilla Pipeline Schemas](https://www.github.com/mozilla-services/mozilla-pipeline-services)
+for the more generic structure of pings. This library takes those generic structures and fills in
+all of the probes we expect to see in the appropriate places.
+
+## Telemetry Integration
+
+There are two pings we are targeting for integration with this library:
+
+1. [The Main Ping](http://gecko-docs.mozilla.org.s3.amazonaws.com/toolkit/components/telemetry/telemetry/data/main-ping.html)
+   is the historical Firefox Desktop ping, and contains many more than ten-thousand total pieces of data.
+2. [The Glean Ping](https://github.com/mozilla/glean_parser) is the new ping-type being created for
+   more generic data collection.
+
+This library takes the information for what should be in those pings from the [Probe Info Service](https://www.github.com/mozilla/probe-scraper).
+
+## Data Store Integration
+
+The primary use of the schemas is for integration with the
+[Schema Transpiler](https://www.github.com/mozilla/jsonschema-transpiler). 
+The schemas that this repository generates can be transpiled into Avro and Bigquery. They define
+the schema of the Avro and BigQuery tables that the [BQ Sink](https://www.github.com/mozilla/gcp-ingestion)
+writes to.
+
+### BigQuery Limitations and Splitting
+
+BigQuery has a hard limit of ten thousand columns on any single table. This library
+can take that limitation into account by splitting schemas into multiple tables. Each
+table has some common information that are duplicated in every table, and then a set
+of fields that are unique to that table. The join of these tables gives the full
+set of fields available from the ping.
+
+To decide on a table split, we include the `table_group` configuration in the configuration
+file. For example, `payload/histograms` has `table_group: histograms`; this indicates that
+there will be a table outputted with just histograms.
+
+Currently, generates tables for:
+- Histograms
+- Keyed Histograms
+- Scalars
+- Keyed Scalars
+- Everything else
+
+If a single table expands beyond 9000 columns, we move the new fields to the next table.
+For example, main_histograms_1 and main_histograms_2.
+
+Note: Tables are only split if the `--split` parameter is provided.
+
+## Validation
+
+A secondary use-case of these schemas is for validation. The schemas produced are guaranteed to
+be more correct, since they include explicit definitions of every metric and probe.
+
+## Usage
+
+### Main Ping
+
+Generate the Full Main Ping schema:
+
+```
+mozilla-schema-generator generate-main-ping
+```
+
+Generate the Main Ping schema divided among tables (for BigQuery):
+```
+mozilla-schema-generator generate-main-ping --split --out-dir main-ping
+```
+
+The `out-dir` parameter will be the namespace for the pings.
+
+To see a full list of options, run `mozilla-schema-generator generate-main-ping --help`.
+
+
+### Glean
+
+Generate all Glean ping schemas - one for each application, for each ping
+that application sends:
+
+```
+mozilla-schema-generator generate-glean-pings
+```
+
+Write schemas to a directory:
+```
+mozilla-schema-generator generate-glean-pings --out-dir glean-ping
+```
+
+To see a full list of options, run `mozilla-schema-generator generate-glean-pings --help`.
+
+
+## Configuration Files
+
+Configuration files are default found in `/config`. You can also specify your own when running the generator.
+
+Configuration files match certain parts of a ping to certain types of probes or metrics. The nesting
+of the config file matches the ping it is filling in. For example, Glean stores probe types under
+the `metrics` key, so the nesting looks like this:
+```
+{
+    "metrics": {
+        "string": {
+            <METRIC_ID>: {...}
+        }
+    }
+}
+```
+
+While the generic schema doesn't include information about the specific `<METRIC_ID>`s being included,
+the schema-generator does. To include the correct metrics that we would find in that section of the ping,
+we would organize the `config.yaml` file like this:
+
+```
+metrics:
+    string:
+        match:
+            type: string
+```
+
+The `match` key indicates that we should fill-in this section of the ping schema with metrics,
+and the `type: string` makes sure we only put string metrics in there. You can do an exact
+match on any field available in the ping info from the [probe-info-service](https://probeinfo.telemetry.mozilla.org/glean/glean/metrics),
+which also contains the [Desktop probes](https://probeinfo.telemetry.mozilla.org/firefox/all/main/all_probes).
+
+There are a few additional keywords allowable under any field:
+* `contains` - e.g. `process: contains: main`, indicates that the `process` field is an array
+  and it should only match those that include the entry `main`.
+* `not` - e.g. `send_in_pings: not: glean_ping_info`, indicates that we should match
+  any field for `send_in_pings` _except_ `glean_ping_info`.
+
+### `table_group` Key
+
+This specific field is for indicating which table group that section of the ping should be included in when
+splitting the schema. Currently we do not split the Glean ping, only the Main. See the section on [BigQuery
+Limitations and Splitting](#bigquery-limitations-and-splitting) for more info.
+
+## Development and Testing
+
+Install requirements:
+```
+make install-requirements
+```
+
+Run tests:
+```
+make test
+```
--- a/config.yml
+++ b/config.yml
@ -0,0 +1,86 @@
+####################
+# CircleCI configuration reference:
+#   https://circleci.com/docs/2.0/configuration-reference
+####################
+
+version: 2
+
+#####################################################
+# Jobs: see https://circleci.com/docs/2.0/jobs-steps/
+#####################################################
+
+jobs:
+  test:
+    docker: # run the steps with Docker
+      - image: circleci/python:3.7.2
+    steps:
+      - checkout
+      - run:
+          name: Run tests
+          command: |
+            python3 -m virtualenv venv
+            . venv/bin/activate
+            make install-requirements
+            make test
+            make lint
+            make coverage
+
+  deploy:
+    docker:
+      - image: docker:18.02.0-ce
+    working_directory: ~/mozilla/mozilla-schema-generator
+    steps:
+      - checkout
+      - setup_remote_docker
+      - run: |
+          printf '{"commit":"%s","version":"%s","source":"https://github.com/%s/%s","build":"%s"}\n' "$CIRCLE_SHA1" "$CIRCLE_TAG" "$CIRCLE_PROJECT_USERNAME" "$CIRCLE_PROJECT_REPONAME" "$CIRCLE_BUILD_URL" > version.json
+      - run: docker build -t app:build .
+      - run:
+          name: Deploy to Dockerhub
+          command: |
+            # Deploy master
+            if [ "${CIRCLE_BRANCH}" == "master" ]; then
+              docker login -u $DOCKER_USER -p $DOCKER_PASS
+              docker tag app:build ${DOCKERHUB_REPO}:latest
+              docker push ${DOCKERHUB_REPO}:latest
+            elif  [ ! -z "${CIRCLE_TAG}" ]; then
+            # Deploy a release tag...
+              docker login -u $DOCKER_USER -p $DOCKER_PASS
+              echo "${DOCKERHUB_REPO}:${CIRCLE_TAG}"
+              docker tag app:build "${DOCKERHUB_REPO}:${CIRCLE_TAG}"
+              docker images
+              docker push "${DOCKERHUB_REPO}:${CIRCLE_TAG}"
+            fi
+
+#########################################################
+# Workflows: see https://circleci.com/docs/2.0/workflows/
+#########################################################
+
+workflows:
+  version: 2
+  commit:
+    jobs:
+      - test:
+          filters:
+            tags:
+              only: /.*/
+
+      - deploy:
+          requires:
+            - test
+          filters:
+            tags:
+              only: /.*/
+            branches:
+              only: master
+
+  nightly: # Test nightly to test external dependencies (MPS, probe-info-service)
+    triggers:
+      - schedule:
+          cron: "0 0 * * *"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - test
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -0,0 +1,12 @@
+version: '2'
+
+services:
+  app:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    restart: "no"
+    command: "true"
+    environment:
+      - GCLOUD_SERVICE_KEY
+      - GOOGLE_APPLICATION_CREDENTIALS
--- a/python_application/init.py
+++ b/python_application/init.py
@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+__author__ = 'Frank Bertsch'
+__email__ = 'frank@mozilla.com'
+__version__ = '0.1.0'
--- a/python_application/main.py
+++ b/python_application/main.py
@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import click
+import logging
+import sys
+
+from .application import App
+
+
+@click.command()
+def hello_world():
+    app = App()
+    print(app.get_hello_world())
+
+
+@click.group()
+def main(args=None):
+    """Command line utility"""
+    logging.basicConfig(stream=sys.stderr, level=logging.INFO)
+
+
+main.add_command(hello_world)
+
+
+if __name__ == "__main__":
+    sys.exit(main())
--- a/python_application/application.py
+++ b/python_application/application.py
@ -0,0 +1,8 @@
+
+class App(object):
+    
+    def __init__(self):
+        pass
+
+    def get_hello_world(self):
+        return "Hello, World"
--- a/requirements/requirements.txt
+++ b/requirements/requirements.txt
--- a/requirements/test_requirements.txt
+++ b/requirements/test_requirements.txt
@ -0,0 +1 @@
+pytest==5.2.0
--- a/setup.py
+++ b/setup.py
@ -0,0 +1,37 @@
+#!/usr/bin/env python
+
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import sys
+
+from setuptools import setup, find_packages
+
+readme = open('README.md').read()
+
+setup(
+    name='generic-docker-python',
+    python_requires='>=3.6.0',
+    version='0.0.0',
+    description='A sample application',
+    long_description=readme,
+    author='Frank Bertsch',
+    author_email='frank@mozilla.com',
+    url='https://github.com/fbertsch/generic-docker-python',
+    packages=find_packages(include=['python_application']),
+    package_dir={'python-application': 'python_application'},
+    entry_points={
+        'console_scripts': [
+            'python-application=python_application.__main__:main',
+        ],
+    },
+    include_package_data=True,
+    install_requires=[
+        'click',
+    ],
+    license='Mozilla',
+)
--- a/tests/test_application.py
+++ b/tests/test_application.py
@ -0,0 +1,14 @@
+import pytest
+
+from python_application.application import App
+
+
+@pytest.fixture
+def app():
+    return App()
+
+
+class TestApplication(object):
+
+    def test_return_value(self, app):
+        assert app.get_hello_world() == "Hello, World"