From fa80fe5bc4448fd0c59594c18f84d8b066524aad Mon Sep 17 00:00:00 2001 From: Bastien Abadie Date: Fri, 18 Oct 2019 12:29:51 +0200 Subject: [PATCH] backend: Add django base web application (#161) --- .dockerignore | 2 + .gitignore | 2 + .isort.cfg | 4 +- .taskcluster.yml | 91 ++++++++++ backend/Dockerfile | 15 ++ backend/README.md | 14 ++ backend/VERSION | 1 + backend/code_review_backend/__init__.py | 0 backend/code_review_backend/app/__init__.py | 0 backend/code_review_backend/app/settings.py | 179 ++++++++++++++++++++ backend/code_review_backend/app/urls.py | 13 ++ backend/code_review_backend/app/wsgi.py | 21 +++ backend/manage.py | 26 +++ backend/requirements-dev.txt | 1 + backend/requirements.txt | 10 ++ backend/setup.py | 47 +++++ 16 files changed, 424 insertions(+), 2 deletions(-) create mode 100644 .dockerignore create mode 100644 backend/Dockerfile create mode 100644 backend/README.md create mode 100644 backend/VERSION create mode 100644 backend/code_review_backend/__init__.py create mode 100644 backend/code_review_backend/app/__init__.py create mode 100644 backend/code_review_backend/app/settings.py create mode 100644 backend/code_review_backend/app/urls.py create mode 100644 backend/code_review_backend/app/wsgi.py create mode 100755 backend/manage.py create mode 100644 backend/requirements-dev.txt create mode 100644 backend/requirements.txt create mode 100644 backend/setup.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..57d9cd57 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +*/node_modules +*.sqlite* diff --git a/.gitignore b/.gitignore index 2e7fa2ee..fa99d4b3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ .vscode/ *.pyc *.egg-info +*.sqlite* +backend/hgmo diff --git a/.isort.cfg b/.isort.cfg index 41aa15ce..948a75fc 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,6 +1,6 @@ [settings] -known_first_party = code_review_bot,code_review_tools,code_review_events,conftest -known_third_party = influxdb,libmozdata,libmozevent,logbook,parsepatch,pytest,raven,requests,responses,setuptools,structlog,taskcluster,toml +known_first_party = code_review_backend,code_review_bot,code_review_tools,code_review_events,conftest +known_third_party = dj_database_url,django,influxdb,libmozdata,libmozevent,logbook,parsepatch,pytest,raven,requests,responses,setuptools,structlog,taskcluster,toml force_single_line = True default_section=FIRSTPARTY line_length=159 diff --git a/.taskcluster.yml b/.taskcluster.yml index a4eb5529..713dc086 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -98,6 +98,26 @@ tasks: owner: bastien@mozilla.com source: https://github.com/mozilla/code-review + - taskId: {$eval: as_slugid("backend_check_tests")} + provisionerId: aws-provisioner-v1 + workerType: github-worker + created: {$fromNow: ''} + deadline: {$fromNow: '1 hour'} + payload: + maxRunTime: 3600 + image: python:3 + command: + - sh + - -lxce + - "git clone --quiet ${repository} /src && cd /src && git checkout ${head_rev} -b checks && + cd /src/backend && pip install -q . && pip install -q -r requirements-dev.txt && + ./manage.py test" + metadata: + name: "Code Review Backend checks: unit tests" + description: Check python code with Django tests + owner: bastien@mozilla.com + source: https://github.com/mozilla/code-review + - taskId: {$eval: as_slugid("frontend_build")} provisionerId: aws-provisioner-v1 workerType: github-worker @@ -205,6 +225,47 @@ tasks: owner: bastien@mozilla.com source: https://github.com/mozilla/code-review + - taskId: {$eval: as_slugid("backend_build")} + created: {$fromNow: ''} + deadline: {$fromNow: '1 hour'} + provisionerId: aws-provisioner-v1 + workerType: releng-svc + dependencies: + - {$eval: as_slugid("check_lint")} + - {$eval: as_slugid("backend_check_tests")} + payload: + capabilities: + privileged: true + maxRunTime: 3600 + image: "${taskboot_image}" + env: + GIT_REPOSITORY: ${repository} + GIT_REVISION: ${head_rev} + command: + - taskboot + - build + - --image + - mozilla/code-review + - --tag + - "${channel}" + - --tag + - "${head_rev}" + - --write + - /backend.tar + - backend/Dockerfile + artifacts: + public/code-review-backend.tar: + expires: {$fromNow: '2 weeks'} + path: /backend.tar + type: file + scopes: + - docker-worker:capability:privileged + metadata: + name: Code Review Backend docker build + description: Build docker image of code review backend + owner: bastien@mozilla.com + source: https://github.com/mozilla/code-review + - $if: 'channel in ["testing", "production"]' then: taskId: {$eval: as_slugid("frontend_deploy")} @@ -326,3 +387,33 @@ tasks: description: Deploy docker image on Heroku owner: bastien@mozilla.com source: https://github.com/mozilla/code-review + + - $if: 'channel in ["testing", "production"]' + then: + taskId: {$eval: as_slugid("backend_deploy")} + created: {$fromNow: ''} + deadline: {$fromNow: '1 hour'} + provisionerId: aws-provisioner-v1 + workerType: github-worker + dependencies: + - {$eval: as_slugid("backend_build")} + payload: + features: + taskclusterProxy: true + maxRunTime: 3600 + image: "${taskboot_image}" + command: + - taskboot + - deploy-heroku + - --heroku-app + - "code-review-backend-${channel}" + - web:public/code-review-backend.tar + env: + TASKCLUSTER_SECRET: "project/relman/code-review/deploy-${channel}" + scopes: + - "secrets:get:project/relman/code-review/deploy-${channel}" + metadata: + name: "Code Review Backend deployment (${channel})" + description: Deploy docker image on Heroku + owner: bastien@mozilla.com + source: https://github.com/mozilla/code-review diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 00000000..963b4804 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.7-slim + +ADD backend /src/backend + +WORKDIR /src/backend + +# Activate Django settings for in docker image +ENV DJANGO_DOCKER=true + +RUN pip install --no-cache-dir . + +# Collect all static files +RUN ./manage.py collectstatic --no-input + +CMD gunicorn code_review_backend.app.wsgi diff --git a/backend/README.md b/backend/README.md new file mode 100644 index 00000000..9f51ccbf --- /dev/null +++ b/backend/README.md @@ -0,0 +1,14 @@ +# Code Review Backend + +## Developer setup + +``` +mkvirtualenv -p /usr/bin/python3 code-review-backend +cd backend +pip install -r requirements.txt +./manage.py migrate +./manage.py createsuperuser +./manage.py runserver +``` + +At this point, you can log into http://127.0.0.1:8000/admin/ with the credentials you mentioned during the `createsuperuser` step. diff --git a/backend/VERSION b/backend/VERSION new file mode 100644 index 00000000..ee90284c --- /dev/null +++ b/backend/VERSION @@ -0,0 +1 @@ +1.0.4 diff --git a/backend/code_review_backend/__init__.py b/backend/code_review_backend/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/code_review_backend/app/__init__.py b/backend/code_review_backend/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/code_review_backend/app/settings.py b/backend/code_review_backend/app/settings.py new file mode 100644 index 00000000..124d28f0 --- /dev/null +++ b/backend/code_review_backend/app/settings.py @@ -0,0 +1,179 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Django settings for backend project. + +Generated by 'django-admin startproject' using Django 2.2.6. + +For more information on this file, see +https://docs.djangoproject.com/en/2.2/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/2.2/ref/settings/ +""" + +import logging +import os + +import dj_database_url + +logger = logging.getLogger(__name__) + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +ROOT_DIR = os.path.dirname(BASE_DIR) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = "t!+s!@x5p!85x19q83jufr#95_z0fv7$!u5z*c&gi!%hr3^w+r" + +# Only use DEBUG mode for local development +# When running on Heroku, we disable that mode (see end of file & DYNO mode) +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "rest_framework", +] + +MIDDLEWARE = [ + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", +] + +ROOT_URLCONF = "code_review_backend.app.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ] + }, + } +] + +WSGI_APPLICATION = "code_review_backend.app.wsgi.application" + + +# Database +# https://docs.djangoproject.com/en/2.2/ref/settings/#databases + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": os.path.join(ROOT_DIR, "db.sqlite3"), + } +} + + +# Password validation +# https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator" + }, + {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"}, + {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"}, + {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"}, +] + + +# Internationalization +# https://docs.djangoproject.com/en/2.2/topics/i18n/ + +LANGUAGE_CODE = "en-us" + +TIME_ZONE = "UTC" + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + +# API configuration +REST_FRAMEWORK = { + # Use Django's standard `django.contrib.auth` permissions, + # or allow read-only access for unauthenticated users. + "DEFAULT_PERMISSION_CLASSES": [ + "rest_framework.permissions.IsAuthenticatedOrReadOnly" + ], + # Setup pagination + "PAGE_SIZE": 50, + "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination", +} + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/2.2/howto/static-files/ + +STATIC_URL = "/static/" + +# Static files are set in a dedicated path in Docker image +if "DJANGO_DOCKER" in os.environ: + STATIC_ROOT = "/static" + + # Enable GZip and cache, and build a manifest during collectstatic + STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage" + + +# Internal logging setup +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "handlers": {"console": {"class": "logging.StreamHandler"}}, + "loggers": { + "django": {"handlers": ["console"], "level": "INFO"}, + "code_review_backend": {"handlers": ["console"], "level": "INFO"}, + }, +} + +# Heroku settings override to run the web app in production mode +if "DYNO" in os.environ: + logger.info("Setting up Heroku environment") + ALLOWED_HOSTS = ["*"] + DEBUG = os.environ.get("DEBUG", "false").lower() == "true" + + # Database setup + if "DATABASE_URL" in os.environ: + logger.info("Using remote database from $DATABASE_URL") + DATABASES["default"] = dj_database_url.parse( + os.environ["DATABASE_URL"], ssl_require=True + ) + else: + logger.info("DATABASE_URL not found, will use sqlite. Data may be lost.") + + # Insert Whitenoise Middleware after the security one + MIDDLEWARE.insert(1, "whitenoise.middleware.WhiteNoiseMiddleware") + + # Use Secret key from env + SECRET_KEY = os.environ.get("SECRET_KEY", SECRET_KEY) diff --git a/backend/code_review_backend/app/urls.py b/backend/code_review_backend/app/urls.py new file mode 100644 index 00000000..cfacc87b --- /dev/null +++ b/backend/code_review_backend/app/urls.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from django.contrib import admin +from django.shortcuts import redirect +from django.urls import path + +urlpatterns = [ + path("", lambda request: redirect("admin/", permanent=False)), + path("admin/", admin.site.urls), +] diff --git a/backend/code_review_backend/app/wsgi.py b/backend/code_review_backend/app/wsgi.py new file mode 100644 index 00000000..ca3a145e --- /dev/null +++ b/backend/code_review_backend/app/wsgi.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +WSGI config for backend project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/2.2/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "code_review_backend.app.settings") + +application = get_wsgi_application() diff --git a/backend/manage.py b/backend/manage.py new file mode 100755 index 00000000..fac67871 --- /dev/null +++ b/backend/manage.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "code_review_backend.app.settings") + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == "__main__": + main() diff --git a/backend/requirements-dev.txt b/backend/requirements-dev.txt new file mode 100644 index 00000000..f555665e --- /dev/null +++ b/backend/requirements-dev.txt @@ -0,0 +1 @@ +pre-commit==1.18.3 diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 00000000..0222ff9f --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,10 @@ +Django==2.2.6 +dj-database-url==0.5.0 +djangorestframework==3.10.3 +gunicorn==19.9.0 +psycopg2-binary==2.8.3 +pytz==2019.3 +sqlparse==0.3.0 +taskcluster==19.0.0 +taskcluster-urls==11.0.0 +whitenoise==4.1.4 diff --git a/backend/setup.py b/backend/setup.py new file mode 100644 index 00000000..1dc11cae --- /dev/null +++ b/backend/setup.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import setuptools + + +def read_requirements(file_): + lines = [] + with open(file_) as f: + for line in f.readlines(): + line = line.strip() + if ( + line.startswith("-e ") + or line.startswith("http://") + or line.startswith("https://") + ): + extras = "" + if "[" in line: + extras = "[" + line.split("[")[1].split("]")[0] + "]" + line = line.split("#")[1].split("egg=")[1] + extras + elif line == "" or line.startswith("#") or line.startswith("-"): + continue + line = line.split("#")[0].strip() + lines.append(line) + return sorted(list(set(lines))) + + +with open("VERSION") as f: + VERSION = f.read().strip() + + +setuptools.setup( + name="code_review_backend", + version=VERSION, + description="Store and compare issues found in Mozilla code review tasks", + author="Mozilla Release Management", + author_email="release-mgmt-analysis@mozilla.com", + url="https://github.com/mozilla/code-review", + tests_require=read_requirements("requirements-dev.txt"), + install_requires=read_requirements("requirements.txt"), + packages=setuptools.find_packages(), + include_package_data=True, + zip_safe=False, + license="MPL2", +)