Use the new bugbug_http module, fixing tests and docker build

.dockerignore

@@ -97,5 +97,4 @@ venv/
 # Project-specific stuff
 cache/
 data/
-http_service/
 .taskcluster.yml

.isort.cfg

@@ -2,4 +2,5 @@
 multi_line_output=3
 include_trailing_comma=True
 line_length=88
-known_third_party = adr,apispec,apispec_webframeworks,boto3,cerberus,dateutil,flask,flask_cors,hglib,imblearn,joblib,jsone,jsonschema,libmozdata,lmdb,marshmallow,matplotlib,microannotate,models,mozci,numpy,orjson,pandas,pkg_resources,pyemd,pytest,redis,requests,responses,rq,scipy,setuptools,shap,sklearn,tabulate,taskcluster,tenacity,tqdm,xgboost,yaml,zstandard
+known_first_party = bugbug,bugbug_http
+known_third_party = adr,apispec,apispec_webframeworks,boto3,cerberus,dateutil,flask,flask_cors,hglib,imblearn,joblib,jsone,jsonschema,libmozdata,lmdb,marshmallow,matplotlib,microannotate,mozci,numpy,orjson,pandas,pkg_resources,pyemd,pytest,redis,requests,responses,rq,scipy,setuptools,shap,sklearn,tabulate,taskcluster,tenacity,tqdm,xgboost,yaml,zstandard

.pre-commit-config.yaml

@@ -50,6 +50,15 @@ repos:
     rev: v0.761
     hooks:
     -   id: mypy
+        name: mypy-bugbug
+        files: ^bugbug/
+        entry: mypy bugbug/
+        pass_filenames: false
+    -   id: mypy
+        name: mypy-bugbug-http
+        files: ^http_service/
+        entry: mypy http_service/
+        pass_filenames: false
 -   repo: meta
     hooks:
     -   id: check-useless-excludes

.taskcluster.yml

@@ -130,7 +130,7 @@ tasks:
                git -c advice.detachedHead=false checkout ${head_rev} &&
                pip install --disable-pip-version-check --quiet --no-cache-dir . &&
                pip install --disable-pip-version-check --quiet --no-cache-dir -r test-requirements.txt &&
-               pip install --disable-pip-version-check --quiet --no-cache-dir -r http_service/requirements.txt &&
+               pip install --disable-pip-version-check --quiet --no-cache-dir ./http_service &&
                pytest --cov=http_service http_service/tests/ -vvv &&
                bash <(curl -s https://codecov.io/bash)"
         metadata:

http_service/Dockerfile

@@ -2,13 +2,12 @@ ARG BUGBUG_VERSION=latest
 
 FROM mozilla/bugbug-base:$BUGBUG_VERSION
 
-COPY requirements.txt /code/http_service/
+# Install dependencies first
+COPY requirements.txt /requirements-http.txt
+RUN pip install --disable-pip-version-check --quiet --no-cache-dir -r /requirements-http.txt
 
-RUN pip install --disable-pip-version-check --quiet --no-cache-dir -r /code/http_service/requirements.txt
+# Setup http service as package
+COPY . /code/http_service
+RUN pip install --disable-pip-version-check --quiet --no-cache-dir /code/http_service
 
-COPY . /code/http_service/
-
-# Load the models
-WORKDIR /code/
-
-CMD gunicorn -b 0.0.0.0:$PORT http_service.app --preload --timeout 30 -w 3
+CMD gunicorn -b 0.0.0.0:$PORT bugbug_http.app --preload --timeout 30 -w 3

http_service/Dockerfile.bg_worker

@@ -2,11 +2,13 @@ ARG BUGBUG_VERSION=latest
 
 FROM mozilla/bugbug-base:$BUGBUG_VERSION
 
-COPY requirements.txt /code/http_service/
+# Install dependencies first
+COPY requirements.txt /requirements-http.txt
+RUN pip install --disable-pip-version-check --quiet --no-cache-dir -r /requirements-http.txt
 
-RUN pip install --disable-pip-version-check --quiet --no-cache-dir -r /code/http_service/requirements.txt
-
-COPY . /code/http_service/
+# Setup http service as package
+COPY . /code/http_service
+RUN pip install --disable-pip-version-check --quiet --no-cache-dir /code/http_service
 
 # Load the models
 WORKDIR /code/
@@ -16,4 +18,4 @@ ENV CHECK_MODELS="${CHECK_MODELS}"
 
 RUN bash /code/http_service/ensure_models.sh
 
-CMD python /code/http_service/worker.py high default low
+CMD bugbug-http-worker high default low

http_service/MANIFEST.in

@@ -0,0 +1 @@
+include bugbug_http/templates/*.html

http_service/bugbug_http/app.py

@@ -22,9 +22,8 @@ from rq.exceptions import NoSuchJobError
 from rq.job import Job
 
 from bugbug import get_bugbug_version
-
-from .models import MODELS_NAMES, change_time_key, classify_bug, result_key
-from .utils import get_bugzilla_http_client
+from bugbug_http.models import MODELS_NAMES, change_time_key, classify_bug, result_key
+from bugbug_http.utils import get_bugzilla_http_client
 
 API_TOKEN = "X-Api-Key"
 

http_service/bugbug_http/check_models.py

@@ -7,8 +7,7 @@ import logging
 import os
 import sys
 
-# Non-relative imports might be brittle
-from models import MODELS_NAMES, get_model
+from bugbug_http.models import MODELS_NAMES, get_model
 
 logging.basicConfig(level=logging.INFO)
 LOGGER = logging.getLogger()

http_service/bugbug_http/download_models.py

@@ -3,8 +3,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# Non-relative imports might be brittle
-from models import MODELS_NAMES, retrieve_model
+from bugbug_http.models import MODELS_NAMES, retrieve_model
 
 
 def preload_models():

http_service/bugbug_http/worker.py

@@ -6,29 +6,28 @@
 
 import os
 import sys
-from os.path import abspath, dirname, join
 
 from redis import Redis
 from rq import Connection, Worker
 
-sys.path.insert(0, abspath(join(dirname(__file__), "..")))
+import bugbug_http.models
 
-# We need to make sure that the models.py file is imported with the same name
-# than in the HTTP docker container or the cache will not match. If we import
-# it as models, the cache will be located at
-# `sys.modules['models'].MODEL_CACHE` while the job will look the models in
-# `sys.modules['http_service.models'].MODEL_CACHE`
-import http_service.models  # noqa: E402 isort:skip
 
-# Preload libraries
-http_service.models.preload_models()
+def main():
 
-# Provide queue names to listen to as arguments to this script,
-# similar to rq worker
-redis_url = os.environ.get("REDIS_URL", "redis://localhost/0")
-redis_conn = Redis.from_url(redis_url)
-with Connection(connection=redis_conn):
-    qs = sys.argv[1:] or ["default"]
+    # Preload libraries
+    bugbug_http.models.preload_models()
 
-    w = Worker(qs)
-    w.work()
+    # Provide queue names to listen to as arguments to this script,
+    # similar to rq worker
+    redis_url = os.environ.get("REDIS_URL", "redis://localhost/0")
+    redis_conn = Redis.from_url(redis_url)
+    with Connection(connection=redis_conn):
+        qs = sys.argv[1:] or ["default"]
+
+        w = Worker(qs)
+        w.work()
+
+
+if __name__ == "__main__":
+    main()

http_service/ensure_models.sh

@@ -8,13 +8,11 @@
 
 set -eox pipefail
 
-CURRENT_DIR=$(dirname "$0")
-
 if [ "$CHECK_MODELS" == "0" ]; then
     echo "Skipping downloading and checking models!"
     exit 0;
 fi
 
-python "$CURRENT_DIR/download_models.py"
+python -m bugbug_http.download_models
 
-python "$CURRENT_DIR/check_models.py"
+python -m bugbug_http.check_models

http_service/setup.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+
+import pkg_resources
+from setuptools import find_packages, setup
+
+here = os.path.dirname(__file__)
+
+
+def read_requirements(file_):
+    with open(os.path.join(here, file_)) as f:
+        return sorted(list(set(line.split("#")[0].strip() for line in f)))
+
+
+install_requires = read_requirements("requirements.txt")
+
+# Use same version as bugbug
+version = pkg_resources.get_distribution("bugbug").version
+
+setup(
+    name="bugbug-http-service",
+    version=version,
+    description="ML tools for Mozilla projects",
+    author="Marco Castelluccio",
+    author_email="mcastelluccio@mozilla.com",
+    install_requires=install_requires,
+    packages=find_packages(),
+    include_package_data=True,
+    license="MPL2",
+    entry_points={"console_scripts": ["bugbug-http-worker = bugbug_http.worker:main"]},
+    classifiers=[
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3 :: Only",
+        "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
+    ],
+)

http_service/tests/test_app.py

@@ -7,10 +7,7 @@ import json
 
 import pytest
 
-from http_service.app import (  # TODO: Move http_service under bugbug to solve this import name
-    API_TOKEN,
-    application,
-)
+from bugbug_http.app import API_TOKEN, application
 
 
 @pytest.fixture

scripts/integration_test.sh

@@ -58,11 +58,11 @@ sleep 1
 redis-cli -n 4 FLUSHDB
 
 # Start the http server
-gunicorn -b 127.0.0.1:8000 http_service.app --preload --timeout 30 -w 3 &
+gunicorn -b 127.0.0.1:8000 bugbug_http.app --preload --timeout 30 -w 3 &
 gunicorn_pid=$!
 
 # Start the background worker
-env BUGBUG_ALLOW_MISSING_MODELS=1 python http_service/worker.py high default low &
+env BUGBUG_ALLOW_MISSING_MODELS=1 bugbug-http-worker high default low &
 worker_pid=$!
 
 # Ensure we take down the containers at the end