Implement deployment to Heroku (#458)

2019-05-23 20:39:02 +02:00 · 2019-05-23 20:39:02 +02:00 · 1bae5834ab
--- a/.isort.cfg
+++ b/.isort.cfg
@ -4,4 +4,4 @@ include_trailing_comma=True
 force_grid_wrap=0
 use_parentheses=True
 line_length=88
-known_third_party = dateutil,flask,hglib,imblearn,jsone,libmozdata,numpy,pandas,pytest,requests,responses,setuptools,shap,sklearn,taskcluster,tqdm,xgboost,yaml,zstandard
+known_third_party = dateutil,flask,hglib,imblearn,jsone,libmozdata,models,numpy,pandas,pytest,requests,responses,setuptools,shap,sklearn,taskcluster,tqdm,xgboost,yaml,zstandard
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@ -132,7 +132,7 @@ tasks:
          capabilities:
            privileged: true
          maxRunTime: 10800
-          image: mozilla/taskboot:0.1.2
+          image: mozilla/taskboot:0.1.3
          command:
            - "/bin/sh"
            - "-lcxe"
@ -210,7 +210,7 @@ tasks:
              taskclusterProxy:
                true
            maxRunTime: 3600
-            image: mozilla/taskboot:0.1.2
+            image: mozilla/taskboot:0.1.3
            env:
              TASKCLUSTER_SECRET: project/relman/bugbug/deploy
            command:
@ -242,7 +242,7 @@ tasks:
              taskclusterProxy:
                true
            maxRunTime: 3600
-            image: mozilla/taskboot:0.1.2
+            image: mozilla/taskboot:0.1.3
            env:
              GIT_REPOSITORY: ${repository}
              GIT_REVISION: ${head_rev}
@ -277,7 +277,7 @@ tasks:
              taskclusterProxy:
                true
            maxRunTime: 3600
-            image: mozilla/taskboot:0.1.2
+            image: mozilla/taskboot:0.1.3
            env:
              GIT_REPOSITORY: ${repository}
              GIT_REVISION: ${head_rev}
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -62,6 +62,7 @@ services:
    image: mozilla/bugbug-http-service
    environment:
      - BUGBUG_BUGZILLA_TOKEN
+      - PORT=8000
    ports:
      - target: 8000
        published: 8000
--- a/http_service/Dockerfile
+++ b/http_service/Dockerfile
@ -18,4 +18,4 @@ ENV CHECK_MODELS="${CHECK_MODELS}"

 RUN python /code/bugbug_http_service/check_models.py

-CMD ["gunicorn", "-b", "0.0.0.0:8000", "bugbug_http_service.app", "--preload", "--timeout", "30", "-w", "3"]
+CMD gunicorn -b 0.0.0.0:$PORT bugbug_http_service.app --preload --timeout 30 -w 3
--- a/http_service/check_models.py
+++ b/http_service/check_models.py
@ -7,30 +7,15 @@ import logging
 import os
 import sys

-from bugbug.models.component import ComponentModel
-from bugbug.models.defect_enhancement_task import DefectEnhancementTaskModel
-from bugbug.models.regression import RegressionModel
+# Non-relative imports might be brittle
+from models import MODELS_NAMES, load_model

 logging.basicConfig(level=logging.INFO)
 LOGGER = logging.getLogger()

-MODELS = {
-    "defectenhancementtask": DefectEnhancementTaskModel,
-    "component": ComponentModel,
-    "regression": RegressionModel,
-}
-MODELS_DIR = os.path.join(os.path.dirname(__file__), "models")
-
-
-def load_model(model):
-    model_file_path = os.path.join(MODELS_DIR, f"{model}model")
-    LOGGER.info(f"Lookup model in {model_file_path}")
-    model = MODELS[model].load(model_file_path)
-    return model
-

 def check_models():
-    for model_name in MODELS.keys():
+    for model_name in MODELS_NAMES:
        # Try loading the model
        load_model(model_name)

@ -47,7 +32,7 @@ if __name__ == "__main__":
        check_models()
    except Exception:
        LOGGER.warning(
-            "Failed to validate the models, please run `python models.py download`",
+            "Failed to validate the models, please run `python download_models.py`",
            exc_info=True,
        )
        sys.exit(1)
--- a/http_service/download_models.py
+++ b/http_service/download_models.py
@ -3,57 +3,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.

-import logging
-import lzma
-import os
-import shutil
-from urllib.request import urlretrieve
-
-import requests
-
-logging.basicConfig(level=logging.INFO)
-LOGGER = logging.getLogger()
-
-BASE_URL = "https://index.taskcluster.net/v1/task/project.relman.bugbug.train_{}.latest/artifacts/public"
-
-MODELS_NAMES = ("defectenhancementtask", "component", "regression")
-MODELS_DIR = os.path.join(os.path.dirname(__file__), "models")
-
-
-def retrieve_model(name):
-    os.makedirs(MODELS_DIR, exist_ok=True)
-
-    file_name = f"{name}model"
-    file_path = os.path.join(MODELS_DIR, file_name)
-
-    base_model_url = BASE_URL.format(name)
-    model_url = f"{base_model_url}/{file_name}.xz"
-    LOGGER.info(f"Checking ETAG of {model_url}")
-    r = requests.head(model_url, allow_redirects=True)
-    r.raise_for_status()
-    new_etag = r.headers["ETag"]
-
-    try:
-        with open(f"{file_path}.etag", "r") as f:
-            old_etag = f.read()
-    except IOError:
-        old_etag = None
-
-    if old_etag != new_etag:
-        LOGGER.info(f"Downloading the model from {model_url}")
-        urlretrieve(model_url, f"{file_path}.xz")
-
-        with lzma.open(f"{file_path}.xz", "rb") as input_f:
-            with open(file_path, "wb") as output_f:
-                shutil.copyfileobj(input_f, output_f)
-                LOGGER.info(f"Written model in {file_path}")
-
-        with open(f"{file_path}.etag", "w") as f:
-            f.write(new_etag)
-    else:
-        LOGGER.info(f"ETAG for {model_url} is ok")
-
-    return file_path
+# Non-relative imports might be brittle
+from models import MODELS_NAMES, retrieve_model


 def preload_models():
--- a/http_service/models.py
+++ b/http_service/models.py
@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import logging
+import lzma
+import os
+import shutil
+from urllib.request import urlretrieve
+
+import requests
+
+from bugbug.models import load_model as bugbug_load_model
+
+logging.basicConfig(level=logging.INFO)
+LOGGER = logging.getLogger()
+
+MODELS_NAMES = ["defectenhancementtask", "component", "regression"]
+MODELS_DIR = os.path.join(os.path.dirname(__file__), "models")
+BASE_URL = "https://index.taskcluster.net/v1/task/project.relman.bugbug.train_{}.latest/artifacts/public"
+
+
+def load_model(model):
+    # TODO: Do not crash when the asked model is not one of the trained models
+    return bugbug_load_model(model, MODELS_DIR)
+
+
+def retrieve_model(name):
+    os.makedirs(MODELS_DIR, exist_ok=True)
+
+    file_name = f"{name}model"
+    file_path = os.path.join(MODELS_DIR, file_name)
+
+    base_url = BASE_URL.format(name)
+    model_url = f"{base_url}/{file_name}.xz"
+    LOGGER.info(f"Checking ETAG of {model_url}")
+    r = requests.head(model_url, allow_redirects=True)
+    r.raise_for_status()
+    new_etag = r.headers["ETag"]
+
+    try:
+        with open(f"{file_path}.etag", "r") as f:
+            old_etag = f.read()
+    except IOError:
+        old_etag = None
+
+    if old_etag != new_etag:
+        LOGGER.info(f"Downloading the model from {model_url}")
+        urlretrieve(model_url, f"{file_path}.xz")
+
+        with lzma.open(f"{file_path}.xz", "rb") as input_f:
+            with open(file_path, "wb") as output_f:
+                shutil.copyfileobj(input_f, output_f)
+                LOGGER.info(f"Written model in {file_path}")
+
+        with open(f"{file_path}.etag", "w") as f:
+            f.write(new_etag)
+    else:
+        LOGGER.info(f"ETAG for {model_url} is ok")
+
+    return file_path
--- a/infra/data-pipeline.yml
+++ b/infra/data-pipeline.yml
@ -263,7 +263,7 @@ tasks:
      capabilities:
        privileged: true
      maxRunTime: 3600
-      image: mozilla/taskboot:0.1.2
+      image: mozilla/taskboot:0.1.3
      command:
        - "/bin/sh"
        - "-lcxe"
@ -304,7 +304,7 @@ tasks:
        taskclusterProxy:
          true
      maxRunTime: 3600
-      image: mozilla/taskboot:0.1.2
+      image: mozilla/taskboot:0.1.3
      env:
        TASKCLUSTER_SECRET: project/relman/bugbug/deploy
      command:
@ -319,3 +319,38 @@ tasks:
      description: bugbug docker http push
      owner: release-mgmt-analysis@mozilla.com
      source: https://github.com/mozilla/bugbug/raw/master/data-pipeline.yml
+
+  - ID: http-deploy
+    dependencies:
+      - docker-build
+    scopes:
+      - secrets:get:project/relman/bugbug/deploy
+    created: {$fromNow: ''}
+    deadline: {$fromNow: '120 hours'}
+    expires: {$fromNow: '1 month'}
+    provisionerId: aws-provisioner-v1
+    workerType: github-worker
+    payload:
+      features:
+        taskclusterProxy:
+          true
+      maxRunTime: 3600
+      image: mozilla/taskboot:0.1.3
+      env:
+        TASKCLUSTER_SECRET: project/relman/bugbug/deploy
+      command:
+        - taskboot
+        - deploy-heroku
+        - --artifact-filter
+        - public/bugbug/bugbug-http-service.tar
+        - --heroku-app
+        - bugbug
+
+    routes:
+      - notify.email.release-mgmt-analysis@mozilla.com.on-failed"
+      - notify.irc-channel.#bugbug.on-failed
+    metadata:
+      name: bugbug docker http service deploy
+      description: bugbug docker http service deploy
+      owner: release-mgmt-analysis@mozilla.com
+      source: https://github.com/mozilla/bugbug/raw/master/data-pipeline.yml