Docker full build using compose (#274)

.gitignore

@@ -1,4 +1,5 @@
 *model
+http_service/models/*model*
 
 data/
 sheets/

.taskcluster.yml

@@ -156,20 +156,19 @@ tasks:
             privileged: true
           maxRunTime: 3600
           image: babadie/taskboot:latest
-          env:
-            GIT_REPOSITORY: ${repository}
-            GIT_REVISION: ${head_rev}
           command:
-            - taskboot
-            - build
-            - infra/dockerfile.base
-            - --write
-            - /base.tar
+            - "/bin/sh"
+            - "-lcxe"
+            - "git clone ${repository} /code &&
+               cd /code &&
+               git checkout ${head_rev} &&
+               python http_service/download_models.py &&
+               taskboot --target /code build-compose --write /images"
           artifacts:
-            public/bugbug-base.tar:
+            public/bugbug:
               expires: {$fromNow: '2 weeks'}
-              path: /base.tar
-              type: file
+              path: /images
+              type: directory
         scopes:
           - docker-worker:capability:privileged
         metadata:

http_service/Dockerfile

@@ -9,8 +9,8 @@ COPY . /code/bugbug_http_service/
 # Load the models
 WORKDIR /code/
 
-COPY models/ /code/models/
+COPY ./models /code/models
 
-RUN python bugbug_http_service/check_models.py
+RUN python /code/bugbug_http_service/check_models.py
 
 CMD ["gunicorn", "-b", "0.0.0.0:8000", "bugbug_http_service.app", "--preload", "--timeout", "30", "-w", "3"]

http_service/check_models.py

@@ -19,17 +19,12 @@ MODELS = {
     "component": ComponentModel,
     "regression": RegressionModel,
 }
-
-
-def get_model_path(name):
-    file_name = f"{name}model"
-    file_path = os.path.join("models", file_name)
-
-    return file_path
+MODELS_DIR = os.path.join(os.path.dirname(__file__), 'models')
 
 
 def load_model(model):
-    model_file_path = get_model_path(model)
+    model_file_path = os.path.join(MODELS_DIR, f"{model}model")
+    LOGGER.info(f'Lookup model in {model_file_path}')
     model = MODELS[model].load(model_file_path)
     return model
 

http_service/download_models.py

@@ -21,12 +21,13 @@ MODELS_NAMES = (
     "component",
     "regression",
 )
+MODELS_DIR = os.path.join(os.path.dirname(__file__), 'models')
 
 def retrieve_model(name):
-    os.makedirs("models", exist_ok=True)
+    os.makedirs(MODELS_DIR, exist_ok=True)
 
     file_name = f"{name}model"
-    file_path = os.path.join("models", file_name)
+    file_path = os.path.join(MODELS_DIR, file_name)
 
     model_url = f"{BASE_URL}/{file_name}.xz"
     LOGGER.info(f"Checking ETAG of {model_url}")
@@ -47,6 +48,7 @@ def retrieve_model(name):
         with lzma.open(f"{file_path}.xz", "rb") as input_f:
             with open(file_path, "wb") as output_f:
                 shutil.copyfileobj(input_f, output_f)
+                LOGGER.info(f'Written model in {file_path}')
 
         with open(f"{file_path}.etag", "w") as f:
             f.write(new_etag)

infra/dockerfile.base

@@ -1,9 +1,12 @@
+#syntax=docker/dockerfile:experimental
 FROM python:3.7.3
 
 COPY requirements.txt /code/bugbug/
 
-RUN pip install --disable-pip-version-check --no-cache-dir -r /code/bugbug/requirements.txt
+RUN --mount=type=cache,target=/cache/pip \
+	pip install --disable-pip-version-check --cache-dir=/cache/pip -r /code/bugbug/requirements.txt
 
 COPY . /code/bugbug/
 
-RUN pip install /code/bugbug/
+RUN --mount=type=cache,target=/cache/pip \
+	pip install --cache-dir=/cache/pip /code/bugbug/