Ensure the integration tests are green before deploying a new HTTP service (#979)

Fixes #949

.taskcluster.yml

@@ -437,7 +437,7 @@ tasks:
                 apt-get update &&
                 apt-get install -y redis-server &&
                 python -c 'import os; print(os.environ.keys())' &&
-                bash ./scripts/integration_test.sh || true"
+                bash ./scripts/integration_test.sh"
           metadata:
             name: bugbug integration test
             description: bugbug integration test

infra/data-pipeline.yml

@@ -633,9 +633,44 @@ tasks:
         owner: release-mgmt-analysis@mozilla.com
         source: https://github.com/mozilla/bugbug/raw/master/data-pipeline.yml
 
+    - ID: integration-test
+      created: {$fromNow: ''}
+      deadline: {$fromNow: '119 hours'}
+      expires: {$fromNow: '1 month'}
+      provisionerId: aws-provisioner-v1
+      workerType: relman-svc
+      scopes:
+        - secrets:get:project/relman/bugbug/integration
+      payload:
+        features:
+          taskclusterProxy:
+            true
+        maxRunTime: 10800
+        image: mozilla/bugbug-commit-retrieval:latest
+        env:
+          TC_SECRET_ID: project/relman/bugbug/integration
+        command:
+          - "/bin/bash"
+          - "-lcx"
+          - "git clone --quiet https://github.com/mozilla/bugbug &&
+            cd bugbug &&
+            git checkout ${version} &&
+            pip install --quiet . &&
+            pip install --quiet -r test-requirements.txt &&
+            apt-get update &&
+            apt-get install -y redis-server &&
+            python -c 'import os; print(os.environ.keys())' &&
+            bash ./scripts/integration_test.sh"
+      metadata:
+        name: bugbug integration test
+        description: bugbug integration test
+        owner: release-mgmt-analysis@mozilla.com
+        source: https://github.com/mozilla/bugbug/raw/master/data-pipeline.yml
+
     - ID: docker-push
       dependencies:
         - docker-build
+        - integration-test
       scopes:
         - secrets:get:project/relman/bugbug/deploy
       created: {$fromNow: ''}

scripts/integration_test.sh

@@ -8,9 +8,10 @@ set -euox pipefail
 
 cd http_service/models/;
 
-# Remove the models
+# Remove the models and any old data
 rm defectenhancementtaskmodel* || true;
 rm backout* || true;
+rm -Rf data || true;
 
 # First retrieve a subset of bugs data
 # TODO: Let the script download the previous DB as it should be pretty fast?