Merge pull request #241 from mozilla/bug-1076774-implement-buildapi-queue

Bug 1076774 - use a separate worker for jobs ingestion

bin/run_celery_worker_buildapi

@@ -0,0 +1,25 @@
+#!/bin/bash
+curr_dir=$( dirname "${BASH_SOURCE[0]}" )
+cd $( dirname $curr_dir)
+source /etc/profile.d/treeherder.sh
+
+if [[ ${NEW_RELIC_LICENSE_KEY+isset} = isset ]]; then
+    NEWRELIC_ADMIN="newrelic-admin run-program"
+fi
+
+if [ -f ../venv/bin/python ]; then
+    source ../venv/bin/activate
+    PYTHON=../venv/bin/python
+else
+    PYTHON=python
+fi
+
+LOGFILE=/var/log/celery/celery_worker_buildapi.log
+
+if [ ! -f $LOGFILE ]; then
+    touch $LOGFILE
+fi
+
+exec $NEWRELIC_ADMIN $PYTHON manage.py celery worker -Q buildapi \
+    --concurrency=5 --logfile=$LOGFILE -l INFO \
+    --maxtasksperchild=500

deployment/supervisord/etl_node.conf

@@ -17,3 +17,13 @@ priority=998
 stdout_logfile=/var/log/celery/worker_pushlog.log
 stderr_logfile=/var/log/celery/worker_pushlog_err.log
 
+[program:celery_buildapi]
+command=/home/vagrant/treeherder-service/bin/run_celery_worker_buildapi
+user=vagrant
+autostart=true
+autorestart=true
+startsecs=10
+stopwaitsecs = 600
+priority=998
+stdout_logfile=/var/log/celery/worker_buildapi.log
+stderr_logfile=/var/log/celery/worker_buildapi_err.log

treeherder/model/fixtures/tasks.json

@@ -90,7 +90,7 @@
         "routing_key": null,
         "crontab": null,
         "interval": 2,
-        "queue": null,
+        "queue": "buildapi",
         "total_run_count": 0,
         "expires": null,
         "kwargs": "{}",
@@ -132,7 +132,7 @@
         "routing_key": null,
         "crontab": null,
         "interval": 2,
-        "queue": null,
+        "queue": "buildapi",
         "total_run_count": 0,
         "expires": null,
         "kwargs": "{}",
@@ -153,7 +153,7 @@
         "routing_key": null,
         "crontab": null,
         "interval": 1,
-        "queue": null,
+        "queue": "buildapi",
         "total_run_count": 0,
         "expires": null,
         "kwargs": "{}",

treeherder/settings/base.py

@@ -192,6 +192,7 @@ CELERY_QUEUES = (
     # queue for mirroring the sheriffing activity to tbpl
     Queue('high_priority', Exchange('default'), routing_key='high_priority'),
     Queue('pushlog', Exchange('default'), routing_key='pushlog'),
+    Queue('buildapi', Exchange('default'), routing_key='buildapi')
 )
 
 CELERY_ACCEPT_CONTENT = ['json']