Bug 1138392 - Deploy script: Make IRC output more accurate

Anything in the deploy script decorated with @task or @hostgroups ends
up having it's progress displayed in IRC by the bot - eg:
[2015-02-23 20:09:40] Finished deploy_log (0.454s)

However, the "deploy_X" @hostgroup functions were actually only
rsyncing the code (which is fast compared to restarting the processes),
but the name used to imply the whole deploy for that node type was
complete.

The new structure not only reduces code duplication, but makes the IRC
output more accurate, to provide a better idea as to what stage the
deploy is at, and which are the problematic steps in the case of errors.

deployment/update/update.py

@@ -3,7 +3,7 @@
 # file, you can obtain one at http://mozilla.org/MPL/2.0/.
 
 """
-Deploy this project in dev/stage/production.
+Deploy this project in stage/production.
 
 Requires commander_ which is installed on the systems that need it.
 
@@ -19,7 +19,6 @@ th_service_src = os.path.join(settings.SRC_DIR, 'treeherder-service')
 th_ui_src = os.path.join(settings.SRC_DIR, 'treeherder-ui')
 
 is_prod = 'treeherder.mozilla.org' in settings.SRC_DIR
-env_flag = '-p' if is_prod else '-s'
 
 
 @task
@@ -46,22 +45,16 @@ def update(ctx):
     with ctx.lcd(th_service_src):
         # Collect the static files (eg for the Persona or Django admin UI)
         ctx.local("python2.7 manage.py collectstatic --noinput")
-
         # Rebuild the Cython code (eg the log parser)
         ctx.local("python2.7 setup.py build_ext --inplace")
-
         # Update the database schema, if necessary.
         ctx.local("python2.7 manage.py migrate")
-
         # Update reference data & tasks config from the in-repo fixtures.
         ctx.local("python2.7 manage.py load_initial_data")
-
         # Populate the datasource table and create the connected databases.
         ctx.local("python2.7 manage.py init_datasources")
-
         # Update oauth credentials.
         ctx.local("python2.7 manage.py export_project_credentials")
-
         # Clear the cache.
         ctx.local("python2.7 manage.py clear_cache")
 
@@ -70,39 +63,44 @@ def update(ctx):
 def deploy(ctx):
     # Use the local, IT-written deploy script to check in changes.
     ctx.local(settings.DEPLOY_SCRIPT)
+    # Rsync the updated code to the nodes & restart processes. These are
+    # separated out into their own functions, since the IRC bot output includes
+    # the task function name which is useful given how long these steps take.
+    deploy_rabbit(ctx)
+    deploy_web_app(ctx)
+    deploy_etl(ctx)
+    deploy_log(ctx)
 
-    def restart_jobs(ctx, type):
-        ctx.local('/root/bin/restart-jobs %s %s' % (env_flag, type))
 
-    # Restart celerybeat on the admin node.
-    @hostgroups(settings.RABBIT_HOSTGROUP, remote_kwargs={'ssh_key': settings.SSH_KEY})
-    def deploy_rabbit(ctx):
+@task
+def deploy_rabbit(ctx):
+    deploy_nodes(ctx, settings.RABBIT_HOSTGROUP, 'rabbit')
+
+
+@task
+def deploy_web_app(ctx):
+    deploy_nodes(ctx, settings.WEB_HOSTGROUP, 'web')
+
+
+@task
+def deploy_etl(ctx):
+    deploy_nodes(ctx, settings.ETL_HOSTGROUP, 'etl')
+
+
+@task
+def deploy_log(ctx):
+    deploy_nodes(ctx, settings.LOG_HOSTGROUP, 'log')
+
+
+def deploy_nodes(ctx, hostgroup, node_type):
+    # Run the remote update script on each node in the specified hostgroup.
+    @hostgroups(hostgroup, remote_kwargs={'ssh_key': settings.SSH_KEY})
+    def rsync_code(ctx, node_type):
         ctx.remote(settings.REMOTE_UPDATE_SCRIPT)
+        if node_type == 'web':
+            # TODO: Move this to the restart-jobs script.
+            ctx.remote('{0}/service httpd graceful'.format(settings.SBIN_DIR))
 
-    deploy_rabbit()
-    restart_jobs(ctx, 'rabbit')
-
-    @hostgroups(settings.WEB_HOSTGROUP, remote_kwargs={'ssh_key': settings.SSH_KEY})
-    def deploy_web_app(ctx):
-        # Call the remote update script to push changes to webheads.
-        ctx.remote(settings.REMOTE_UPDATE_SCRIPT)
-        ctx.remote('{0}/service httpd graceful'.format(settings.SBIN_DIR))
-
-    deploy_web_app()
-    restart_jobs(ctx, 'web')
-
-    @hostgroups(settings.ETL_HOSTGROUP, remote_kwargs={'ssh_key': settings.SSH_KEY})
-    def deploy_etl(ctx):
-        # Call the remote update script to push changes to workers.
-        ctx.remote(settings.REMOTE_UPDATE_SCRIPT)
-
-    deploy_etl()
-    restart_jobs(ctx, 'etl')
-
-    @hostgroups(settings.LOG_HOSTGROUP, remote_kwargs={'ssh_key': settings.SSH_KEY})
-    def deploy_log(ctx):
-        # Call the remote update script to push changes to workers.
-        ctx.remote(settings.REMOTE_UPDATE_SCRIPT)
-
-    deploy_log()
-    restart_jobs(ctx, 'log')
+    rsync_code(ctx, node_type)
+    env_flag = '-p' if is_prod else '-s'
+    ctx.local('/root/bin/restart-jobs %s %s' % (env_flag, node_type))